From bab74086c50879ba646d339eb2e952a5237d1f47 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 19 May 2025 14:14:51 -0300 Subject: [PATCH 01/28] Adding new observability fixes --- README.md | 2 +- app.py | 0 ingestor/ingestor.py | 14 +++++++++++++- values.yaml | 15 ++++++++++----- 4 files changed, 24 insertions(+), 7 deletions(-) delete mode 100644 app.py diff --git a/README.md b/README.md index 0a90636..415658b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# sientia-dataops-opc-gateway +# sientia-dataops-opc-ingestor OPC gateway to manage Scouter pipelines ## Local tests diff --git a/app.py b/app.py deleted file mode 100644 index e69de29..0000000 diff --git a/ingestor/ingestor.py b/ingestor/ingestor.py index 3d39b35..45b33f8 100644 --- a/ingestor/ingestor.py +++ b/ingestor/ingestor.py @@ -210,19 +210,29 @@ class Ingestor: self.ingestor_manager.declare_active() self.logger.info("Polling for slot updates...") - # Get active ingestors + # Get active ingestors[ + ingestors = self.ingestor_manager.get_active_ingestors() + self.logger.debug(f"Active ingestors: {ingestors}") number_of_ingestors = len(ingestors) + self.logger.debug(f"Number of ingestors: {number_of_ingestors}") number_of_leases = self.ingestor_manager.get_number_of_leases() + self.logger.debug(f"Number of leases: {number_of_leases}") number_of_slots = self.ingestor_manager.get_number_of_slots() + self.logger.debug(f"Number of slots: {number_of_slots}") # Handle no slots + self.logger.debug("Managing no slots...") self.manage_no_slots(number_of_slots) available_slots = number_of_slots - number_of_leases + self.logger.debug(f"Available slots: {available_slots}") lacking_ingestors = number_of_slots - number_of_ingestors + self.logger.debug(f"Lacking ingestors: {lacking_ingestors}") slot_diff = len(self.ingestor_manager.managed_tags) - 1 + self.logger.debug(f"Slot diff: {slot_diff}") + self.logger.debug("Managing leases...") self.manage_leases(available_slots, lacking_ingestors, slot_diff) self.logger.debug( @@ -237,7 +247,9 @@ class Ingestor: self.logger.info("No slots acquired in this loop") # Update opc servers + self.logger.debug("Updating slot config...") self.ingestor_manager.update_slot_config() # Check OPC cycles + self.logger.debug("Checking OPC servers integrity...") self.ingestor_manager.check_opc_servers_integrity() diff --git a/values.yaml b/values.yaml index 08d9a14..53ba990 100644 --- a/values.yaml +++ b/values.yaml @@ -19,7 +19,7 @@ imagePullSecrets: # This is to override the chart name. nameOverride: "sientia-opc-ingestor" fullnameOverride: "sientia-opc-ingestor" -namespace: sientia-opc +namespace: sientia # This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ serviceAccount: @@ -123,7 +123,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-opc-ingestor.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-988-criar-ingestor-opc" + value: "main" - name: PYTHON_APP value: "ingestor.app" @@ -154,9 +154,14 @@ env: ssh: enabled: true - secretName: git-ssh-key-temp + secretName: git-ssh-key-sientia-opc-ingestor sshPath: /mnt/.ssh knownHostsPath: /mnt/known_hosts -# kubectl create secret docker-registry docker-hub-secret --namespace sientia-opc --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp -# helm upgrade --install sientia-dataops-opc-ingestor sientia/sientia-module -n sientia-opc --create-namespace -f ./values.yaml \ No newline at end of file +# kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp +# helm upgrade --install sientia-opc-ingestor sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.1.0-uat + +# kubectl create secret generic git-ssh-key-sientia-opc-ingestor \ +# --namespace sientia \ +# --from-file=ssh-privatekey=git_key \ +# --type=kubernetes.io/ssh-auth \ No newline at end of file From c13d7cf39197c50325ee731604a0fdfb37ee1f07 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 19 May 2025 14:15:25 -0300 Subject: [PATCH 02/28] log level fix --- values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index 53ba990..8f8fb7c 100644 --- a/values.yaml +++ b/values.yaml @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.0.1" + tag: "0.0.2" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: @@ -150,6 +150,8 @@ env: value: "30" - name: POLL_INTERVAL value: "10" + - name: LOG_LEVEL + value: "DEBUG" ssh: From 9196861033dd52fdf45dddee5cee363320252a5f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 19 May 2025 17:06:29 -0300 Subject: [PATCH 03/28] SIENTIAPDE-1049Enhance OPC connection handling with error logging and update branch in values.yaml --- ingestor/managers/ingestor_manager.py | 26 ++++++++++++++++++-------- ingestor/managers/opc_manager.py | 11 +++++++++-- values.yaml | 2 +- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index 8c05c44..06631c4 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -146,15 +146,25 @@ class IngestorManager(): f"Desconnecting from server." ) opc_manager.disconnect() - self.opc_managers[server] = self.initialize_opc_from_config( - opc_manager.config, self.data_manager, self.logger + + self.logger.warning( + f"Reconnecting to OPC server {server}" ) - self.update_opc_servers() - for slot, slot_config in self.managed_tags.items(): - if server in slot_config: - self.manage_server( - slot, server, slot_config[server], slot_config[server]['tags'] - ) + try: + self.opc_managers[server] = self.initialize_opc_from_config( + opc_manager.config, self.data_manager, self.logger + ) + except Exception as e: + self.logger.error( + f"Failed to reconnect to OPC server {server}: {e}" + ) + else: + self.update_opc_servers() + for slot, slot_config in self.managed_tags.items(): + if server in slot_config: + self.manage_server( + slot, server, slot_config[server], slot_config[server]['tags'] + ) def declare_active(self): """ diff --git a/ingestor/managers/opc_manager.py b/ingestor/managers/opc_manager.py index b8065f3..55a9b65 100644 --- a/ingestor/managers/opc_manager.py +++ b/ingestor/managers/opc_manager.py @@ -190,11 +190,18 @@ class OpcManager(): try: [self.subscriptions[sub].delete() for sub in self.subscriptions] self.logger.warning("Deleted all subscriptions.") + except Exception as sub_error: + self.logger.error(f"Failed to clean up subscription: {sub_error}") + + try: + self.client.disconnect() + except Exception as conn_error: + self.logger.error( + f"Failed to disconnect from OPC UA server: {conn_error}") + finally: del self.client self.client = None self.logger.warning("Disconnected from OPC UA server.") - except Exception as sub_error: - self.logger.error(f"Failed to clean up subscription: {sub_error}") def datachange_notification(self, node, _val, data): """ diff --git a/values.yaml b/values.yaml index 8f8fb7c..a94bca0 100644 --- a/values.yaml +++ b/values.yaml @@ -123,7 +123,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-opc-ingestor.git" - name: GITHUB_BRANCH - value: "main" + value: "SIENTIAPDE-1049-lidar-com-erros-de-reconexao-no-ingestor-opc" - name: PYTHON_APP value: "ingestor.app" From bb4ab7d209476f4f4d2a0cb9e1ff0044fd558c5d Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 20 May 2025 09:12:49 -0300 Subject: [PATCH 04/28] SIENTIAPDE-1049 Refactor logging statements for consistency and clarity; update environment variable parsing to handle string inputs. --- ingestor/ingestor.py | 44 +++++++++++----------- ingestor/managers/ingestor_manager.py | 54 ++++++++++++++------------- 2 files changed, 50 insertions(+), 48 deletions(-) diff --git a/ingestor/ingestor.py b/ingestor/ingestor.py index 45b33f8..f7fb6f5 100644 --- a/ingestor/ingestor.py +++ b/ingestor/ingestor.py @@ -1,9 +1,10 @@ from logging import Formatter, StreamHandler, getLogger from os import getenv -from ingestor.managers.ingestor_manager import IngestorManager from sientia_do.notifications.handlers import NotificationHandler +from ingestor.managers.ingestor_manager import IngestorManager + class Ingestor: def __init__(self): @@ -29,13 +30,13 @@ class Ingestor: kafka_servers = getenv("KAFKA_SERVERS", "localhost:9092") self.redis_host = getenv("REDIS_HOST", "localhost") - self.redis_port = int(getenv("REDIS_PORT", 6379)) + self.redis_port = int(getenv("REDIS_PORT", '6379')) self.redis_username = getenv("REDIS_USERNAME", None) self.redis_password = getenv("REDIS_PASSWORD", None) - self.lease_ttl = int(getenv("LEASE_TTL", 10)) - self.heartbeat_ttl = int(getenv("HEARTBEAT_TTL", 20)) + self.lease_ttl = int(getenv("LEASE_TTL", '10')) + self.heartbeat_ttl = int(getenv("HEARTBEAT_TTL", '20')) self.pod_id = getenv("HOSTNAME", "localhost") - self.poll_interval = int(getenv("POLL_INTERVAL", 5)) + self.poll_interval = int(getenv("POLL_INTERVAL", '5')) self.kafka_servers = kafka_servers.split(",") self.logger = None @@ -49,9 +50,8 @@ class Ingestor: model_name="-", model="-" ) - # build args for build notificarions components - # call build notifications components + self.ingestor_manager = None def init_logger(self): """ @@ -128,7 +128,7 @@ class Ingestor: # Get slot lease acquired = self.ingestor_manager.get_slot_leases() - self.logger.info(f"Acquired slots: {acquired}") + self.logger.info("Acquired slots: %s", acquired) self.handle_acquired_tags(acquired) @@ -171,7 +171,7 @@ class Ingestor: if available_slots > 0 and lacking_ingestors > 0: # Some ingestors are innactive, so theres "available_slots" slots available - self.logger.info(f"Slots available: {available_slots}") + self.logger.info("Slots available: %s", available_slots) # Get slot lease acquired = self.ingestor_manager.get_slot_leases(available_slots) @@ -180,7 +180,7 @@ class Ingestor: elif lacking_ingestors == 0 and slot_diff > 0: - self.logger.info(f"Extra slots available: {slot_diff}") + self.logger.info("Extra slots available: %s", slot_diff) # There's enough slots for all ingestors, but this ingestor has more than one slot # So we need to drop the extra leases @@ -210,37 +210,35 @@ class Ingestor: self.ingestor_manager.declare_active() self.logger.info("Polling for slot updates...") - # Get active ingestors[ + # Get active ingestors ingestors = self.ingestor_manager.get_active_ingestors() - self.logger.debug(f"Active ingestors: {ingestors}") number_of_ingestors = len(ingestors) - self.logger.debug(f"Number of ingestors: {number_of_ingestors}") number_of_leases = self.ingestor_manager.get_number_of_leases() - self.logger.debug(f"Number of leases: {number_of_leases}") number_of_slots = self.ingestor_manager.get_number_of_slots() - self.logger.debug(f"Number of slots: {number_of_slots}") # Handle no slots self.logger.debug("Managing no slots...") self.manage_no_slots(number_of_slots) available_slots = number_of_slots - number_of_leases - self.logger.debug(f"Available slots: {available_slots}") lacking_ingestors = number_of_slots - number_of_ingestors - self.logger.debug(f"Lacking ingestors: {lacking_ingestors}") slot_diff = len(self.ingestor_manager.managed_tags) - 1 - self.logger.debug(f"Slot diff: {slot_diff}") self.logger.debug("Managing leases...") self.manage_leases(available_slots, lacking_ingestors, slot_diff) self.logger.debug( - f"Active ingestors: {ingestors}, " - f"Number of slots: {number_of_slots}, " - f"Number of leases: {number_of_leases}, " - f"Managed tags: {self.ingestor_manager.managed_tags}" - f"Managed servers: {self.ingestor_manager.opc_managers}" + "Active ingestors: %s, " + "Number of slots: %s, " + "Number of leases: %s, " + "Managed tags: %s, " + "Managed servers: %s", + ingestors, + number_of_slots, + number_of_leases, + self.ingestor_manager.managed_tags, + self.ingestor_manager.opc_managers ) if not self.ingestor_manager.managed_tags: # No slots acquired diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index 06631c4..187a234 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -2,11 +2,11 @@ from logging import Logger import traceback from typing import Dict, List from copy import deepcopy +from sientia_do.notifications.handlers import NotificationHandler +from sientia_do.notifications.models import NotificationLevel from ingestor.managers.data_manager import DataManager from ingestor.managers.opc_manager import OpcManager from ingestor.managers.resource_manager import ResourceManager -from sientia_do.notifications.models import Notification, NotificationLevel -from sientia_do.notifications.handlers import NotificationHandler class IngestorManager(): @@ -30,7 +30,8 @@ class IngestorManager(): self.notification_handler = notification_handler - def initialize_opc_from_config(self, server_config: dict, data_manager: DataManager, logger: Logger) -> OpcManager | None: + def initialize_opc_from_config(self, server_config: dict, + data_manager: DataManager, logger: Logger) -> OpcManager | None: """ Initializes an OPC Manager instance using the provided server configuration. Args: @@ -53,9 +54,11 @@ class IngestorManager(): self.logger.info( f"Initializing OpcManager at {server_config['url']}") manager = OpcManager( - server_config['name'], server_config['url'], data_manager, logger, server_config['server_uri'], - self.notification_handler, server_config.get('cert_path'), server_config.get( - 'private_key_path'), server_config.get('server_cert_path') + server_config['name'], server_config['url'], + data_manager, logger, server_config['server_uri'], + self.notification_handler, server_config.get('cert_path'), + server_config.get('private_key_path'), + server_config.get('server_cert_path') ) manager.config = server_config @@ -91,7 +94,8 @@ class IngestorManager(): 4. Disconnects and removes OPC servers that are no longer registered. Attributes: self.managed_tags (dict): A nested dictionary containing slot and server configurations. - self.opc_managers (dict): A dictionary mapping server names to their OPC manager instances. + self.opc_managers (dict): A dictionary mapping server names to their + OPC manager instances. self.data_manager: An object responsible for managing data operations. self.logger: A logging object for recording warnings and other messages. Raises: @@ -101,19 +105,24 @@ class IngestorManager(): """ registered_servers = [] - for slot, slot_config in self.managed_tags.items(): + for _slot, slot_config in self.managed_tags.items(): for server, server_config in slot_config.items(): registered_servers.append(server) server_config = server_config.copy() server_config.pop('tags', None) server_instance = None if server not in self.opc_managers: - + self.logger.debug( + f"Initializing OPC manager for server {server}" + ) server_instance = self.initialize_opc_from_config( server_config, self.data_manager, self.logger ) elif self.opc_managers[server].config != server_config: + self.logger.warning( + f"Reinitializing OPC manager for server {server}" + ) self.opc_managers[server].disconnect() del self.opc_managers[server] server_instance = self.initialize_opc_from_config( @@ -150,16 +159,9 @@ class IngestorManager(): self.logger.warning( f"Reconnecting to OPC server {server}" ) - try: - self.opc_managers[server] = self.initialize_opc_from_config( - opc_manager.config, self.data_manager, self.logger - ) - except Exception as e: - self.logger.error( - f"Failed to reconnect to OPC server {server}: {e}" - ) - else: - self.update_opc_servers() + self.opc_managers.pop(server, None) + self.update_opc_servers() + if self.opc_managers.get(server) is not None: for slot, slot_config in self.managed_tags.items(): if server in slot_config: self.manage_server( @@ -222,14 +224,16 @@ class IngestorManager(): Dict: A dictionary where the keys are the slot identifiers (as strings) and the values are the leased slot details. Behavior: - - Iterates through available slots and attempts to lease them using the resource manager. + - Iterates through available slots and attempts to lease + them using the resource manager. - Logs the leasing of each slot. - Updates the `managed_tags` attribute with the acquired slots. - Stops leasing once the specified `max_slots` are acquired. - - If unable to acquire the requested number of slots, logs a warning and returns the slots that were leased. + - If unable to acquire the requested number of slots, + logs a warning and returns the slots that were leased. Notes: - - If a slot is leased but its details cannot be retrieved (i.e., `get_tag_slot` returns None), - that slot is skipped. + - If a slot is leased but its details cannot be retrieved + (i.e., `get_tag_slot` returns None), that slot is skipped. """ acquired = {} @@ -331,8 +335,8 @@ class IngestorManager(): None """ - for id in ids: - self.resource_manager.drop_tag_lease(id) + for lease_id in ids: + self.resource_manager.drop_tag_lease(lease_id) def manage_server(self, slot: str, server: str, server_config: dict, tags: dict) -> int: """ From b03fe75753bc4963a32ac7f846e45e7270adf781 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 20 May 2025 09:34:01 -0300 Subject: [PATCH 05/28] SIENTIAPDE-1049 Handling size change --- ingestor/managers/ingestor_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index 187a234..462cb3e 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -145,7 +145,8 @@ class IngestorManager(): """ Checks the integrity of the OPC servers and updates the OPC servers if necessary. """ - for server, opc_manager in self.opc_managers.items(): + current_managers = self.opc_managers.copy() + for server, opc_manager in current_managers.items(): opc_manager.check_cycles() is_lost = opc_manager.check_opc_listenning() From 39530f115e436b9a22417cd88a12d8e57ca16997 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 20 May 2025 10:10:17 -0300 Subject: [PATCH 06/28] SIENTIAPDE-1049 Integrity fix --- ingestor/managers/ingestor_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index 462cb3e..bc18465 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -168,6 +168,9 @@ class IngestorManager(): self.manage_server( slot, server, slot_config[server], slot_config[server]['tags'] ) + else: + for slot, slot_config in self.managed_tags.items(): + self.managed_tags[slot].pop(server, None) def declare_active(self): """ From 1fa65589864ccd6178258320c113c6291e97ddec Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 20 May 2025 10:18:37 -0300 Subject: [PATCH 07/28] SIENTIAPDE-1049 Integrity fix --- ingestor/managers/ingestor_manager.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index bc18465..42657d7 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -169,6 +169,10 @@ class IngestorManager(): slot, server, slot_config[server], slot_config[server]['tags'] ) else: + self.logger.warning( + f"OPC server {server} reconnecting failed. " + f"Removing server from managed tags." + ) for slot, slot_config in self.managed_tags.items(): self.managed_tags[slot].pop(server, None) @@ -301,6 +305,8 @@ class IngestorManager(): for slot, slot_config in self.managed_tags.items(): self.resource_manager.renew_tag_lease(slot) update = self.resource_manager.get_tag_slot(slot) + self.logger.debug( + f"Comparing slot {slot} configuration: {slot_config} with update: {update}") if update is None: self.logger.warning( f"Slot {slot} configuration not found. " From 2fa1d2175e68152163aecbe86a9390bb0c20611f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 20 May 2025 11:04:41 -0300 Subject: [PATCH 08/28] SIENTIAPDE-1049 Stability fix try --- ingestor/managers/ingestor_manager.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index 42657d7..78b442f 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -131,6 +131,13 @@ class IngestorManager(): if server_instance is not None: self.opc_managers[server] = server_instance + else: + self.logger.warning( + f"Failed to initialize OPC manager for server {server}, " + f"removing server from managed tags." + ) + _a = [self.managed_tags[slot].pop(server, None) + for slot, _value in self.managed_tags.items()] for server in list(self.opc_managers.keys()): if server not in registered_servers: @@ -168,13 +175,6 @@ class IngestorManager(): self.manage_server( slot, server, slot_config[server], slot_config[server]['tags'] ) - else: - self.logger.warning( - f"OPC server {server} reconnecting failed. " - f"Removing server from managed tags." - ) - for slot, slot_config in self.managed_tags.items(): - self.managed_tags[slot].pop(server, None) def declare_active(self): """ From 92ed3c9cb2173cdb249a472251efaa9a56348842 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 20 May 2025 11:30:36 -0300 Subject: [PATCH 09/28] SIENTIAPDE-1049 Fix --- ingestor/managers/ingestor_manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index 78b442f..382bc94 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -105,7 +105,8 @@ class IngestorManager(): """ registered_servers = [] - for _slot, slot_config in self.managed_tags.items(): + current_managed_tags = self.managed_tags.copy() + for _slot, slot_config in current_managed_tags.items(): for server, server_config in slot_config.items(): registered_servers.append(server) server_config = server_config.copy() @@ -137,7 +138,7 @@ class IngestorManager(): f"removing server from managed tags." ) _a = [self.managed_tags[slot].pop(server, None) - for slot, _value in self.managed_tags.items()] + for slot, _value in current_managed_tags.items()] for server in list(self.opc_managers.keys()): if server not in registered_servers: From c7dbe3585b102fffbdd2791403feba3b37431f46 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 20 May 2025 17:19:41 -0300 Subject: [PATCH 10/28] SIENTIAPDE-1049 Update ingestor and data manager for improved shutdown handling and logging - Incremented project version in quality gate configuration. - Commented out ingestor service in docker-compose for clarity. - Enhanced main loop in app.py to handle exit signals and exceptions. - Added shutdown methods in Ingestor and DataManager classes for graceful resource cleanup. - Updated unit tests to validate shutdown behavior and exception handling. - Introduced coverage configuration to omit specific files. --- .coveragerc | 3 + .github/workflows/quality-gate.yml | 3 +- docker-compose.yaml | 26 ++++---- ingestor/app.py | 35 ++++++++-- ingestor/ingestor.py | 57 +++++++++++++++-- ingestor/managers/data_manager.py | 19 ++++-- ingestor/managers/ingestor_manager.py | 67 ++++++++------------ ingestor/managers/opc_manager.py | 22 ++++--- simulator/redis-feeder.py | 16 +++-- tests/unit/managers/test_data_manager.py | 30 ++++++--- tests/unit/managers/test_ingestor_manager.py | 51 --------------- tests/unit/managers/test_opc_manager.py | 17 ++++- tests/unit/test_ingestor.py | 52 +++++++++++---- 13 files changed, 235 insertions(+), 163 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..2249368 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + ingestor/app.py diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 7efa623..e13ecb2 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -80,4 +80,5 @@ jobs: -Dsonar.host.url=$SONAR_HOST_URL \ -Dsonar.token=$SONAR_TOKEN \ -Dsonar.python.version=3.11 \ - -Dsonar.projectVersion=1.0.0 + -Dsonar.projectVersion=1.0.1 \ + -Dsonar.coverage.exclusions=ingestor/app.py diff --git a/docker-compose.yaml b/docker-compose.yaml index 891fc9b..2216ee5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,19 +1,19 @@ version: '3.8' services: - ingestor: - build: - context: . - environment: - HOSTNAME: ingestor - container_name: ingestor - depends_on: - - kafka - - redis - networks: - - kafka-net - env_file: - - .env + # ingestor: + # build: + # context: . + # environment: + # HOSTNAME: ingestor + # container_name: ingestor + # depends_on: + # - kafka + # - redis + # networks: + # - kafka-net + # env_file: + # - .env zookeeper: image: confluentinc/cp-zookeeper:latest diff --git a/ingestor/app.py b/ingestor/app.py index b9885fc..7b3f05b 100644 --- a/ingestor/app.py +++ b/ingestor/app.py @@ -1,19 +1,42 @@ -from time import sleep +from threading import Event +import signal +import os +import traceback from ingestor.ingestor import Ingestor +exit_signal = Event() + def main(): ingestor = Ingestor() - ingestor.prepare_ingestor() + ingestor.logger.info("Ingestor prepared. Starting main loop.") - while True: + while not exit_signal.is_set(): + try: + ingestor.loop() - ingestor.loop() + exit_signal.wait(ingestor.poll_interval) - # Sleep for poll interval - sleep(ingestor.poll_interval) + except Exception: + print("Exception in main loop. Setting exit_signal flag.") + traceback.print_exc() + exit_signal.set() + + ingestor.shutdown() + + ingestor.logger.info("Main loop exit_signaled.") + + os._exit(0) + + +def signal_handler(_signum, _frame): + print(f"Received signal {_signum}. Setting exit_signal flag.") + exit_signal.set() if __name__ == "__main__": + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGHUP, signal_handler) main() diff --git a/ingestor/ingestor.py b/ingestor/ingestor.py index f7fb6f5..4c7536e 100644 --- a/ingestor/ingestor.py +++ b/ingestor/ingestor.py @@ -1,5 +1,7 @@ from logging import Formatter, StreamHandler, getLogger from os import getenv +from copy import deepcopy +from typing import Dict, Any from sientia_do.notifications.handlers import NotificationHandler @@ -11,7 +13,8 @@ class Ingestor: """ Initializes the ingestor with configuration values retrieved from environment variables. Environment Variables: - KAFKA_SERVERS (str): Comma-separated list of Kafka server addresses. Defaults to "localhost:9092". + KAFKA_SERVERS (str): Comma-separated list of Kafka server addresses. + Defaults to "localhost:9092". REDIS_HOST (str): Hostname of the Redis server. Defaults to "localhost". REDIS_PORT (int): Port number of the Redis server. Defaults to 6379. LEASE_TTL (int): Time-to-live for leases in seconds. Defaults to 10. @@ -53,6 +56,13 @@ class Ingestor: self.ingestor_manager = None + def shutdown(self): + if self.ingestor_manager: + self.ingestor_manager.shutdown() + + def __del__(self): + self.shutdown() + def init_logger(self): """ Initializes a logger instance for the class. @@ -147,9 +157,7 @@ class Ingestor: # This ingestor is active and has no slots, so we need to try to # Get slot lease - acquired = self.ingestor_manager.get_slot_leases(1) - - self.handle_acquired_tags(acquired) + self.ingestor_manager.get_slot_leases(1) def manage_leases(self, available_slots: int, lacking_ingestors: int, slot_diff: int): """ @@ -174,9 +182,7 @@ class Ingestor: self.logger.info("Slots available: %s", available_slots) # Get slot lease - acquired = self.ingestor_manager.get_slot_leases(available_slots) - - self.handle_acquired_tags(acquired) + self.ingestor_manager.get_slot_leases(available_slots) elif lacking_ingestors == 0 and slot_diff > 0: @@ -188,6 +194,36 @@ class Ingestor: self.ingestor_manager.drop_slot_leases(overleases) + def update_ingestor_manager(self, old_managed_tags: Dict[str, Any]): + """ + Updates the ingestor manager with the new managed tags. + Args: + old_managed_tags (Dict[str, Any]): The old managed tags. + """ + + self.logger.debug("Current managed tags: %s", + self.ingestor_manager.managed_tags) + + self.ingestor_manager.update_opc_servers() + new_managed_tags = deepcopy(self.ingestor_manager.managed_tags) + + self.logger.debug("Comparing new managed tags %s" + "with old managed tags %s", + new_managed_tags, old_managed_tags) + + for slot, config in new_managed_tags.items(): + if slot not in old_managed_tags: + self.ingestor_manager.subscribe_to_tags({slot: config}) + continue + + if config != old_managed_tags[slot]: + self.ingestor_manager.unsubscribe_slot(slot) + self.ingestor_manager.subscribe_to_tags({slot: config}) + + for slot in old_managed_tags.keys(): + if slot not in new_managed_tags: + self.ingestor_manager.unsubscribe_slot(slot) + def loop(self): """ Executes the main loop for managing ingestors and slots. @@ -212,6 +248,8 @@ class Ingestor: self.logger.info("Polling for slot updates...") # Get active ingestors + current_managed_tags = deepcopy(self.ingestor_manager.managed_tags) + ingestors = self.ingestor_manager.get_active_ingestors() number_of_ingestors = len(ingestors) number_of_leases = self.ingestor_manager.get_number_of_leases() @@ -251,3 +289,8 @@ class Ingestor: # Check OPC cycles self.logger.debug("Checking OPC servers integrity...") self.ingestor_manager.check_opc_servers_integrity() + + self.logger.debug("Updating managed tags...") + self.update_ingestor_manager( + current_managed_tags + ) diff --git a/ingestor/managers/data_manager.py b/ingestor/managers/data_manager.py index 3f9ca3c..b16b933 100644 --- a/ingestor/managers/data_manager.py +++ b/ingestor/managers/data_manager.py @@ -51,14 +51,21 @@ class DataManager(): self.logger = logger self.notification_handler = notification_handler - def __del__(self): - """Destructor to close the producer connection.""" - print("Closing Kafka producer...") + def shutdown(self): + """Closes the Kafka producer connection.""" if self.kafka_producer: - self.kafka_producer.flush(timeout=10) - self.kafka_producer.close() + try: + self.kafka_producer.flush(timeout=10) + self.kafka_producer.close() + except Exception as e: + self.logger.error( + f"Error closing Kafka producer: {e}") else: - print("Kafka producer is already closed or not initialized.") + self.logger.warning( + "Kafka producer is already closed or not initialized.") + + def __del__(self): + self.shutdown() def delivery_report(self, msg: str): """Callback for delivery reports from Kafka.""" diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index 382bc94..433fcef 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -79,6 +79,14 @@ class IngestorManager(): return manager + def shutdown(self): + for _server_name, server in self.opc_managers.items(): + server.disconnect() + self.data_manager.shutdown() + + def __del__(self): + self.shutdown() + def update_opc_servers(self): """ Updates the OPC (OLE for Process Control) server connections managed by the ingestor. @@ -105,14 +113,14 @@ class IngestorManager(): """ registered_servers = [] - current_managed_tags = self.managed_tags.copy() + current_managed_tags = deepcopy(self.managed_tags) for _slot, slot_config in current_managed_tags.items(): for server, server_config in slot_config.items(): registered_servers.append(server) - server_config = server_config.copy() + server_config = deepcopy(server_config) server_config.pop('tags', None) - server_instance = None - if server not in self.opc_managers: + server_instance = self.opc_managers.get(server, None) + if server_instance is None: self.logger.debug( f"Initializing OPC manager for server {server}" ) @@ -120,15 +128,19 @@ class IngestorManager(): server_config, self.data_manager, self.logger ) - elif self.opc_managers[server].config != server_config: + elif server_instance.config != server_config: self.logger.warning( f"Reinitializing OPC manager for server {server}" ) - self.opc_managers[server].disconnect() + server_instance.disconnect() del self.opc_managers[server] server_instance = self.initialize_opc_from_config( server_config, self.data_manager, self.logger ) + else: + self.logger.debug( + f"OPC manager for server {server} is already initialized and up to date" + ) if server_instance is not None: self.opc_managers[server] = server_instance @@ -153,29 +165,18 @@ class IngestorManager(): """ Checks the integrity of the OPC servers and updates the OPC servers if necessary. """ - current_managers = self.opc_managers.copy() - for server, opc_manager in current_managers.items(): + for server, opc_manager in self.opc_managers.items(): opc_manager.check_cycles() is_lost = opc_manager.check_opc_listenning() if is_lost: self.logger.warning( f"OPC server {server} is lost. " - f"Desconnecting from server." + f"Server will be disconnected." ) - opc_manager.disconnect() - self.logger.warning( - f"Reconnecting to OPC server {server}" - ) - self.opc_managers.pop(server, None) - self.update_opc_servers() - if self.opc_managers.get(server) is not None: - for slot, slot_config in self.managed_tags.items(): - if server in slot_config: - self.manage_server( - slot, server, slot_config[server], slot_config[server]['tags'] - ) + for slot, _config in self.managed_tags.items(): + self.managed_tags[slot].pop(server, None) def declare_active(self): """ @@ -303,35 +304,17 @@ class IngestorManager(): removed_slots = [] update = {} - for slot, slot_config in self.managed_tags.items(): + for slot, _slot_config in self.managed_tags.items(): self.resource_manager.renew_tag_lease(slot) update = self.resource_manager.get_tag_slot(slot) - self.logger.debug( - f"Comparing slot {slot} configuration: {slot_config} with update: {update}") if update is None: - self.logger.warning( - f"Slot {slot} configuration not found. " - f"Removing slot from managed tags." - ) - self.unsubscribe_slot(slot) removed_slots.append(slot) - continue - if update != slot_config: - self.logger.info( - f"Slot {slot} configuration updated. " - f"Old: {slot_config}, New: {update}" - ) - self.managed_tags[slot] = update - - self.unsubscribe_slot(slot) - self.update_opc_servers() - self.subscribe_to_tags({slot: update}) + self.managed_tags[slot] = update for slot in removed_slots: - del self.managed_tags[slot] - self.update_opc_servers() + self.managed_tags.pop(slot, None) def drop_slot_leases(self, ids: List[str]) -> None: """ diff --git a/ingestor/managers/opc_manager.py b/ingestor/managers/opc_manager.py index 55a9b65..e218956 100644 --- a/ingestor/managers/opc_manager.py +++ b/ingestor/managers/opc_manager.py @@ -188,7 +188,8 @@ class OpcManager(): self.logger.warning("Client already disconnected.") return try: - [self.subscriptions[sub].delete() for sub in self.subscriptions] + _a = [self.subscriptions[sub].delete() + for sub in self.subscriptions] self.logger.warning("Deleted all subscriptions.") except Exception as sub_error: self.logger.error(f"Failed to clean up subscription: {sub_error}") @@ -230,10 +231,8 @@ class OpcManager(): tag = str(node) self.logger.debug( - f"Data change notification received for tag: {tag} after {self.nodes[tag]['cycle_rule']['cycle_count']} cycles") - - self.logger.debug( - f"Resetting cycle count for tag: {tag} after {self.non_receive_count} OPC cycles") + f"Data change notification received for tag:" + f"{tag} after {self.nodes[tag]['cycle_rule']['cycle_count']} cycles") self.nodes[tag]['cycle_rule']['cycle_count'] = 0 self.non_receive_count = 0 @@ -245,12 +244,13 @@ class OpcManager(): 'value': value } - [self.data_manager.publish(e, data) - for e in self.nodes[tag]['topics']] + _a = [self.data_manager.publish(e, data) + for e in self.nodes[tag]['topics']] def check_cycles(self): """ - Checks the cycle counts for all monitored nodes and sends notifications if thresholds are exceeded. + Checks the cycle counts for all monitored nodes and sends + notifications if thresholds are exceeded. This method iterates through all monitored nodes and updates their cycle counts based on configured increments. If a node's cycle count exceeds a threshold (5 cycles), it triggers @@ -258,7 +258,8 @@ class OpcManager(): """ for node, config in self.nodes.items(): - self.nodes[node]['cycle_rule']['cycle_count'] += self.nodes[node]['cycle_rule']['cycle_increment'] + self.nodes[node]['cycle_rule']['cycle_count'] += config[ + 'cycle_rule']['cycle_increment'] if self.nodes[node]['cycle_rule']['cycle_count'] >= 5: name = config['tag_name'] cycles = self.nodes[node]['cycle_rule']['cycle_count'] @@ -280,7 +281,8 @@ class OpcManager(): if self.non_receive_count >= 5: self.notification_handler.build_and_send_notification( notification_id=f'OPC_LISTENNING_STOPPED__{self.name}', - message=f'{self.non_receive_count} cycles without receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}', + message=f'{self.non_receive_count} cycles without ' + f'receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}', block="opc_manager", level=NotificationLevel.ERROR ) diff --git a/simulator/redis-feeder.py b/simulator/redis-feeder.py index 3f7e36a..d3dde93 100644 --- a/simulator/redis-feeder.py +++ b/simulator/redis-feeder.py @@ -1,14 +1,18 @@ import redis import json -import os # Redis connection settings -redis_host = "localhost" -redis_port = 6379 +REDIS_HOST = "localhost" +REDIS_PORT = 6379 +REDIS_USERNAME = None # "default" +REDIS_PASSWORD = None # "bdnZOpcyiL" + +OPC_URL = "opc.tcp://sientia-opc-simulator-service.sientia-opc.svc.cluster.local:4840" +OPC_URL = "opc.tcp://localhost:4840" # Connect to Redis -r = redis.Redis(host=redis_host, port=redis_port, - decode_responses=True, username='default', password='bdnZOpcyiL') +r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, + decode_responses=True, username=REDIS_USERNAME, password=REDIS_PASSWORD) # Define the key pattern to target pattern = "slot:opc_tags:*" @@ -25,7 +29,7 @@ new_data = { "slot:opc_tags:1": { "server1": { "name": "server1", - "url": "opc.tcp://sientia-opc-simulator-service.sientia-opc.svc.cluster.local:4840", + "url": OPC_URL, "server_uri": "http://opcua-server.simulator", "tags": { 'ns=2;i=2': { diff --git a/tests/unit/managers/test_data_manager.py b/tests/unit/managers/test_data_manager.py index 34c387a..80f75e0 100644 --- a/tests/unit/managers/test_data_manager.py +++ b/tests/unit/managers/test_data_manager.py @@ -108,31 +108,45 @@ def test___init___failure_max_attempts(kafka): assert False, "Expected NoBrokersAvailable exception was not raised." -def test___del___has_producer(data_manager): +def test_shutdown_has_producer(data_manager): flush_mock = MagicMock() close_mock = MagicMock() data_manager.kafka_producer.flush = flush_mock data_manager.kafka_producer.close = close_mock - data_manager.__del__() + data_manager.shutdown() flush_mock.assert_called_once() close_mock.assert_called_once() -@patch("ingestor.managers.data_manager.print") -def test___del___no_producer(print, data_manager): +def test_shutdown_no_producer(data_manager): data_manager.kafka_producer = None - # Call the __del__ method - data_manager.__del__() + data_manager.shutdown() - # Check if the print statement was called - print.assert_any_call( + data_manager.logger.warning.assert_any_call( "Kafka producer is already closed or not initialized." ) +def test_shutdown_exception(data_manager): + data_manager.kafka_producer.flush = MagicMock( + side_effect=Exception("Test error")) + data_manager.kafka_producer.close = MagicMock() + + data_manager.shutdown() + data_manager.logger.error.assert_called_once_with( + "Error closing Kafka producer: Test error" + ) + + +def test___del__(data_manager): + data_manager.shutdown = MagicMock() + data_manager.__del__() + data_manager.shutdown.assert_called_once() + + def test_delivery_report(data_manager): msg = MagicMock() msg.topic = "test_topic" diff --git a/tests/unit/managers/test_ingestor_manager.py b/tests/unit/managers/test_ingestor_manager.py index 6e59103..fa3f7be 100644 --- a/tests/unit/managers/test_ingestor_manager.py +++ b/tests/unit/managers/test_ingestor_manager.py @@ -331,14 +331,6 @@ def test_update_slot_config(ingestor_manager): "config": "new_config"} assert "slot3" not in ingestor_manager.managed_tags - assert ingestor_manager.update_opc_servers.call_count == 2 - ingestor_manager.subscribe_to_tags.assert_called_once_with( - {'slot1': {"config": "updated_config"}} - ) - ingestor_manager.unsubscribe_slot.assert_any_call("slot3") - ingestor_manager.unsubscribe_slot.assert_any_call("slot1") - assert ingestor_manager.unsubscribe_slot.call_count == 2 - ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot1") ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot3") ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot2") @@ -546,27 +538,9 @@ def test_check_opc_servers_integrity_server_lost(ingestor_manager): ingestor_manager.initialize_opc_from_config = MagicMock( return_value=new_manager) - # Mock update_opc_servers and manage_server - ingestor_manager.update_opc_servers = MagicMock() - ingestor_manager.manage_server = MagicMock() - # Call the method ingestor_manager.check_opc_servers_integrity() - # Verify that the lost server was disconnected - opc_manager.disconnect.assert_called_once() - - # Verify that a new manager was initialized - ingestor_manager.initialize_opc_from_config.assert_called_once_with( - opc_manager.config, ingestor_manager.data_manager, ingestor_manager.logger - ) - - # Verify that the new manager was assigned - assert ingestor_manager.opc_managers["server1"] == new_manager - - # Verify that update_opc_servers was called - ingestor_manager.update_opc_servers.assert_called_once() - def test_check_opc_servers_integrity_server_lost_with_tags(ingestor_manager): # Setup mock OPC manager that will be lost @@ -594,30 +568,5 @@ def test_check_opc_servers_integrity_server_lost_with_tags(ingestor_manager): ingestor_manager.initialize_opc_from_config = MagicMock( return_value=new_manager) - # Mock update_opc_servers and manage_server - ingestor_manager.update_opc_servers = MagicMock() - ingestor_manager.manage_server = MagicMock() - # Call the method ingestor_manager.check_opc_servers_integrity() - - # Verify that the lost server was disconnected - opc_manager.disconnect.assert_called_once() - - # Verify that a new manager was initialized - ingestor_manager.initialize_opc_from_config.assert_called_once_with( - opc_manager.config, ingestor_manager.data_manager, ingestor_manager.logger - ) - - # Verify that the new manager was assigned - assert ingestor_manager.opc_managers["server1"] == new_manager - - # Verify that update_opc_servers was called - ingestor_manager.update_opc_servers.assert_called_once() - - # Verify that manage_server was called with the correct tags - ingestor_manager.manage_server.assert_called_once_with( - "slot1", "server1", - {"config": "config1", "tags": {"tag1": "value1"}}, - {"tag1": "value1"} - ) diff --git a/tests/unit/managers/test_opc_manager.py b/tests/unit/managers/test_opc_manager.py index 8886709..a57ef46 100644 --- a/tests/unit/managers/test_opc_manager.py +++ b/tests/unit/managers/test_opc_manager.py @@ -188,7 +188,7 @@ def test_disconnect_success(opc_manager_subscribed): assert opc_manager_subscribed.client is None -def test_disconnect_error(opc_manager_subscribed): +def test_disconnect_error_unsubscribe(opc_manager_subscribed): opc_manager_subscribed.client = MagicMock() opc_manager_subscribed.subscriptions['sub1'] = MagicMock( delete=MagicMock(side_effect=Exception("Test error")) @@ -197,11 +197,24 @@ def test_disconnect_error(opc_manager_subscribed): opc_manager_subscribed.disconnect() opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once() - opc_manager_subscribed.client.disconnect.assert_not_called() + opc_manager_subscribed.client = None opc_manager_subscribed.logger.error.assert_called_once_with( "Failed to clean up subscription: Test error") +def test_disconnect_error(opc_manager_subscribed): + opc_manager_subscribed.client = MagicMock() + opc_manager_subscribed.client.disconnect = MagicMock( + side_effect=Exception("Test error")) + + opc_manager_subscribed.disconnect() + + opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once() + opc_manager_subscribed.client = None + opc_manager_subscribed.logger.error.assert_called_once_with( + "Failed to disconnect from OPC UA server: Test error") + + def test_datachange_notification(opc_manager_subscribed): data = MagicMock( monitored_item=MagicMock( diff --git a/tests/unit/test_ingestor.py b/tests/unit/test_ingestor.py index 448363c..54ebdc7 100644 --- a/tests/unit/test_ingestor.py +++ b/tests/unit/test_ingestor.py @@ -1,4 +1,5 @@ -from unittest.mock import ANY, MagicMock, patch +from unittest.mock import ANY, MagicMock, patch, call +from os import getenv from pytest import fixture from ingestor.ingestor import Ingestor @@ -24,13 +25,13 @@ def test___init__(notification_handler, init_logger, getenv): getenv.assert_any_call("KAFKA_SERVERS", "localhost:9092") getenv.assert_any_call("REDIS_HOST", "localhost") - getenv.assert_any_call("REDIS_PORT", 6379) + getenv.assert_any_call("REDIS_PORT", '6379') getenv.assert_any_call("REDIS_USERNAME", None) getenv.assert_any_call("REDIS_PASSWORD", None) - getenv.assert_any_call("LEASE_TTL", 10) - getenv.assert_any_call("HEARTBEAT_TTL", 20) + getenv.assert_any_call("LEASE_TTL", '10') + getenv.assert_any_call("HEARTBEAT_TTL", '20') getenv.assert_any_call("HOSTNAME", "localhost") - getenv.assert_any_call("POLL_INTERVAL", 5) + getenv.assert_any_call("POLL_INTERVAL", '5') assert ingestor.kafka_servers == ["localhost:9092", "localhost:35"] assert ingestor.redis_host == "localhost1" @@ -58,7 +59,7 @@ def test___init__(notification_handler, init_logger, getenv): @patch("ingestor.ingestor.getenv") @patch("ingestor.ingestor.Ingestor.init_logger") @patch("ingestor.ingestor.NotificationHandler") -def ingestor(notification_handler, init_logger, getenv): +def ingestor(_notification_handler, _init_logger, _getenv): ing = Ingestor() ing.logger = MagicMock() @@ -82,7 +83,8 @@ def test_init_logger(formatter, stream_handler, get_logger, ingestor): stream_handler.assert_called_once() formatter.assert_called_once_with( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') - ingestor.logger.setLevel.assert_called_once_with("INFO") + ingestor.logger.setLevel.assert_called_once_with( + getenv("LOG_LEVEL", "INFO")) ingestor.logger.addHandler.assert_called_once_with( stream_handler.return_value) stream_handler.return_value.setFormatter.assert_called_once_with( @@ -172,8 +174,6 @@ def test_manage_slots_none_available_none_available(ingestor_manager_started): ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with( 1) - ingestor_manager_started.handle_acquired_tags.assert_called_once_with( - ingestor_manager_started.ingestor_manager.get_slot_leases.return_value) def test_manage_leases_no_available_slots_no_extra_slots(ingestor_manager_started): @@ -193,8 +193,6 @@ def test_manage_leases_available_slots_innactive_ingestors(ingestor_manager_star ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with( 2) - ingestor_manager_started.handle_acquired_tags.assert_called_once_with( - ingestor_manager_started.ingestor_manager.get_slot_leases.return_value) ingestor_manager_started.ingestor_manager.drop_slot_leases.assert_not_called() @@ -267,3 +265,35 @@ def test_loop_no_managed(ingestor_manager_started): ingestor_manager_started.ingestor_manager.update_slot_config.assert_called_once() ingestor_manager_started.logger.info.assert_any_call( "No slots acquired in this loop") + + +def test_update_ingestor_manager(ingestor_manager_started): + ingestor_manager_started.ingestor_manager.managed_tags = { + "slot_to_create": "new_config", + "slot_to_update": "new_config", + "slot_to_do_nothing": "old_config" + } + + old_managed_tags = { + "slot_to_update": "old_config", + "slot_to_delete": "old_config", + "slot_to_do_nothing": "old_config" + } + + ingestor_manager_started.update_ingestor_manager(old_managed_tags) + + ingestor_manager_started.ingestor_manager.update_opc_servers.assert_called_once() + ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_has_calls( + [ + call( + {"slot_to_create": "new_config"}), + call( + {"slot_to_update": "new_config"}) + ] + ) + ingestor_manager_started.ingestor_manager.unsubscribe_slot.assert_has_calls( + [ + call("slot_to_update"), + call("slot_to_delete") + ] + ) From 2612148455c7d7299a4c048ffc19919a116161a3 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 21 May 2025 07:50:11 -0300 Subject: [PATCH 11/28] lease fix --- ingestor/ingestor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestor/ingestor.py b/ingestor/ingestor.py index 4c7536e..166367f 100644 --- a/ingestor/ingestor.py +++ b/ingestor/ingestor.py @@ -184,7 +184,7 @@ class Ingestor: # Get slot lease self.ingestor_manager.get_slot_leases(available_slots) - elif lacking_ingestors == 0 and slot_diff > 0: + elif lacking_ingestors <= 0 and slot_diff > 0: self.logger.info("Extra slots available: %s", slot_diff) # There's enough slots for all ingestors, but this ingestor has more than one slot From 019dc33fc9d3d0070ebc670c73a03aa085885b62 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 21 May 2025 08:17:44 -0300 Subject: [PATCH 12/28] subscribe fix --- ingestor/ingestor.py | 4 ++++ values.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ingestor/ingestor.py b/ingestor/ingestor.py index 166367f..b508b56 100644 --- a/ingestor/ingestor.py +++ b/ingestor/ingestor.py @@ -194,6 +194,10 @@ class Ingestor: self.ingestor_manager.drop_slot_leases(overleases) + for lease in overleases: + self.ingestor_manager.unsubscribe_slot(lease) + self.ingestor_manager.managed_tags.pop(lease) + def update_ingestor_manager(self, old_managed_tags: Dict[str, Any]): """ Updates the ingestor manager with the new managed tags. diff --git a/values.yaml b/values.yaml index a94bca0..a009115 100644 --- a/values.yaml +++ b/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 1 +replicaCount: 2 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: From 26356c05dd8aa46f8cbaff8fe7a4bb8a1bd61be2 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 21 May 2025 08:43:47 -0300 Subject: [PATCH 13/28] final adjust in replicaCount --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index a009115..a94bca0 100644 --- a/values.yaml +++ b/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 2 +replicaCount: 1 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: From 861101ffc6f9f546cbc62c4f8c7d0abe6c5838c3 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 22 May 2025 10:06:03 -0300 Subject: [PATCH 14/28] SIENTIAPDE-1075 Add run instructions for the Ingestor --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 415658b..e1b09a7 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,20 @@ docker compose build --ssh default=$HOME/.ssh/id_ed25519_docker docker compose up -d ``` +### Run Ingestor +The ingestor can be run as a docker container or as a python script. + +As a docker container: +Uncomment the ingestor service in docker-compose.yaml +``` +docker compose up -d ingestor +``` + +As a python script: +``` +python -m ingestor.app +``` + ### Populate redis server Create venv with python3.11 ``` From 14c71580e00b5d898eea661c15af082198a2a70b Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Wed, 28 May 2025 14:47:52 -0300 Subject: [PATCH 15/28] SIENTIAPDE-1083: add prometheus and app metrics. --- .vscode/settings.json | 6 +- ingestor/app.py | 35 ++++++++- ingestor/metrics.py | 168 ++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 3 +- 4 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 ingestor/metrics.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 3e99ede..78e50ab 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,5 +3,9 @@ "." ], "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true + "python.testing.pytestEnabled": true, + "sonarlint.connectedMode.project": { + "connectionId": "sonardev-sientia-ai", + "projectKey": "Aignosi_sientia-dataops-opc-ingestor_1642c8bf-a148-4911-8362-0903d7fef99a" + } } \ No newline at end of file diff --git a/ingestor/app.py b/ingestor/app.py index 7b3f05b..20d2a9f 100644 --- a/ingestor/app.py +++ b/ingestor/app.py @@ -1,32 +1,53 @@ -from threading import Event -import signal import os +import signal import traceback +from threading import Event +from time import sleep, time + +from prometheus_client import start_http_server + +import ingestor.metrics as metrics from ingestor.ingestor import Ingestor exit_signal = Event() +POD_ID = os.getenv("HOSTNAME", "localhost") def main(): + start_prometheus_server() ingestor = Ingestor() ingestor.prepare_ingestor() ingestor.logger.info("Ingestor prepared. Starting main loop.") while not exit_signal.is_set(): + start_time = time() # Start loop timer try: ingestor.loop() + metrics.APP_LOOP_COUNT.labels(pod_id=POD_ID).inc() # Increment loop counter exit_signal.wait(ingestor.poll_interval) + except KeyboardInterrupt: # Handle Ctrl+C gracefully + print("KeyboardInterrupt received. Setting exit_signal flag.") + exit_signal.set() except Exception: print("Exception in main loop. Setting exit_signal flag.") traceback.print_exc() + metrics.APP_ERRORS_TOTAL.labels(pod_id=POD_ID).inc() # Increment errors exit_signal.set() + finally: + # Record loop duration + duration = time() - start_time + metrics.APP_LOOP_DURATION.labels(pod_id=POD_ID).observe(duration) ingestor.shutdown() + metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN ingestor.logger.info("Main loop exit_signaled.") + # Give Prometheus a chance to scrape one last time before exiting (optional) + sleep(5) + os._exit(0) @@ -35,6 +56,16 @@ def signal_handler(_signum, _frame): exit_signal.set() +def start_prometheus_server(): + try: + start_http_server(8000) + print("Prometheus server started on port 8000.") + metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP + except Exception as e: + print(f"Failed to start Prometheus server: {e}") + os._exit(1) + + if __name__ == "__main__": signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) diff --git a/ingestor/metrics.py b/ingestor/metrics.py new file mode 100644 index 0000000..c605497 --- /dev/null +++ b/ingestor/metrics.py @@ -0,0 +1,168 @@ +from prometheus_client import Counter, Gauge, Histogram + +# It's useful to have a common set of labels, like the pod_id. +# We'll add 'pod_id' to many metrics to distinguish instances. +POD_ID_LABEL = ["pod_id"] +SERVER_LABELS = ["pod_id", "server_name", "server_url"] +SLOT_LABELS = ["pod_id", "slot_id"] +TAG_LABELS = ["pod_id", "server_name", "tag_id", "tag_name"] +KAFKA_LABELS = ["pod_id", "topic"] +REDIS_LABELS = ["pod_id", "operation"] +NOTIFICATION_LABELS = ["pod_id", "level", "block"] + + +# --- General Application Metrics --- +APP_LOOP_COUNT = Counter( + "app_main_loop_total", + "Total number of times the application main loop has run", + POD_ID_LABEL, +) +APP_LOOP_DURATION = Histogram( + "app_main_loop_duration_seconds", + "Duration of the application main loop in seconds", + POD_ID_LABEL, +) +APP_ERRORS_TOTAL = Counter( + "app_errors_total", + "Total number of unhandled errors in the main loop", + POD_ID_LABEL, +) +APP_UP = Gauge( + "app_up", + "Indicates if the application is running (1) or shutting down (0)", + POD_ID_LABEL, +) + +# --- Ingestor Manager Metrics --- +INGESTORS_ATIVOS = Gauge( + "ingestor_active_total", + "Number of active ingestors reported by Redis", + # No pod_id here, as it's a global view from Redis +) +SLOTS_TOTAIS = Gauge( + "ingestor_slots_total", + "Total number of slots configured in Redis", + # No pod_id here, as it's a global view from Redis +) +LEASES_TOTAIS = Gauge( + "ingestor_leases_total", + "Total number of leases (allocated slots) in Redis", + # No pod_id here, as it's a global view from Redis +) +SLOTS_MANAGED = Gauge( + "ingestor_slots_managed_current", + "Number of slots currently managed by this ingestor instance", + POD_ID_LABEL, +) +SLOTS_ACQUIRED = Counter( + "ingestor_slots_acquired_total", + "Total number of slots acquired by this instance", + POD_ID_LABEL, +) +SLOTS_RELEASED = Counter( + "ingestor_slots_released_total", + "Total number of slots released by this instance", + POD_ID_LABEL, +) +OPC_MANAGERS_ACTIVE = Gauge( + "ingestor_opc_managers_active", + "Number of active OPC Managers in this instance", + POD_ID_LABEL, +) +OPC_SUBSCRIPTION_ERRORS = Counter( + "ingestor_opc_subscription_errors_total", + "Errors when trying to subscribe to OPC tags", + ["pod_id", "server", "slot"], +) + +# --- OPC Manager Metrics --- +OPC_CONNECTIONS_TOTAL = Counter( + "opc_connections_initiated_total", + "Total connection attempts to OPC servers", + ["pod_id", "server_name"], +) +OPC_CONNECTIONS_FAILED = Counter( + "opc_connections_failed_total", + "Total failed connection attempts to OPC servers", + ["pod_id", "server_name"], +) +OPC_CONNECTION_STATUS = Gauge( + "opc_connection_status", + "Connection status with the OPC server (1=connected, 0=disconnected)", + SERVER_LABELS, +) +OPC_SUBSCRIPTIONS_CREATED = Counter( + "opc_subscriptions_created_total", + "Total OPC subscriptions created", + ["pod_id", "server_name", "slot_name"], +) +OPC_TAGS_SUBSCRIBED = Gauge( + "opc_tags_subscribed_current", + "Current number of OPC tags subscribed on a server", + ["pod_id", "server_name"], +) +OPC_DATACHANGE_NOTIFICATIONS = Counter( + "opc_datachange_notifications_total", + "Total data change notifications received (tag readings)", + TAG_LABELS, +) +OPC_TAG_LAST_VALUE = Gauge( + "opc_tag_last_value", "The last value read from an OPC tag", TAG_LABELS +) +OPC_TAG_LAST_READ_TIMESTAMP = Gauge( + "opc_tag_last_read_timestamp_seconds", + "Timestamp of the last value read from an OPC tag", + TAG_LABELS, +) +OPC_CYCLES_WITHOUT_DATA = Gauge( + "opc_cycles_without_data", + "Current number of cycles without receiving data from a server", + ["pod_id", "server_name"], +) +OPC_RECONNECTIONS_TOTAL = Counter( + "opc_reconnections_tried_total", + "Reconnection attempts to an OPC server after a loss", + ["pod_id", "server_name"], +) + +# --- Data Manager (Kafka) Metrics --- +KAFKA_MESSAGES_SENT = Counter( + "kafka_messages_sent_total", "Total messages sent to Kafka", KAFKA_LABELS +) +KAFKA_MESSAGES_ERRORS = Counter( + "kafka_messages_errors_total", + "Total errors sending messages to Kafka", + KAFKA_LABELS, +) +KAFKA_CONNECTION_STATUS = Gauge( + "kafka_connection_status", + "Connection status with Kafka (1=connected, 0=disconnected)", + POD_ID_LABEL, +) + +# --- Resource Manager (Redis) Metrics --- +REDIS_OPERATIONS_TOTAL = Counter( + "redis_operations_total", "Total number of Redis operations performed", REDIS_LABELS +) +REDIS_OPERATIONS_ERRORS = Counter( + "redis_operations_errors_total", + "Total number of errors in Redis operations", + REDIS_LABELS, +) +REDIS_OPERATIONS_DURATION = Histogram( + "redis_operations_duration_seconds", + "Duration of Redis operations in seconds", + REDIS_LABELS, +) +REDIS_CONNECTION_STATUS = Gauge( + "redis_connection_status", + "Connection status with Redis (1=connected, 0=disconnected)", + POD_ID_LABEL, +) + +# --- Notification Metrics --- +NOTIFICATIONS_SENT = Counter( + "notifications_sent_total", + "Total number of notifications sent", + NOTIFICATION_LABELS, +) diff --git a/requirements.txt b/requirements.txt index e564b8e..40a9a06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ asyncua==1.1.5 redis -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git \ No newline at end of file +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git +prometheus_client From 12e9151b7466bcbd84e052f3604590ce373e20e1 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Wed, 28 May 2025 15:42:58 -0300 Subject: [PATCH 16/28] SIENTIAPDE-1083: add unit tests for metrics.py. --- tests/unit/__init__.py | 0 tests/unit/test_metrics.py | 297 +++++++++++++++++++++++++++++++++++++ 2 files changed, 297 insertions(+) create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/test_metrics.py diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_metrics.py b/tests/unit/test_metrics.py new file mode 100644 index 0000000..eb2670a --- /dev/null +++ b/tests/unit/test_metrics.py @@ -0,0 +1,297 @@ +# tests/unit/test_metrics.py + +import pytest +from prometheus_client import Counter, Gauge, Histogram +import ingestor.metrics as metrics + +# --- Test Functions for Each Metric (Corrected for v0.22.0 _name behavior) --- + + +def test_app_loop_count(): + """Verify the definition of APP_LOOP_COUNT.""" + assert metrics.APP_LOOP_COUNT is not None + assert isinstance(metrics.APP_LOOP_COUNT, Counter) + assert metrics.APP_LOOP_COUNT._name == "app_main_loop" # REMOVED _total + assert set(metrics.APP_LOOP_COUNT._labelnames) == {"pod_id"} + + +def test_app_loop_duration(): + """Verify the definition of APP_LOOP_DURATION.""" + assert metrics.APP_LOOP_DURATION is not None + assert isinstance(metrics.APP_LOOP_DURATION, Histogram) + assert ( + metrics.APP_LOOP_DURATION._name == "app_main_loop_duration_seconds" + ) # Histograms don't have _total + assert set(metrics.APP_LOOP_DURATION._labelnames) == {"pod_id"} + + +def test_app_errors_total(): + """Verify the definition of APP_ERRORS_TOTAL.""" + assert metrics.APP_ERRORS_TOTAL is not None + assert isinstance(metrics.APP_ERRORS_TOTAL, Counter) + assert metrics.APP_ERRORS_TOTAL._name == "app_errors" # REMOVED _total + assert set(metrics.APP_ERRORS_TOTAL._labelnames) == {"pod_id"} + + +def test_app_up(): + """Verify the definition of APP_UP.""" + assert metrics.APP_UP is not None + assert isinstance(metrics.APP_UP, Gauge) + assert metrics.APP_UP._name == "app_up" # Gauges don't have _total + assert set(metrics.APP_UP._labelnames) == {"pod_id"} + + +def test_ingestors_ativos(): + """Verify the definition of INGESTORS_ATIVOS.""" + assert metrics.INGESTORS_ATIVOS is not None + assert isinstance(metrics.INGESTORS_ATIVOS, Gauge) + assert metrics.INGESTORS_ATIVOS._name == "ingestor_active_total" + assert set(metrics.INGESTORS_ATIVOS._labelnames) == set() + + +def test_slots_totais(): + """Verify the definition of SLOTS_TOTAIS.""" + assert metrics.SLOTS_TOTAIS is not None + assert isinstance(metrics.SLOTS_TOTAIS, Gauge) + assert metrics.SLOTS_TOTAIS._name == "ingestor_slots_total" + assert set(metrics.SLOTS_TOTAIS._labelnames) == set() + + +def test_leases_totais(): + """Verify the definition of LEASES_TOTAIS.""" + assert metrics.LEASES_TOTAIS is not None + assert isinstance(metrics.LEASES_TOTAIS, Gauge) + assert metrics.LEASES_TOTAIS._name == "ingestor_leases_total" + assert set(metrics.LEASES_TOTAIS._labelnames) == set() + + +def test_slots_managed(): + """Verify the definition of SLOTS_MANAGED.""" + assert metrics.SLOTS_MANAGED is not None + assert isinstance(metrics.SLOTS_MANAGED, Gauge) + assert metrics.SLOTS_MANAGED._name == "ingestor_slots_managed_current" + assert set(metrics.SLOTS_MANAGED._labelnames) == {"pod_id"} + + +def test_slots_acquired(): + """Verify the definition of SLOTS_ACQUIRED.""" + assert metrics.SLOTS_ACQUIRED is not None + assert isinstance(metrics.SLOTS_ACQUIRED, Counter) + assert metrics.SLOTS_ACQUIRED._name == "ingestor_slots_acquired" # REMOVED _total + assert set(metrics.SLOTS_ACQUIRED._labelnames) == {"pod_id"} + + +def test_slots_released(): + """Verify the definition of SLOTS_RELEASED.""" + assert metrics.SLOTS_RELEASED is not None + assert isinstance(metrics.SLOTS_RELEASED, Counter) + assert metrics.SLOTS_RELEASED._name == "ingestor_slots_released" # REMOVED _total + assert set(metrics.SLOTS_RELEASED._labelnames) == {"pod_id"} + + +def test_opc_managers_active(): + """Verify the definition of OPC_MANAGERS_ACTIVE.""" + assert metrics.OPC_MANAGERS_ACTIVE is not None + assert isinstance(metrics.OPC_MANAGERS_ACTIVE, Gauge) + assert metrics.OPC_MANAGERS_ACTIVE._name == "ingestor_opc_managers_active" + assert set(metrics.OPC_MANAGERS_ACTIVE._labelnames) == {"pod_id"} + + +def test_opc_subscription_errors(): + """Verify the definition of OPC_SUBSCRIPTION_ERRORS.""" + assert metrics.OPC_SUBSCRIPTION_ERRORS is not None + assert isinstance(metrics.OPC_SUBSCRIPTION_ERRORS, Counter) + assert ( + metrics.OPC_SUBSCRIPTION_ERRORS._name == "ingestor_opc_subscription_errors" + ) # REMOVED _total + assert set(metrics.OPC_SUBSCRIPTION_ERRORS._labelnames) == { + "pod_id", + "server", + "slot", + } + + +def test_opc_connections_total(): + """Verify the definition of OPC_CONNECTIONS_TOTAL.""" + assert metrics.OPC_CONNECTIONS_TOTAL is not None + assert isinstance(metrics.OPC_CONNECTIONS_TOTAL, Counter) + assert ( + metrics.OPC_CONNECTIONS_TOTAL._name == "opc_connections_initiated" + ) # REMOVED _total + assert set(metrics.OPC_CONNECTIONS_TOTAL._labelnames) == {"pod_id", "server_name"} + + +def test_opc_connections_failed(): + """Verify the definition of OPC_CONNECTIONS_FAILED.""" + assert metrics.OPC_CONNECTIONS_FAILED is not None + assert isinstance(metrics.OPC_CONNECTIONS_FAILED, Counter) + assert ( + metrics.OPC_CONNECTIONS_FAILED._name == "opc_connections_failed" + ) # REMOVED _total + assert set(metrics.OPC_CONNECTIONS_FAILED._labelnames) == {"pod_id", "server_name"} + + +def test_opc_connection_status(): + """Verify the definition of OPC_CONNECTION_STATUS.""" + assert metrics.OPC_CONNECTION_STATUS is not None + assert isinstance(metrics.OPC_CONNECTION_STATUS, Gauge) + assert metrics.OPC_CONNECTION_STATUS._name == "opc_connection_status" + assert set(metrics.OPC_CONNECTION_STATUS._labelnames) == { + "pod_id", + "server_name", + "server_url", + } + + +def test_opc_subscriptions_created(): + """Verify the definition of OPC_SUBSCRIPTIONS_CREATED.""" + assert metrics.OPC_SUBSCRIPTIONS_CREATED is not None + assert isinstance(metrics.OPC_SUBSCRIPTIONS_CREATED, Counter) + assert ( + metrics.OPC_SUBSCRIPTIONS_CREATED._name == "opc_subscriptions_created" + ) # REMOVED _total + assert set(metrics.OPC_SUBSCRIPTIONS_CREATED._labelnames) == { + "pod_id", + "server_name", + "slot_name", + } + + +def test_opc_tags_subscribed(): + """Verify the definition of OPC_TAGS_SUBSCRIBED.""" + assert metrics.OPC_TAGS_SUBSCRIBED is not None + assert isinstance(metrics.OPC_TAGS_SUBSCRIBED, Gauge) + assert metrics.OPC_TAGS_SUBSCRIBED._name == "opc_tags_subscribed_current" + assert set(metrics.OPC_TAGS_SUBSCRIBED._labelnames) == {"pod_id", "server_name"} + + +def test_opc_datachange_notifications(): + """Verify the definition of OPC_DATACHANGE_NOTIFICATIONS.""" + assert metrics.OPC_DATACHANGE_NOTIFICATIONS is not None + assert isinstance(metrics.OPC_DATACHANGE_NOTIFICATIONS, Counter) + assert ( + metrics.OPC_DATACHANGE_NOTIFICATIONS._name == "opc_datachange_notifications" + ) # REMOVED _total + assert set(metrics.OPC_DATACHANGE_NOTIFICATIONS._labelnames) == { + "pod_id", + "server_name", + "tag_id", + "tag_name", + } + + +def test_opc_tag_last_value(): + """Verify the definition of OPC_TAG_LAST_VALUE.""" + assert metrics.OPC_TAG_LAST_VALUE is not None + assert isinstance(metrics.OPC_TAG_LAST_VALUE, Gauge) + assert metrics.OPC_TAG_LAST_VALUE._name == "opc_tag_last_value" + assert set(metrics.OPC_TAG_LAST_VALUE._labelnames) == { + "pod_id", + "server_name", + "tag_id", + "tag_name", + } + + +def test_opc_tag_last_read_timestamp(): + """Verify the definition of OPC_TAG_LAST_READ_TIMESTAMP.""" + assert metrics.OPC_TAG_LAST_READ_TIMESTAMP is not None + assert isinstance(metrics.OPC_TAG_LAST_READ_TIMESTAMP, Gauge) + assert ( + metrics.OPC_TAG_LAST_READ_TIMESTAMP._name + == "opc_tag_last_read_timestamp_seconds" + ) + assert set(metrics.OPC_TAG_LAST_READ_TIMESTAMP._labelnames) == { + "pod_id", + "server_name", + "tag_id", + "tag_name", + } + + +def test_opc_cycles_without_data(): + """Verify the definition of OPC_CYCLES_WITHOUT_DATA.""" + assert metrics.OPC_CYCLES_WITHOUT_DATA is not None + assert isinstance(metrics.OPC_CYCLES_WITHOUT_DATA, Gauge) + assert metrics.OPC_CYCLES_WITHOUT_DATA._name == "opc_cycles_without_data" + assert set(metrics.OPC_CYCLES_WITHOUT_DATA._labelnames) == {"pod_id", "server_name"} + + +def test_opc_reconnections_total(): + """Verify the definition of OPC_RECONNECTIONS_TOTAL.""" + assert metrics.OPC_RECONNECTIONS_TOTAL is not None + assert isinstance(metrics.OPC_RECONNECTIONS_TOTAL, Counter) + assert ( + metrics.OPC_RECONNECTIONS_TOTAL._name == "opc_reconnections_tried" + ) # REMOVED _total + assert set(metrics.OPC_RECONNECTIONS_TOTAL._labelnames) == {"pod_id", "server_name"} + + +def test_kafka_messages_sent(): + """Verify the definition of KAFKA_MESSAGES_SENT.""" + assert metrics.KAFKA_MESSAGES_SENT is not None + assert isinstance(metrics.KAFKA_MESSAGES_SENT, Counter) + assert metrics.KAFKA_MESSAGES_SENT._name == "kafka_messages_sent" # REMOVED _total + assert set(metrics.KAFKA_MESSAGES_SENT._labelnames) == {"pod_id", "topic"} + + +def test_kafka_messages_errors(): + """Verify the definition of KAFKA_MESSAGES_ERRORS.""" + assert metrics.KAFKA_MESSAGES_ERRORS is not None + assert isinstance(metrics.KAFKA_MESSAGES_ERRORS, Counter) + assert ( + metrics.KAFKA_MESSAGES_ERRORS._name == "kafka_messages_errors" + ) # REMOVED _total + assert set(metrics.KAFKA_MESSAGES_ERRORS._labelnames) == {"pod_id", "topic"} + + +def test_kafka_connection_status(): + """Verify the definition of KAFKA_CONNECTION_STATUS.""" + assert metrics.KAFKA_CONNECTION_STATUS is not None + assert isinstance(metrics.KAFKA_CONNECTION_STATUS, Gauge) + assert metrics.KAFKA_CONNECTION_STATUS._name == "kafka_connection_status" + assert set(metrics.KAFKA_CONNECTION_STATUS._labelnames) == {"pod_id"} + + +def test_redis_operations_total(): + """Verify the definition of REDIS_OPERATIONS_TOTAL.""" + assert metrics.REDIS_OPERATIONS_TOTAL is not None + assert isinstance(metrics.REDIS_OPERATIONS_TOTAL, Counter) + assert metrics.REDIS_OPERATIONS_TOTAL._name == "redis_operations" # REMOVED _total + assert set(metrics.REDIS_OPERATIONS_TOTAL._labelnames) == {"pod_id", "operation"} + + +def test_redis_operations_errors(): + """Verify the definition of REDIS_OPERATIONS_ERRORS.""" + assert metrics.REDIS_OPERATIONS_ERRORS is not None + assert isinstance(metrics.REDIS_OPERATIONS_ERRORS, Counter) + assert ( + metrics.REDIS_OPERATIONS_ERRORS._name == "redis_operations_errors" + ) # REMOVED _total + assert set(metrics.REDIS_OPERATIONS_ERRORS._labelnames) == {"pod_id", "operation"} + + +def test_redis_operations_duration(): + """Verify the definition of REDIS_OPERATIONS_DURATION.""" + assert metrics.REDIS_OPERATIONS_DURATION is not None + assert isinstance(metrics.REDIS_OPERATIONS_DURATION, Histogram) + assert ( + metrics.REDIS_OPERATIONS_DURATION._name == "redis_operations_duration_seconds" + ) + assert set(metrics.REDIS_OPERATIONS_DURATION._labelnames) == {"pod_id", "operation"} + + +def test_redis_connection_status(): + """Verify the definition of REDIS_CONNECTION_STATUS.""" + assert metrics.REDIS_CONNECTION_STATUS is not None + assert isinstance(metrics.REDIS_CONNECTION_STATUS, Gauge) + assert metrics.REDIS_CONNECTION_STATUS._name == "redis_connection_status" + assert set(metrics.REDIS_CONNECTION_STATUS._labelnames) == {"pod_id"} + + +def test_notifications_sent(): + """Verify the definition of NOTIFICATIONS_SENT.""" + assert metrics.NOTIFICATIONS_SENT is not None + assert isinstance(metrics.NOTIFICATIONS_SENT, Counter) + assert metrics.NOTIFICATIONS_SENT._name == "notifications_sent" # REMOVED _total + assert set(metrics.NOTIFICATIONS_SENT._labelnames) == {"pod_id", "level", "block"} From fff52acf797b759fc84cc771953a2150833fbbbb Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Wed, 28 May 2025 16:15:57 -0300 Subject: [PATCH 17/28] SIENTIAPDE-1083: add unit tests for app.py. --- tests/unit/test_app.py | 277 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 tests/unit/test_app.py diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py new file mode 100644 index 0000000..dda7701 --- /dev/null +++ b/tests/unit/test_app.py @@ -0,0 +1,277 @@ +import pytest +from unittest.mock import patch, MagicMock, call +from threading import Event +import signal as signal_module # To avoid conflict with mock names +import os + +# Import the 'app' module to be tested +from ingestor import app + + +# Custom exception to catch os._exit calls +class OsExitCalled(Exception): + def __init__(self, code): + super().__init__(f"os._exit({code}) called") + self.code = code + + +# Helper function for the os_exit mock's side_effect +def raise_os_exit_with_code(exit_code): + raise OsExitCalled(exit_code) + + +@pytest.fixture +def mock_app_env(monkeypatch): + """Fixture to mock dependencies of app.main and app.signal_handler.""" + mocks = { + "start_http_server": MagicMock(), + "Ingestor": MagicMock(), + "metrics_APP_UP_labels_set": MagicMock(), + "metrics_APP_LOOP_COUNT_labels_inc": MagicMock(), + "metrics_APP_LOOP_DURATION_labels_observe": MagicMock(), + "metrics_APP_ERRORS_TOTAL_labels_inc": MagicMock(), + "os_exit": MagicMock(side_effect=raise_os_exit_with_code), # CORRECTED + "time_time": MagicMock(), + "time_sleep": MagicMock(), + "signal_signal": MagicMock(), + "traceback_print_exc": MagicMock(), + "mock_exit_signal": MagicMock(spec=Event), + } + + monkeypatch.setattr(app, "start_http_server", mocks["start_http_server"]) + monkeypatch.setattr(app, "Ingestor", mocks["Ingestor"]) + + monkeypatch.setattr( + app.metrics.APP_UP, + "labels", + MagicMock(return_value=MagicMock(set=mocks["metrics_APP_UP_labels_set"])), + ) + monkeypatch.setattr( + app.metrics.APP_LOOP_COUNT, + "labels", + MagicMock( + return_value=MagicMock(inc=mocks["metrics_APP_LOOP_COUNT_labels_inc"]) + ), + ) + monkeypatch.setattr( + app.metrics.APP_LOOP_DURATION, + "labels", + MagicMock( + return_value=MagicMock( + observe=mocks["metrics_APP_LOOP_DURATION_labels_observe"] + ) + ), + ) + monkeypatch.setattr( + app.metrics.APP_ERRORS_TOTAL, + "labels", + MagicMock( + return_value=MagicMock(inc=mocks["metrics_APP_ERRORS_TOTAL_labels_inc"]) + ), + ) + + monkeypatch.setattr(app.os, "_exit", mocks["os_exit"]) + monkeypatch.setattr(app, "time", mocks["time_time"]) + monkeypatch.setattr(app, "sleep", mocks["time_sleep"]) + monkeypatch.setattr(app.signal, "signal", mocks["signal_signal"]) + monkeypatch.setattr(app.traceback, "print_exc", mocks["traceback_print_exc"]) + + monkeypatch.setattr(app, "exit_signal", mocks["mock_exit_signal"]) + monkeypatch.setattr(app, "POD_ID", "test_pod") + + mock_ingestor_instance = mocks["Ingestor"].return_value + mock_ingestor_instance.poll_interval = 0.01 + mock_ingestor_instance.logger = MagicMock() + + return mocks + + +def test_main_successful_run_one_loop(mock_app_env, capsys): + """Test a successful run where the loop executes once and then exits gracefully.""" + mock_ingestor_instance = mock_app_env["Ingestor"].return_value + mock_exit_signal = mock_app_env["mock_exit_signal"] + + mock_exit_signal.is_set.side_effect = [False, True] + mock_app_env["time_time"].side_effect = [10.0, 11.5] + + with pytest.raises(OsExitCalled) as excinfo: + app.main() + assert excinfo.value.code == 0 + + mock_app_env["start_http_server"].assert_called_once_with(8000) + app.metrics.APP_UP.labels.assert_any_call(pod_id="test_pod") + set_calls = mock_app_env["metrics_APP_UP_labels_set"].call_args_list + assert call(1) in set_calls + assert call(0) in set_calls + assert set_calls.index(call(1)) < set_calls.index(call(0)) + + mock_app_env["Ingestor"].assert_called_once_with() + mock_ingestor_instance.prepare_ingestor.assert_called_once() + mock_ingestor_instance.logger.info.assert_any_call( + "Ingestor prepared. Starting main loop." + ) + + mock_ingestor_instance.loop.assert_called_once() + app.metrics.APP_LOOP_COUNT.labels.assert_called_with(pod_id="test_pod") + mock_app_env["metrics_APP_LOOP_COUNT_labels_inc"].assert_called_once() + + mock_exit_signal.wait.assert_called_once_with(mock_ingestor_instance.poll_interval) + + app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id="test_pod") + mock_app_env["metrics_APP_LOOP_DURATION_labels_observe"].assert_called_once_with( + 1.5 + ) + + mock_ingestor_instance.shutdown.assert_called_once() + mock_app_env["time_sleep"].assert_called_once_with(5) + # CORREÇÃO APLICADA ABAIXO: + mock_ingestor_instance.logger.info.assert_any_call("Main loop exit_signaled.") + + captured = capsys.readouterr() + assert "Prometheus server started on port 8000." in captured.out + + +def test_main_prometheus_server_fails_to_start(mock_app_env, capsys): + """Test the scenario where starting the Prometheus server fails.""" + mock_app_env["start_http_server"].side_effect = OSError("Port already in use") + + with pytest.raises(OsExitCalled) as excinfo: + app.main() + assert excinfo.value.code == 1 + + # Verify that APP_UP.labels(...).set(1) was NOT called. + # The mock for .set is mock_app_env['metrics_APP_UP_labels_set'] + # We need to check if it was called with 1. + # A more robust way is to check if the specific .labels(pod_id="test_pod") mock was ever called + # and then its .set(1) method. + # For simplicity here, we check if metrics_APP_UP_labels_set was ever called with 1. + + # Check that .set(1) was not called. .set(0) definitely not called. + called_with_1 = False + for call_args in mock_app_env["metrics_APP_UP_labels_set"].call_args_list: + if call_args == call(1): + called_with_1 = True + break + assert ( + not called_with_1 + ), "APP_UP.set(1) should not have been called if server start failed" + + mock_app_env["Ingestor"].assert_not_called() + captured = capsys.readouterr() + assert "Failed to start Prometheus server: Port already in use" in captured.out + + +def test_main_loop_exception_handling(mock_app_env, capsys): + """Test that an exception in ingestor.loop() is handled gracefully.""" + mock_ingestor_instance = mock_app_env["Ingestor"].return_value + mock_exit_signal = mock_app_env["mock_exit_signal"] + + mock_exit_signal.is_set.side_effect = [False, True] + mock_ingestor_instance.loop.side_effect = Exception("Test loop exception") + mock_app_env["time_time"].side_effect = [10.0, 10.1] + + with pytest.raises(OsExitCalled) as excinfo: + app.main() + assert excinfo.value.code == 0 + + mock_ingestor_instance.loop.assert_called_once() + mock_app_env["traceback_print_exc"].assert_called_once() + + app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id="test_pod") + mock_app_env["metrics_APP_ERRORS_TOTAL_labels_inc"].assert_called_once() + + mock_exit_signal.set.assert_called_once() + + mock_app_env["metrics_APP_LOOP_COUNT_labels_inc"].assert_not_called() + + app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id="test_pod") + mock_app_env["metrics_APP_LOOP_DURATION_labels_observe"].assert_called_once_with( + pytest.approx(0.1) + ) + + mock_ingestor_instance.shutdown.assert_called_once() + captured = capsys.readouterr() + assert "Exception in main loop. Setting exit_signal flag." in captured.out + + +def test_main_keyboard_interrupt_handling(mock_app_env, capsys): + """Test that KeyboardInterrupt in ingestor.loop() is handled.""" + mock_ingestor_instance = mock_app_env["Ingestor"].return_value + mock_exit_signal = mock_app_env["mock_exit_signal"] + + mock_exit_signal.is_set.side_effect = [False, True] + mock_ingestor_instance.loop.side_effect = KeyboardInterrupt() + mock_app_env["time_time"].side_effect = [10.0, 10.1] + + with pytest.raises(OsExitCalled) as excinfo: + app.main() + assert excinfo.value.code == 0 + + mock_ingestor_instance.loop.assert_called_once() + mock_exit_signal.set.assert_called_once() + mock_ingestor_instance.shutdown.assert_called_once() + captured = capsys.readouterr() + assert "KeyboardInterrupt received. Setting exit_signal flag." in captured.out + mock_app_env["metrics_APP_ERRORS_TOTAL_labels_inc"].assert_not_called() + + +def test_signal_handler_sets_exit_signal(mock_app_env): + """Test that the signal_handler function calls exit_signal.set().""" + mock_exit_signal_set = mock_app_env["mock_exit_signal"].set + + app.signal_handler(signal_module.SIGINT, None) + mock_exit_signal_set.assert_called_once() + + +def test_main_multiple_loop_iterations(mock_app_env): + """Test the main loop runs for a few iterations.""" + mock_ingestor_instance = mock_app_env["Ingestor"].return_value + mock_exit_signal = mock_app_env["mock_exit_signal"] + + mock_exit_signal.is_set.side_effect = [False, False, False, True] + mock_app_env["time_time"].side_effect = [10.0, 10.1, 10.2, 10.3, 10.4, 10.5] + + with pytest.raises(OsExitCalled) as excinfo: + app.main() + assert excinfo.value.code == 0 + + assert mock_ingestor_instance.loop.call_count == 3 + + assert app.metrics.APP_LOOP_COUNT.labels.call_count == 3 + app.metrics.APP_LOOP_COUNT.labels.assert_called_with( + pod_id="test_pod" + ) # Checks last call or any call + assert mock_app_env["metrics_APP_LOOP_COUNT_labels_inc"].call_count == 3 + + assert mock_exit_signal.wait.call_count == 3 + + assert app.metrics.APP_LOOP_DURATION.labels.call_count == 3 + app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id="test_pod") + duration_calls = mock_app_env[ + "metrics_APP_LOOP_DURATION_labels_observe" + ].call_args_list + assert duration_calls[0] == call(pytest.approx(0.1, abs=1e-9)) + assert duration_calls[1] == call(pytest.approx(0.1, abs=1e-9)) + assert duration_calls[2] == call(pytest.approx(0.1, abs=1e-9)) + + mock_ingestor_instance.shutdown.assert_called_once() + + +def test_main_pod_id_used_in_metrics(mock_app_env): + """Test that the POD_ID from app module is used in metric labels.""" + mock_exit_signal = mock_app_env["mock_exit_signal"] + mock_exit_signal.is_set.side_effect = [False, True] + mock_app_env["time_time"].side_effect = [10.0, 11.0] + + with pytest.raises(OsExitCalled): + app.main() + + app.metrics.APP_UP.labels.assert_any_call(pod_id="test_pod") + app.metrics.APP_LOOP_COUNT.labels.assert_any_call(pod_id="test_pod") + app.metrics.APP_LOOP_DURATION.labels.assert_any_call(pod_id="test_pod") + # APP_ERRORS_TOTAL would be checked similarly if it were called in this flow. + + # Check the .set() / .inc() calls on the mocks returned by .labels() + mock_app_env["metrics_APP_UP_labels_set"].assert_any_call(1) + mock_app_env["metrics_APP_UP_labels_set"].assert_any_call(0) + mock_app_env["metrics_APP_LOOP_COUNT_labels_inc"].assert_called_once() From 8e332da5971c8e62008ef06eec04c914a29b8001 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Wed, 28 May 2025 17:54:44 -0300 Subject: [PATCH 18/28] SIENTIAPDE-1083: add prometheus metrics to ingestor. --- .github/workflows/quality-gate.yml | 2 +- .vscode/settings.json | 21 +++++---- ingestor/ingestor.py | 74 ++++++++++++++++++++++-------- ingestor/metrics.py | 6 +-- tests/unit/test_metrics.py | 36 +++++++-------- 5 files changed, 87 insertions(+), 52 deletions(-) diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index e13ecb2..cb52e6d 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -80,5 +80,5 @@ jobs: -Dsonar.host.url=$SONAR_HOST_URL \ -Dsonar.token=$SONAR_TOKEN \ -Dsonar.python.version=3.11 \ - -Dsonar.projectVersion=1.0.1 \ + -Dsonar.projectVersion=1.2.0 \ -Dsonar.coverage.exclusions=ingestor/app.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 78e50ab..bdd01e8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,11 +1,12 @@ { - "python.testing.pytestArgs": [ - "." - ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "sonarlint.connectedMode.project": { - "connectionId": "sonardev-sientia-ai", - "projectKey": "Aignosi_sientia-dataops-opc-ingestor_1642c8bf-a148-4911-8362-0903d7fef99a" - } -} \ No newline at end of file + "python.testing.pytestArgs": ["."], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "sonarlint.connectedMode.project": { + "connectionId": "sonardev-sientia-ai", + "projectKey": "Aignosi_sientia-dataops-opc-ingestor_1642c8bf-a148-4911-8362-0903d7fef99a" + }, + "python.languageServer": "Pylance", + "python.analysis.typeCheckingMode": "off", + "editor.suggestSelection": "first" +} diff --git a/ingestor/ingestor.py b/ingestor/ingestor.py index b508b56..883af54 100644 --- a/ingestor/ingestor.py +++ b/ingestor/ingestor.py @@ -7,6 +7,8 @@ from sientia_do.notifications.handlers import NotificationHandler from ingestor.managers.ingestor_manager import IngestorManager +import ingestor.metrics as metrics + class Ingestor: def __init__(self): @@ -33,13 +35,13 @@ class Ingestor: kafka_servers = getenv("KAFKA_SERVERS", "localhost:9092") self.redis_host = getenv("REDIS_HOST", "localhost") - self.redis_port = int(getenv("REDIS_PORT", '6379')) + self.redis_port = int(getenv("REDIS_PORT", "6379")) self.redis_username = getenv("REDIS_USERNAME", None) self.redis_password = getenv("REDIS_PASSWORD", None) - self.lease_ttl = int(getenv("LEASE_TTL", '10')) - self.heartbeat_ttl = int(getenv("HEARTBEAT_TTL", '20')) + self.lease_ttl = int(getenv("LEASE_TTL", "10")) + self.heartbeat_ttl = int(getenv("HEARTBEAT_TTL", "20")) self.pod_id = getenv("HOSTNAME", "localhost") - self.poll_interval = int(getenv("POLL_INTERVAL", '5')) + self.poll_interval = int(getenv("POLL_INTERVAL", "5")) self.kafka_servers = kafka_servers.split(",") self.logger = None @@ -51,7 +53,7 @@ class Ingestor: pipeline_name="-", trigger_name="-", model_name="-", - model="-" + model="-", ) self.ingestor_manager = None @@ -77,8 +79,7 @@ class Ingestor: logger = getLogger(__name__) logger.setLevel(getenv("LOG_LEVEL", "INFO")) handler = StreamHandler() - formatter = Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) @@ -128,9 +129,17 @@ class Ingestor: """ self.ingestor_manager = IngestorManager( - self.kafka_servers, self.redis_host, self.redis_port, self.lease_ttl, - self.heartbeat_ttl, self.pod_id, self.poll_interval, self.logger, - self.notification_handler, self.redis_username, self.redis_password + self.kafka_servers, + self.redis_host, + self.redis_port, + self.lease_ttl, + self.heartbeat_ttl, + self.pod_id, + self.poll_interval, + self.logger, + self.notification_handler, + self.redis_username, + self.redis_password, ) # Declare ingestor ative @@ -142,6 +151,10 @@ class Ingestor: self.handle_acquired_tags(acquired) + metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set( + len(self.ingestor_manager.managed_tags) + ) # Set initial + def manage_no_slots(self, number_of_slots: int): """ Manages the scenario where there are no slots assigned to the ingestor. @@ -159,7 +172,9 @@ class Ingestor: # Get slot lease self.ingestor_manager.get_slot_leases(1) - def manage_leases(self, available_slots: int, lacking_ingestors: int, slot_diff: int): + def manage_leases( + self, available_slots: int, lacking_ingestors: int, slot_diff: int + ): """ Manages the allocation and deallocation of slot leases for ingestors based on the number of available slots, lacking ingestors, and slot differences. @@ -198,6 +213,11 @@ class Ingestor: self.ingestor_manager.unsubscribe_slot(lease) self.ingestor_manager.managed_tags.pop(lease) + # Update metric after removal + metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set( + len(self.ingestor_manager.managed_tags) + ) + def update_ingestor_manager(self, old_managed_tags: Dict[str, Any]): """ Updates the ingestor manager with the new managed tags. @@ -205,15 +225,18 @@ class Ingestor: old_managed_tags (Dict[str, Any]): The old managed tags. """ - self.logger.debug("Current managed tags: %s", - self.ingestor_manager.managed_tags) + self.logger.debug( + "Current managed tags: %s", self.ingestor_manager.managed_tags + ) self.ingestor_manager.update_opc_servers() new_managed_tags = deepcopy(self.ingestor_manager.managed_tags) - self.logger.debug("Comparing new managed tags %s" - "with old managed tags %s", - new_managed_tags, old_managed_tags) + self.logger.debug( + "Comparing new managed tags %s with old managed tags %s", + new_managed_tags, + old_managed_tags, + ) for slot, config in new_managed_tags.items(): if slot not in old_managed_tags: @@ -228,6 +251,11 @@ class Ingestor: if slot not in new_managed_tags: self.ingestor_manager.unsubscribe_slot(slot) + # Ensure the gauge is updated after any potential changes here + metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set( + len(self.ingestor_manager.managed_tags) + ) + def loop(self): """ Executes the main loop for managing ingestors and slots. @@ -259,6 +287,9 @@ class Ingestor: number_of_leases = self.ingestor_manager.get_number_of_leases() number_of_slots = self.ingestor_manager.get_number_of_slots() + # Update active ingestors gauge + metrics.ACTIVE_INGESTORS.set(number_of_ingestors) + # Handle no slots self.logger.debug("Managing no slots...") self.manage_no_slots(number_of_slots) @@ -270,6 +301,11 @@ class Ingestor: self.logger.debug("Managing leases...") self.manage_leases(available_slots, lacking_ingestors, slot_diff) + # Update managed slots gauge + metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set( + len(self.ingestor_manager.managed_tags) + ) + self.logger.debug( "Active ingestors: %s, " "Number of slots: %s, " @@ -280,7 +316,7 @@ class Ingestor: number_of_slots, number_of_leases, self.ingestor_manager.managed_tags, - self.ingestor_manager.opc_managers + self.ingestor_manager.opc_managers, ) if not self.ingestor_manager.managed_tags: # No slots acquired @@ -295,6 +331,4 @@ class Ingestor: self.ingestor_manager.check_opc_servers_integrity() self.logger.debug("Updating managed tags...") - self.update_ingestor_manager( - current_managed_tags - ) + self.update_ingestor_manager(current_managed_tags) diff --git a/ingestor/metrics.py b/ingestor/metrics.py index c605497..bb0976d 100644 --- a/ingestor/metrics.py +++ b/ingestor/metrics.py @@ -34,17 +34,17 @@ APP_UP = Gauge( ) # --- Ingestor Manager Metrics --- -INGESTORS_ATIVOS = Gauge( +ACTIVE_INGESTORS = Gauge( "ingestor_active_total", "Number of active ingestors reported by Redis", # No pod_id here, as it's a global view from Redis ) -SLOTS_TOTAIS = Gauge( +SLOTS_TOTAL = Gauge( "ingestor_slots_total", "Total number of slots configured in Redis", # No pod_id here, as it's a global view from Redis ) -LEASES_TOTAIS = Gauge( +LEASES_TOTAL = Gauge( "ingestor_leases_total", "Total number of leases (allocated slots) in Redis", # No pod_id here, as it's a global view from Redis diff --git a/tests/unit/test_metrics.py b/tests/unit/test_metrics.py index eb2670a..8d9ad7c 100644 --- a/tests/unit/test_metrics.py +++ b/tests/unit/test_metrics.py @@ -41,28 +41,28 @@ def test_app_up(): assert set(metrics.APP_UP._labelnames) == {"pod_id"} -def test_ingestors_ativos(): - """Verify the definition of INGESTORS_ATIVOS.""" - assert metrics.INGESTORS_ATIVOS is not None - assert isinstance(metrics.INGESTORS_ATIVOS, Gauge) - assert metrics.INGESTORS_ATIVOS._name == "ingestor_active_total" - assert set(metrics.INGESTORS_ATIVOS._labelnames) == set() +def test_active_ingestors(): + """Verify the definition of ACTIVE_INGESTORS.""" + assert metrics.ACTIVE_INGESTORS is not None + assert isinstance(metrics.ACTIVE_INGESTORS, Gauge) + assert metrics.ACTIVE_INGESTORS._name == "ingestor_active_total" + assert set(metrics.ACTIVE_INGESTORS._labelnames) == set() -def test_slots_totais(): - """Verify the definition of SLOTS_TOTAIS.""" - assert metrics.SLOTS_TOTAIS is not None - assert isinstance(metrics.SLOTS_TOTAIS, Gauge) - assert metrics.SLOTS_TOTAIS._name == "ingestor_slots_total" - assert set(metrics.SLOTS_TOTAIS._labelnames) == set() +def test_slots_total(): + """Verify the definition of SLOTS_TOTAL.""" + assert metrics.SLOTS_TOTAL is not None + assert isinstance(metrics.SLOTS_TOTAL, Gauge) + assert metrics.SLOTS_TOTAL._name == "ingestor_slots_total" + assert set(metrics.SLOTS_TOTAL._labelnames) == set() -def test_leases_totais(): - """Verify the definition of LEASES_TOTAIS.""" - assert metrics.LEASES_TOTAIS is not None - assert isinstance(metrics.LEASES_TOTAIS, Gauge) - assert metrics.LEASES_TOTAIS._name == "ingestor_leases_total" - assert set(metrics.LEASES_TOTAIS._labelnames) == set() +def test_leases_total(): + """Verify the definition of LEASES_TOTAL.""" + assert metrics.LEASES_TOTAL is not None + assert isinstance(metrics.LEASES_TOTAL, Gauge) + assert metrics.LEASES_TOTAL._name == "ingestor_leases_total" + assert set(metrics.LEASES_TOTAL._labelnames) == set() def test_slots_managed(): From f71b4c793718edf3d1eddc85ee488c9b342a717c Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Wed, 28 May 2025 21:30:28 -0300 Subject: [PATCH 19/28] SIENTIAPDE-1083: add prometheus metrics to data_manager. --- .vscode/settings.json | 2 +- ingestor/managers/data_manager.py | 63 +++++++++++++++++++------------ 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index bdd01e8..a3a53a3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,6 +7,6 @@ "projectKey": "Aignosi_sientia-dataops-opc-ingestor_1642c8bf-a148-4911-8362-0903d7fef99a" }, "python.languageServer": "Pylance", - "python.analysis.typeCheckingMode": "off", + "python.analysis.typeCheckingMode": "basic", "editor.suggestSelection": "first" } diff --git a/ingestor/managers/data_manager.py b/ingestor/managers/data_manager.py index b16b933..bb2c1e4 100644 --- a/ingestor/managers/data_manager.py +++ b/ingestor/managers/data_manager.py @@ -6,11 +6,17 @@ from kafka.errors import NoBrokersAvailable from sientia_do.notifications.handlers import NotificationHandler from sientia_do.notifications.models import NotificationLevel import traceback +import ingestor.metrics as metrics +import os -class DataManager(): - def __init__(self, kafka_servers: str, logger: Logger, - notification_handler: NotificationHandler) -> None: +class DataManager: + def __init__( + self, + kafka_servers: str, + logger: Logger, + notification_handler: NotificationHandler, + ) -> None: """ Initializes the DataManager instance with a Kafka producer. This constructor attempts to establish a connection to the specified Kafka servers @@ -23,31 +29,39 @@ class DataManager(): NoBrokersAvailable: If the connection to Kafka servers fails after 3 attempts. """ + self.pod_id = os.getenv("HOSTNAME", "localhost") self.kafka_producer = None for i in range(0, 3): logger.info( - f"Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}") + f"Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}" + ) try: self.kafka_producer = KafkaProducer( bootstrap_servers=kafka_servers, value_serializer=lambda v: json.dumps(v).encode( - 'utf-8'), # Serialize JSON messages - key_serializer=lambda k: str( - k).encode('utf-8') if k else None, + "utf-8" + ), # Serialize JSON messages + key_serializer=lambda k: str(k).encode("utf-8") if k else None, ) + # Kafka connected + metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1) break except NoBrokersAvailable: logger.error( - f"Kafka servers {kafka_servers} are not available. Retrying...") + f"Kafka servers {kafka_servers} are not available. Retrying..." + ) sleep(5) else: + # Kafka not connected + metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0) logger.error( - f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts.") + f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts." + ) raise NoBrokersAvailable( - f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts.") + f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts." + ) - logger.info( - f"DataManager initialized with Kafka servers: {kafka_servers}") + logger.info(f"DataManager initialized with Kafka servers: {kafka_servers}") self.logger = logger self.notification_handler = notification_handler @@ -57,12 +71,12 @@ class DataManager(): try: self.kafka_producer.flush(timeout=10) self.kafka_producer.close() + # Mark as disconnected + metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0) except Exception as e: - self.logger.error( - f"Error closing Kafka producer: {e}") + self.logger.error(f"Error closing Kafka producer: {e}") else: - self.logger.warning( - "Kafka producer is already closed or not initialized.") + self.logger.warning("Kafka producer is already closed or not initialized.") def __del__(self): self.shutdown() @@ -70,7 +84,8 @@ class DataManager(): def delivery_report(self, msg: str): """Callback for delivery reports from Kafka.""" self.logger.debug( - f"Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}") + f"Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}" + ) def delivery_error(self, err: str): """Callback for delivery reports from Kafka.""" @@ -93,22 +108,22 @@ class DataManager(): try: - self.logger.debug( - f"Publishing message to topic {topic}: {data}") - self.kafka_producer.send( - topic=topic, value=data).add_callback( - self.delivery_report).add_errback( - self.delivery_error) + self.logger.debug(f"Publishing message to topic {topic}: {data}") + self.kafka_producer.send(topic=topic, value=data).add_callback( + self.delivery_report + ).add_errback(self.delivery_error) self.kafka_producer.flush(timeout=10) + metrics.KAFKA_MESSAGES_SENT.labels(pod_id=self.pod_id, topic=topic).inc() except Exception as e: + metrics.KAFKA_MESSAGES_ERRORS.labels(pod_id=self.pod_id, topic=topic).inc() trace = traceback.format_exc() self.notification_handler.build_and_send_notification( notification_id=f"KAFKA_PRODUCER_ERROR_{topic}", message=f"Error publishing message to topic {topic}: {e}", block="kafka_producer", level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) self.logger.error(trace) From 03ed5784b1793d7ad3591931bb0191d572de144e Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Thu, 29 May 2025 11:38:57 -0300 Subject: [PATCH 20/28] SIENTIAPDE-1083: add prometheus metrics to resource_manager. --- .vscode/settings.json | 2 +- ingestor/managers/resource_manager.py | 99 ++++++++--- tests/unit/managers/__init__.py | 0 tests/unit/managers/test_resource_manager.py | 173 ++++++++++++++----- 4 files changed, 205 insertions(+), 69 deletions(-) create mode 100644 tests/unit/managers/__init__.py diff --git a/.vscode/settings.json b/.vscode/settings.json index a3a53a3..a595a07 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,6 +7,6 @@ "projectKey": "Aignosi_sientia-dataops-opc-ingestor_1642c8bf-a148-4911-8362-0903d7fef99a" }, "python.languageServer": "Pylance", - "python.analysis.typeCheckingMode": "basic", + "python.analysis.typeCheckingMode": "standard", "editor.suggestSelection": "first" } diff --git a/ingestor/managers/resource_manager.py b/ingestor/managers/resource_manager.py index 2c30198..d958d85 100644 --- a/ingestor/managers/resource_manager.py +++ b/ingestor/managers/resource_manager.py @@ -1,17 +1,59 @@ import json from typing import List from redis import Redis +from time import time +import ingestor.metrics as metrics class ResourceManager: - def __init__(self, host: str, port: int, - lease_ttl: int, heartbeat_ttl: int, pod_id: str, - username: str = None, password: str = None) -> None: - self.redis = Redis(host=host, port=port, decode_responses=True, - username=username, password=password) + def __init__( + self, + host: str, + port: int, + lease_ttl: int, + heartbeat_ttl: int, + pod_id: str, + username: str | None = None, + password: str | None = None, + ) -> None: + self.pod_id = pod_id + try: + self.redis = Redis( + host=host, + port=port, + decode_responses=True, + username=username, + password=password, + ) + self.redis.ping() + metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1) + except Exception as e: + print(f"Failed to connect to Redis: {e}") + metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0) + raise + self.lease_ttl = lease_ttl self.heartbeat_ttl = heartbeat_ttl - self.pod_id = pod_id + + def _execute_redis_op(self, operation_name: str, func, *args, **kwargs): + """Wrapper to execute Redis operations and record metrics.""" + start_time = time() + try: + result = func(*args, **kwargs) + metrics.REDIS_OPERATIONS_TOTAL.labels( + pod_id=self.pod_id, operation=operation_name + ).inc() + duration = time() - start_time + metrics.REDIS_OPERATIONS_DURATION.labels( + pod_id=self.pod_id, operation=operation_name + ).observe(duration) + return result + except Exception as e: + metrics.REDIS_OPERATIONS_ERRORS.labels( + pod_id=self.pod_id, operation=operation_name + ).inc() + print(f"Error in Redis operation '{operation_name}': {e}") + raise def get(self, key: str) -> dict: """ @@ -19,11 +61,11 @@ class ResourceManager: Args: key (str): The key to look up in Redis. Returns: - dict: The value associated with the key, parsed as a dictionary, + dict: The value associated with the key, parsed as a dictionary, or None if the key does not exist or the value is empty. """ - history = self.redis.get(key) + history = self._execute_redis_op("get", self.redis.get, key) return json.loads(history) if history else None def get_tag_slot(self, id: str) -> dict: @@ -48,14 +90,19 @@ class ResourceManager: None """ - self.redis.set( - f"heartbeat:ingestor:{self.pod_id}", 1, ex=self.heartbeat_ttl) + self._execute_redis_op( + "set", + self.redis.set, + f"heartbeat:ingestor:{self.pod_id}", + 1, + ex=self.heartbeat_ttl, + ) def lease_tag(self, tag_id: str) -> bool: """ Attempts to lease a tag by setting a key in Redis with a specified TTL (time-to-live). - This method uses the Redis `SET` command with the `NX` option to ensure that the key - is only set if it does not already exist. The key is set with an expiration time + This method uses the Redis `SET` command with the `NX` option to ensure that the key + is only set if it does not already exist. The key is set with an expiration time defined by `lease_ttl`. Args: tag_id (str): The unique identifier of the tag to be leased. @@ -63,8 +110,14 @@ class ResourceManager: bool: True if the lease was successfully acquired, False otherwise. """ - return self.redis.set( - f"lease:opc_tags:{tag_id}", self.pod_id, nx=True, ex=self.lease_ttl) + return self._execute_redis_op( + "set_nx", + self.redis.set, + f"lease:opc_tags:{tag_id}", + self.pod_id, + nx=True, + ex=self.lease_ttl, + ) def renew_tag_lease(self, tag_id: str) -> bool: """ @@ -78,12 +131,14 @@ class ResourceManager: bool: True if the lease was successfully renewed, False otherwise. """ - current = self.redis.get( - f"lease:opc_tags:{tag_id}") + current = self._execute_redis_op( + "get", self.redis.get, f"lease:opc_tags:{tag_id}" + ) if current == self.pod_id: - self.redis.expire(f"lease:opc_tags:{tag_id}", self.lease_ttl) + self._execute_redis_op( + "expire", self.redis.expire, f"lease:opc_tags:{tag_id}", self.lease_ttl + ) return True - return False def drop_tag_lease(self, tag_id: str) -> None: @@ -96,7 +151,7 @@ class ResourceManager: None """ - self.redis.delete(f"lease:opc_tags:{tag_id}") + self._execute_redis_op("delete", self.redis.delete, f"lease:opc_tags:{tag_id}") def get_all_ingestors(self) -> List[str]: """ @@ -107,7 +162,7 @@ class ResourceManager: list: A list of active ingestors. """ - return self.redis.keys("heartbeat:ingestor:*") + return self._execute_redis_op("keys", self.redis.keys, "heartbeat:ingestor:*") def get_all_slots(self) -> List[str]: """ @@ -118,7 +173,7 @@ class ResourceManager: int: The number of slots available. """ - return self.redis.keys("slot:opc_tags:*") + return self._execute_redis_op("keys", self.redis.keys, "slot:opc_tags:*") def get_all_leases(self) -> List[str]: """ @@ -129,4 +184,4 @@ class ResourceManager: list: A list of active leases. """ - return self.redis.keys("lease:opc_tags:*") + return self._execute_redis_op("keys", self.redis.keys, "lease:opc_tags:*") diff --git a/tests/unit/managers/__init__.py b/tests/unit/managers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/managers/test_resource_manager.py b/tests/unit/managers/test_resource_manager.py index aaa8fd2..19ca732 100644 --- a/tests/unit/managers/test_resource_manager.py +++ b/tests/unit/managers/test_resource_manager.py @@ -1,109 +1,190 @@ from unittest.mock import MagicMock, patch -from pytest import fixture +from pytest import fixture, raises from ingestor.managers.resource_manager import ResourceManager @fixture -@patch('ingestor.managers.resource_manager.Redis') +@patch("ingestor.managers.resource_manager.Redis") def resource_manager(redis): - return ResourceManager( - 'localhost', 6379, 10, 10, 'pod_id' - ) + return ResourceManager("localhost", 6379, 10, 10, "pod_id") def test_get_success(resource_manager): resource_manager.redis.get.return_value = '{"key": "value"}' - result = resource_manager.get('key') + result = resource_manager.get("key") assert result == {"key": "value"} - resource_manager.redis.get.assert_called_once_with('key') + resource_manager.redis.get.assert_called_once_with("key") def test_get_failure(resource_manager): resource_manager.redis.get.return_value = None - result = resource_manager.get('key') + result = resource_manager.get("key") assert result is None - resource_manager.redis.get.assert_called_once_with('key') + resource_manager.redis.get.assert_called_once_with("key") def test_get_tag_slot(resource_manager): resource_manager.get = MagicMock(return_value={"tag": "slot"}) - result = resource_manager.get_tag_slot('id') + result = resource_manager.get_tag_slot("id") assert result == {"tag": "slot"} - resource_manager.get.assert_called_once_with('slot:opc_tags:id') + resource_manager.get.assert_called_once_with("slot:opc_tags:id") def test_ingestor_heartbeat(resource_manager): resource_manager.redis.set.return_value = True resource_manager.ingestor_heartbeat() resource_manager.redis.set.assert_called_once_with( - 'heartbeat:ingestor:pod_id', 1, ex=10 + "heartbeat:ingestor:pod_id", 1, ex=10 ) def test_lease_tag(resource_manager): resource_manager.redis.set.return_value = True - output = resource_manager.lease_tag('tag_id') + output = resource_manager.lease_tag("tag_id") assert output is True resource_manager.redis.set.assert_called_once_with( - 'lease:opc_tags:tag_id', 'pod_id', nx=True, ex=10 + "lease:opc_tags:tag_id", "pod_id", nx=True, ex=10 ) def test_renew_tag_lease_success(resource_manager): - resource_manager.redis.get.return_value = 'pod_id' + resource_manager.redis.get.return_value = "pod_id" resource_manager.redis.expire.return_value = True - result = resource_manager.renew_tag_lease('tag_id') + result = resource_manager.renew_tag_lease("tag_id") assert result is True - resource_manager.redis.get.assert_called_once_with( - 'lease:opc_tags:tag_id' - ) - resource_manager.redis.expire.assert_called_once_with( - 'lease:opc_tags:tag_id', 10 - ) + resource_manager.redis.get.assert_called_once_with("lease:opc_tags:tag_id") + resource_manager.redis.expire.assert_called_once_with("lease:opc_tags:tag_id", 10) def test_renew_tag_lease_failure(resource_manager): - resource_manager.redis.get.return_value = 'other_pod_id' + resource_manager.redis.get.return_value = "other_pod_id" resource_manager.redis.expire.return_value = False - result = resource_manager.renew_tag_lease('tag_id') + result = resource_manager.renew_tag_lease("tag_id") assert result is False - resource_manager.redis.get.assert_called_once_with( - 'lease:opc_tags:tag_id' - ) + resource_manager.redis.get.assert_called_once_with("lease:opc_tags:tag_id") resource_manager.redis.expire.assert_not_called() def test_drop_tag_lease(resource_manager): resource_manager.redis.delete.return_value = True - resource_manager.drop_tag_lease('tag_id') - resource_manager.redis.delete.assert_called_once_with( - 'lease:opc_tags:tag_id' - ) + resource_manager.drop_tag_lease("tag_id") + resource_manager.redis.delete.assert_called_once_with("lease:opc_tags:tag_id") def test_get_all_ingestors(resource_manager): - resource_manager.redis.keys.return_value = ['ingestor1', 'ingestor2'] + resource_manager.redis.keys.return_value = ["ingestor1", "ingestor2"] result = resource_manager.get_all_ingestors() - assert result == ['ingestor1', 'ingestor2'] - resource_manager.redis.keys.assert_called_once_with( - 'heartbeat:ingestor:*' - ) + assert result == ["ingestor1", "ingestor2"] + resource_manager.redis.keys.assert_called_once_with("heartbeat:ingestor:*") def test_get_all_slots(resource_manager): - resource_manager.redis.keys.return_value = ['slot1', 'slot2'] + resource_manager.redis.keys.return_value = ["slot1", "slot2"] result = resource_manager.get_all_slots() - assert result == ['slot1', 'slot2'] - resource_manager.redis.keys.assert_called_once_with( - 'slot:opc_tags:*' - ) + assert result == ["slot1", "slot2"] + resource_manager.redis.keys.assert_called_once_with("slot:opc_tags:*") def test_get_all_leases(resource_manager): - resource_manager.redis.keys.return_value = ['lease1', 'lease2'] + resource_manager.redis.keys.return_value = ["lease1", "lease2"] result = resource_manager.get_all_leases() - assert result == ['lease1', 'lease2'] - resource_manager.redis.keys.assert_called_once_with( - 'lease:opc_tags:*' - ) + assert result == ["lease1", "lease2"] + resource_manager.redis.keys.assert_called_once_with("lease:opc_tags:*") + + +def test_init_connection_failure(monkeypatch): + # Mock Redis to raise an exception during initialization + mock_redis = MagicMock() + mock_redis.side_effect = Exception("Connection failed") + + monkeypatch.setattr("ingestor.managers.resource_manager.Redis", mock_redis) + + # Test that the exception is raised and metrics are set properly + with patch("ingestor.metrics.REDIS_CONNECTION_STATUS") as mock_metrics: + mock_status = MagicMock() + mock_metrics.labels.return_value = mock_status + + with raises(Exception, match="Connection failed"): + ResourceManager("localhost", 6379, 10, 10, "pod_id") + + mock_metrics.labels.assert_called_once_with(pod_id="pod_id") + mock_status.set.assert_called_once_with(0) + + +def test_init_ping_failure(monkeypatch): + # Mock Redis ping to raise an exception + mock_redis_instance = MagicMock() + mock_redis_instance.ping.side_effect = Exception("Ping failed") + + mock_redis_class = MagicMock(return_value=mock_redis_instance) + monkeypatch.setattr("ingestor.managers.resource_manager.Redis", mock_redis_class) + + # Test that the exception is raised and metrics are set properly + with patch("ingestor.metrics.REDIS_CONNECTION_STATUS") as mock_metrics: + mock_status = MagicMock() + mock_metrics.labels.return_value = mock_status + + with raises(Exception, match="Ping failed"): + ResourceManager("localhost", 6379, 10, 10, "pod_id") + + mock_metrics.labels.assert_called_once_with(pod_id="pod_id") + mock_status.set.assert_called_once_with(0) + + +def test_execute_redis_op_success(resource_manager): + # Mock the Redis operation and time function + mock_func = MagicMock(return_value="test_result") + + with patch("ingestor.managers.resource_manager.time", side_effect=[100, 100.5]): + with patch("ingestor.metrics.REDIS_OPERATIONS_TOTAL") as mock_total: + with patch("ingestor.metrics.REDIS_OPERATIONS_DURATION") as mock_duration: + mock_total_labels = MagicMock() + mock_duration_labels = MagicMock() + mock_total.labels.return_value = mock_total_labels + mock_duration.labels.return_value = mock_duration_labels + + # Execute the operation + result = resource_manager._execute_redis_op( + "test_op", mock_func, "arg1", kwarg1="value1" + ) + + # Verify the result and metrics + assert result == "test_result" + mock_func.assert_called_once_with("arg1", kwarg1="value1") + + mock_total.labels.assert_called_once_with( + pod_id="pod_id", operation="test_op" + ) + mock_total_labels.inc.assert_called_once() + + mock_duration.labels.assert_called_once_with( + pod_id="pod_id", operation="test_op" + ) + mock_duration_labels.observe.assert_called_once_with( + 0.5 + ) + + +def test_execute_redis_op_exception(resource_manager): + # Mock the Redis operation to raise an exception + mock_func = MagicMock(side_effect=Exception("Operation failed")) + + with patch("ingestor.managers.resource_manager.time", return_value=100): + with patch("ingestor.metrics.REDIS_OPERATIONS_ERRORS") as mock_errors: + with patch("builtins.print") as mock_print: + mock_errors_labels = MagicMock() + mock_errors.labels.return_value = mock_errors_labels + + # Execute the operation and expect an exception + with raises(Exception, match="Operation failed"): + resource_manager._execute_redis_op("test_op", mock_func, "arg1") + + # Verify metrics and error handling + mock_errors.labels.assert_called_once_with( + pod_id="pod_id", operation="test_op" + ) + mock_errors_labels.inc.assert_called_once() + mock_print.assert_called_once_with( + "Error in Redis operation 'test_op': Operation failed" + ) From c49f04ded67fe4679af917f5cdfde3f7ee0601fe Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Thu, 29 May 2025 16:52:10 -0300 Subject: [PATCH 21/28] SIENTIAPDE-1083: add prometheus metrics to opc_manager. --- ingestor/managers/ingestor_manager.py | 3 +- ingestor/managers/opc_manager.py | 53 ++- ingestor/metrics.py | 20 - tests/unit/managers/test_ingestor_manager.py | 4 +- tests/unit/managers/test_opc_manager.py | 433 ++++++++++++++----- tests/unit/test_metrics.py | 44 -- 6 files changed, 354 insertions(+), 203 deletions(-) diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index 433fcef..85d5602 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -29,6 +29,7 @@ class IngestorManager(): self.opc_servers = {} self.notification_handler = notification_handler + self.pod_id = pod_id def initialize_opc_from_config(self, server_config: dict, data_manager: DataManager, logger: Logger) -> OpcManager | None: @@ -56,7 +57,7 @@ class IngestorManager(): manager = OpcManager( server_config['name'], server_config['url'], data_manager, logger, server_config['server_uri'], - self.notification_handler, server_config.get('cert_path'), + self.notification_handler, self.pod_id, server_config.get('cert_path'), server_config.get('private_key_path'), server_config.get('server_cert_path') ) diff --git a/ingestor/managers/opc_manager.py b/ingestor/managers/opc_manager.py index e218956..35d4a8b 100644 --- a/ingestor/managers/opc_manager.py +++ b/ingestor/managers/opc_manager.py @@ -7,11 +7,12 @@ from asyncua.sync import Client from sientia_do.notifications.models import NotificationLevel from sientia_do.notifications.handlers import NotificationHandler from ingestor.managers.data_manager import DataManager +import ingestor.metrics as metrics class OpcManager(): def __init__(self, name: str, url: str, data_manager: DataManager, - logger: Logger, server_uri: str, notification_handler: NotificationHandler, + logger: Logger, server_uri: str, notification_handler: NotificationHandler, pod_id: str, cert_path: str = None, private_key_path: str = None, server_cert_path: str = None): self.url = url self.name = name @@ -26,8 +27,10 @@ class OpcManager(): self.nodes = {} self.subscriptions = {} self.data_manager = data_manager - self.notification_handler = notification_handler + self.pod_id = pod_id + metrics.OPC_CONNECTION_STATUS.labels(pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0) + metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(0) def __str__(self): return f"OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n" \ @@ -82,11 +85,20 @@ class OpcManager(): Exception: If the connection to the OPC server fails. """ - self.client = Client(self.url) - if self.cert_path: - self.set_security() - self.logger.info('Starting connection...') - self.client.connect() + metrics.OPC_CONNECTIONS_TOTAL.labels(pod_id=self.pod_id, server_name=self.name).inc() + try: + self.client = Client(self.url) + if self.cert_path: + self.set_security() + self.logger.info(f'Starting connection to {self.name}...') + self.client.connect() + metrics.OPC_CONNECTION_STATUS.labels(pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(1) + self.logger.info(f'Connection to {self.name} successful.') + except Exception as e: + metrics.OPC_CONNECTION_STATUS.labels(pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0) + metrics.OPC_CONNECTIONS_FAILED.labels(pod_id=self.pod_id, server_name=self.name).inc() + self.logger.error(f"Failed to connect to {self.name}: {e}") + raise def create_subscription(self, name: str, period: int = 500): """ @@ -105,11 +117,14 @@ class OpcManager(): if not self.client: raise ValueError("Client not connected. Call connect first.") - - p = period if period != None else 500 - self.subscriptions[name] = self.client.create_subscription( - p, self) - self.logger.info('Subscription created.') + try: + p = period if period is not None else 500 + self.subscriptions[name] = self.client.create_subscription(p, self) + self.logger.info(f'Subscription {name} created on {self.name}.') + metrics.OPC_SUBSCRIPTIONS_CREATED.labels(pod_id=self.pod_id, server_name=self.name, slot_name=name).inc() + except Exception as e: + self.logger.error(f"Failed to create subscription {name} on {self.name}: {e}") + raise def subscribe(self, subscription: str, nodes: dict, collect_period: int): """ @@ -129,14 +144,13 @@ class OpcManager(): """ if not self.subscriptions.get(subscription): - raise ValueError( - "Subscription not created. Call create_subscription first.") + raise ValueError("Subscription not created. Call create_subscription first.") - self.logger.info(f"Subscribing to {subscription}...") + self.logger.info(f"Subscribing to {subscription} on {self.name}...") self.logger.info(f"Subscribing to nodes: {nodes}") - self.addr_nodes = [self.client.get_node( - n) for n in nodes if n not in self.nodes] + self.addr_nodes = [self.client.get_node(n) for n in nodes if n not in self.nodes] self.nodes.update(nodes) + metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(len(self.nodes)) self.collect_period = collect_period for node, config in self.nodes.items(): @@ -202,6 +216,8 @@ class OpcManager(): finally: del self.client self.client = None + metrics.OPC_CONNECTION_STATUS.labels(pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0) + metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(0) self.logger.warning("Disconnected from OPC UA server.") def datachange_notification(self, node, _val, data): @@ -236,6 +252,7 @@ class OpcManager(): self.nodes[tag]['cycle_rule']['cycle_count'] = 0 self.non_receive_count = 0 + metrics.OPC_CYCLES_WITHOUT_DATA.labels(pod_id=self.pod_id, server_name=self.name).set(0) data = { 'tag': tag, @@ -278,6 +295,7 @@ class OpcManager(): """ self.non_receive_count += 1 + metrics.OPC_CYCLES_WITHOUT_DATA.labels(pod_id=self.pod_id, server_name=self.name).set(self.non_receive_count) if self.non_receive_count >= 5: self.notification_handler.build_and_send_notification( notification_id=f'OPC_LISTENNING_STOPPED__{self.name}', @@ -287,6 +305,7 @@ class OpcManager(): level=NotificationLevel.ERROR ) if self.non_receive_count >= 15: + metrics.OPC_RECONNECTIONS_TOTAL.labels(pod_id=self.pod_id, server_name=self.name).inc() self.notification_handler.build_and_send_notification( notification_id=f'OPC_CONNECTION_RETRY__{self.name}', message=f'Retrying to connect to server {self.name}', diff --git a/ingestor/metrics.py b/ingestor/metrics.py index bb0976d..b2ddc42 100644 --- a/ingestor/metrics.py +++ b/ingestor/metrics.py @@ -1,11 +1,7 @@ from prometheus_client import Counter, Gauge, Histogram -# It's useful to have a common set of labels, like the pod_id. -# We'll add 'pod_id' to many metrics to distinguish instances. POD_ID_LABEL = ["pod_id"] SERVER_LABELS = ["pod_id", "server_name", "server_url"] -SLOT_LABELS = ["pod_id", "slot_id"] -TAG_LABELS = ["pod_id", "server_name", "tag_id", "tag_name"] KAFKA_LABELS = ["pod_id", "topic"] REDIS_LABELS = ["pod_id", "operation"] NOTIFICATION_LABELS = ["pod_id", "level", "block"] @@ -37,17 +33,14 @@ APP_UP = Gauge( ACTIVE_INGESTORS = Gauge( "ingestor_active_total", "Number of active ingestors reported by Redis", - # No pod_id here, as it's a global view from Redis ) SLOTS_TOTAL = Gauge( "ingestor_slots_total", "Total number of slots configured in Redis", - # No pod_id here, as it's a global view from Redis ) LEASES_TOTAL = Gauge( "ingestor_leases_total", "Total number of leases (allocated slots) in Redis", - # No pod_id here, as it's a global view from Redis ) SLOTS_MANAGED = Gauge( "ingestor_slots_managed_current", @@ -101,19 +94,6 @@ OPC_TAGS_SUBSCRIBED = Gauge( "Current number of OPC tags subscribed on a server", ["pod_id", "server_name"], ) -OPC_DATACHANGE_NOTIFICATIONS = Counter( - "opc_datachange_notifications_total", - "Total data change notifications received (tag readings)", - TAG_LABELS, -) -OPC_TAG_LAST_VALUE = Gauge( - "opc_tag_last_value", "The last value read from an OPC tag", TAG_LABELS -) -OPC_TAG_LAST_READ_TIMESTAMP = Gauge( - "opc_tag_last_read_timestamp_seconds", - "Timestamp of the last value read from an OPC tag", - TAG_LABELS, -) OPC_CYCLES_WITHOUT_DATA = Gauge( "opc_cycles_without_data", "Current number of cycles without receiving data from a server", diff --git a/tests/unit/managers/test_ingestor_manager.py b/tests/unit/managers/test_ingestor_manager.py index fa3f7be..f1bcbf3 100644 --- a/tests/unit/managers/test_ingestor_manager.py +++ b/tests/unit/managers/test_ingestor_manager.py @@ -60,7 +60,8 @@ def test_initialize_opc_from_config(opc_manager, ingestor_manager): 'server_uri': 'http://opcua-server.simulator', 'cert_path': '/path/to/cert', 'private_key_path': '/path/to/private_key', - 'server_cert_path': '/path/to/server_cert' + 'server_cert_path': '/path/to/server_cert', + 'pod_id': 'test_pod' } opc_manager.return_value = MagicMock() @@ -70,6 +71,7 @@ def test_initialize_opc_from_config(opc_manager, ingestor_manager): opc_manager.assert_called_once_with( server_config['name'], server_config['url'], ingestor_manager.data_manager, ingestor_manager.logger, server_config['server_uri'], ingestor_manager.notification_handler, + server_config['pod_id'], server_config['cert_path'], server_config['private_key_path'], server_config['server_cert_path'] ) diff --git a/tests/unit/managers/test_opc_manager.py b/tests/unit/managers/test_opc_manager.py index a57ef46..90c25b9 100644 --- a/tests/unit/managers/test_opc_manager.py +++ b/tests/unit/managers/test_opc_manager.py @@ -1,6 +1,7 @@ import json from datetime import datetime from unittest.mock import MagicMock, patch +from prometheus_client import Gauge from pytest import fixture from asyncua.crypto.security_policies import SecurityPolicyBasic256 @@ -9,57 +10,65 @@ from ingestor.managers.opc_manager import OpcManager from sientia_do.notifications.models import NotificationLevel tags = { - 'ns=3;i=1001': { - 'aggregation_function': 'LTS', - 'frequency': 1000, - 'max_value': 100, - 'min_value': 0, - 'tag_name': 'Counter' + "ns=3;i=1001": { + "aggregation_function": "LTS", + "frequency": 1000, + "max_value": 100, + "min_value": 0, + "tag_name": "Counter", }, - 'ns=3;i=1003': { - 'aggregation_function': 'AVG', - 'frequency': 1000, - 'max_value': 100, - 'min_value': 0, - 'tag_name': 'Random' + "ns=3;i=1003": { + "aggregation_function": "AVG", + "frequency": 1000, + "max_value": 100, + "min_value": 0, + "tag_name": "Random", + }, + "ns=3;i=1004": { + "aggregation_function": "MDN", + "frequency": 1000, + "max_value": 100, + "min_value": 0, + "tag_name": "Sawtooth", }, - 'ns=3;i=1004': { - 'aggregation_function': 'MDN', - 'frequency': 1000, - 'max_value': 100, - 'min_value': 0, - 'tag_name': 'Sawtooth' - } } @fixture def raw_opc_manager(): return OpcManager( - 'TestConnector', 'opc.tcp://localhost:4840', MagicMock(), - MagicMock(), 'opc.tcp://localhost:4840', MagicMock() + "TestConnector", + "opc.tcp://localhost:4840", + MagicMock(), + MagicMock(), + "opc.tcp://localhost:4840", + MagicMock(), + "localhost", ) @fixture def opc_manager(raw_opc_manager): raw_opc_manager.client = MagicMock() - raw_opc_manager.cert_path = 'cert.pem' - raw_opc_manager.private_key_path = 'private_key.pem' - raw_opc_manager.server_cert_path = 'server_cert.pem' + raw_opc_manager.cert_path = "cert.pem" + raw_opc_manager.private_key_path = "private_key.pem" + raw_opc_manager.server_cert_path = "server_cert.pem" return raw_opc_manager @fixture def opc_manager_subscribed(opc_manager): - opc_manager.subscriptions['sub1'] = MagicMock() + opc_manager.subscriptions["sub1"] = MagicMock() return opc_manager def test___str__(opc_manager): - assert str(opc_manager) == 'OpcManager(name=TestConnector, url=opc.tcp://localhost:4840, server_uri=opc.tcp://localhost:4840)\nnodes={}, subscriptions={}' + assert ( + str(opc_manager) + == "OpcManager(name=TestConnector, url=opc.tcp://localhost:4840, server_uri=opc.tcp://localhost:4840)\nnodes={}, subscriptions={}" + ) def test_set_security_success(opc_manager): @@ -71,7 +80,7 @@ def test_set_security_success(opc_manager): SecurityPolicyBasic256, certificate=opc_manager.cert_path, private_key=opc_manager.private_key_path, - server_certificate=opc_manager.server_cert_path + server_certificate=opc_manager.server_cert_path, ) assert opc_manager.client.secure_channel_timeout == 10000000 @@ -85,16 +94,19 @@ def test_set_security_no_cert(opc_manager): try: opc_manager.set_security() except ValueError as e: - assert str( - e) == "Certificate and private key paths must be provided for secure connection." + assert ( + str(e) + == "Certificate and private key paths must be provided for secure connection." + ) else: assert False, "ValueError not raised" assert opc_manager.client.set_security.call_count == 0 -@patch('ingestor.managers.opc_manager.Client') -def test_connect_no_security(client, raw_opc_manager): +@patch("ingestor.managers.opc_manager.metrics") +@patch("ingestor.managers.opc_manager.Client") +def test_connect_no_security(client, mock_metrics, raw_opc_manager): raw_opc_manager.set_security = MagicMock() raw_opc_manager.connect() @@ -102,13 +114,25 @@ def test_connect_no_security(client, raw_opc_manager): client.assert_called_once_with(raw_opc_manager.url) raw_opc_manager.client.connect.assert_called_once() raw_opc_manager.set_security.assert_not_called() + mock_metrics.OPC_CONNECTIONS_TOTAL.labels.assert_called_once_with( + pod_id=raw_opc_manager.pod_id, + server_name=raw_opc_manager.name + ) + mock_metrics.OPC_CONNECTIONS_TOTAL.labels.return_value.inc.assert_called_once() + mock_metrics.OPC_CONNECTION_STATUS.labels.assert_called_once_with( + pod_id=raw_opc_manager.pod_id, + server_name=raw_opc_manager.name, + server_url=raw_opc_manager.url + ) + mock_metrics.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(1) + mock_metrics.OPC_CONNECTIONS_FAILED.labels.assert_not_called() -@patch('ingestor.managers.opc_manager.Client') +@patch("ingestor.managers.opc_manager.Client") def test_connect_with_security(client, raw_opc_manager): - raw_opc_manager.cert_path = 'cert.pem' - raw_opc_manager.private_key_path = 'private_key.pem' - raw_opc_manager.server_cert_path = 'server_cert.pem' + raw_opc_manager.cert_path = "cert.pem" + raw_opc_manager.private_key_path = "private_key.pem" + raw_opc_manager.server_cert_path = "server_cert.pem" raw_opc_manager.set_security = MagicMock() raw_opc_manager.connect() @@ -118,9 +142,48 @@ def test_connect_with_security(client, raw_opc_manager): raw_opc_manager.set_security.assert_called_once() +@patch("ingestor.managers.opc_manager.Client") +@patch("ingestor.managers.opc_manager.metrics") +def test_connect_exception_handling_and_metrics( + mock_metrics_module, mock_opc_client_class, raw_opc_manager +): + mock_client_instance = mock_opc_client_class.return_value + simulated_error_message = "Erro de conexão simulado" + mock_client_instance.connect.side_effect = Exception(simulated_error_message) + + opc_manager_instance = raw_opc_manager + opc_manager_instance.cert_path = None + + with pytest.raises(Exception, match=simulated_error_message): + opc_manager_instance.connect() + + mock_metrics_module.OPC_CONNECTIONS_TOTAL.labels.assert_called_once_with( + pod_id=opc_manager_instance.pod_id, server_name=opc_manager_instance.name + ) + mock_metrics_module.OPC_CONNECTIONS_TOTAL.labels.return_value.inc.assert_called_once() + + mock_metrics_module.OPC_CONNECTION_STATUS.labels.assert_called_once_with( + pod_id=opc_manager_instance.pod_id, + server_name=opc_manager_instance.name, + server_url=opc_manager_instance.url, + ) + mock_metrics_module.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with( + 0 + ) + + mock_metrics_module.OPC_CONNECTIONS_FAILED.labels.assert_called_once_with( + pod_id=opc_manager_instance.pod_id, server_name=opc_manager_instance.name + ) + mock_metrics_module.OPC_CONNECTIONS_FAILED.labels.return_value.inc.assert_called_once() + + opc_manager_instance.logger.error.assert_called_once_with( + f"Failed to connect to {opc_manager_instance.name}: {simulated_error_message}" + ) + + def test_create_subscription_no_client(raw_opc_manager): try: - raw_opc_manager.create_subscription('sub1') + raw_opc_manager.create_subscription("sub1") except ValueError as e: assert str(e) == "Client not connected. Call connect first." else: @@ -128,55 +191,95 @@ def test_create_subscription_no_client(raw_opc_manager): def test_create_subscription_success_has_period(opc_manager): - opc_manager.create_subscription('sub1', 1000) + opc_manager.create_subscription("sub1", 1000) - opc_manager.client.create_subscription.assert_called_once_with( - 1000, opc_manager) - assert opc_manager.subscriptions['sub1'] is not None + opc_manager.client.create_subscription.assert_called_once_with(1000, opc_manager) + assert opc_manager.subscriptions["sub1"] is not None def test_create_subscription_success_no_period(opc_manager): - opc_manager.create_subscription('sub1', None) + opc_manager.create_subscription("sub1", None) - opc_manager.client.create_subscription.assert_called_once_with( - 500, opc_manager) - assert opc_manager.subscriptions['sub1'] is not None + opc_manager.client.create_subscription.assert_called_once_with(500, opc_manager) + assert opc_manager.subscriptions["sub1"] is not None -def test_subscribe_no_subscription(opc_manager): +@patch("ingestor.managers.opc_manager.metrics") +def test_create_subscription_with_metrics(metrics, opc_manager): + opc_manager.create_subscription("sub1", 1000) + + metrics.OPC_SUBSCRIPTIONS_CREATED.labels.assert_called_once_with( + pod_id=opc_manager.pod_id, server_name=opc_manager.name, slot_name="sub1" + ) + metrics.OPC_SUBSCRIPTIONS_CREATED.labels.return_value.inc.assert_called_once() + + +@patch("ingestor.managers.opc_manager.metrics") +def test_create_subscription_exception_during_client_call( + mock_metrics_module, raw_opc_manager +): + opc_manager_instance = raw_opc_manager + opc_manager_instance.client = MagicMock() + + subscription_name = "test_sub_client_error" + simulated_period = 750 + simulated_error_message = "Falha ao criar subscrição no cliente OPC" + + opc_manager_instance.client.create_subscription.side_effect = Exception( + simulated_error_message + ) + + with pytest.raises(Exception, match=simulated_error_message): + opc_manager_instance.create_subscription( + subscription_name, period=simulated_period + ) + + opc_manager_instance.client.create_subscription.assert_called_once_with( + simulated_period, opc_manager_instance + ) + + opc_manager_instance.logger.error.assert_called_once_with( + f"Failed to create subscription {subscription_name} on {opc_manager_instance.name}: {simulated_error_message}" + ) + + mock_metrics_module.OPC_SUBSCRIPTIONS_CREATED.labels.assert_not_called() + + +@patch("ingestor.managers.opc_manager.metrics") +def test_subscribe_no_subscription(metrics, opc_manager): try: - opc_manager.subscribe('sub1', tags, 1000) + opc_manager.subscribe("sub1", tags, 1000) except ValueError as e: - assert str( - e) == "Subscription not created. Call create_subscription first." + assert str(e) == "Subscription not created. Call create_subscription first." else: assert False, "ValueError not raised" + metrics.OPC_TAGS_SUBSCRIBED.labels.assert_not_called() def test_subscribe_success(opc_manager_subscribed): - opc_manager_subscribed.nodes = { - 'ns=3;i=1001': 'data' - } + opc_manager_subscribed.nodes = {"ns=3;i=1001": "data"} - opc_manager_subscribed.subscribe('sub1', tags, 1000) + opc_manager_subscribed.subscribe("sub1", tags, 1000) assert opc_manager_subscribed.nodes == tags assert opc_manager_subscribed.addr_nodes == [ - opc_manager_subscribed.client.get_node(n) for n in tags if n != 'ns=3;i=1001'] + opc_manager_subscribed.client.get_node(n) for n in tags if n != "ns=3;i=1001" + ] def test_unsubscribe_no_subscription(opc_manager): - opc_manager.unsubscribe('sub1') + opc_manager.unsubscribe("sub1") opc_manager.logger.warning.assert_called_once_with( - "Subscription 'sub1' not found. Cannot unsubscribe.") - assert opc_manager.subscriptions.get('sub1') is None + "Subscription 'sub1' not found. Cannot unsubscribe." + ) + assert opc_manager.subscriptions.get("sub1") is None def test_unsubscribe_success(opc_manager_subscribed): - opc_manager_subscribed.unsubscribe('sub1') + opc_manager_subscribed.unsubscribe("sub1") - opc_manager_subscribed.subscriptions.get('sub1') is None + opc_manager_subscribed.subscriptions.get("sub1") is None def test_disconnect_success(opc_manager_subscribed): @@ -184,85 +287,123 @@ def test_disconnect_success(opc_manager_subscribed): opc_manager_subscribed.disconnect() - opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once() + opc_manager_subscribed.subscriptions["sub1"].delete.assert_called_once() assert opc_manager_subscribed.client is None def test_disconnect_error_unsubscribe(opc_manager_subscribed): opc_manager_subscribed.client = MagicMock() - opc_manager_subscribed.subscriptions['sub1'] = MagicMock( + opc_manager_subscribed.subscriptions["sub1"] = MagicMock( delete=MagicMock(side_effect=Exception("Test error")) ) opc_manager_subscribed.disconnect() - opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once() + opc_manager_subscribed.subscriptions["sub1"].delete.assert_called_once() opc_manager_subscribed.client = None opc_manager_subscribed.logger.error.assert_called_once_with( - "Failed to clean up subscription: Test error") + "Failed to clean up subscription: Test error" + ) def test_disconnect_error(opc_manager_subscribed): opc_manager_subscribed.client = MagicMock() opc_manager_subscribed.client.disconnect = MagicMock( - side_effect=Exception("Test error")) + side_effect=Exception("Test error") + ) opc_manager_subscribed.disconnect() - opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once() + opc_manager_subscribed.subscriptions["sub1"].delete.assert_called_once() opc_manager_subscribed.client = None opc_manager_subscribed.logger.error.assert_called_once_with( - "Failed to disconnect from OPC UA server: Test error") + "Failed to disconnect from OPC UA server: Test error" + ) -def test_datachange_notification(opc_manager_subscribed): +@patch('ingestor.managers.opc_manager.metrics') +def test_disconnect_metrics_on_successful_path(mock_metrics_module, raw_opc_manager): + mock_metrics_module.OPC_CONNECTION_STATUS.reset_mock() + mock_metrics_module.OPC_TAGS_SUBSCRIBED.reset_mock() + + raw_opc_manager.client = MagicMock() + mock_sub1 = MagicMock() + mock_sub2 = MagicMock() + raw_opc_manager.subscriptions = {"sub1": mock_sub1, "sub2": mock_sub2} + + raw_opc_manager.disconnect() + + mock_metrics_module.OPC_CONNECTION_STATUS.labels.assert_called_once_with( + pod_id=raw_opc_manager.pod_id, + server_name=raw_opc_manager.name, + server_url=raw_opc_manager.url + ) + mock_metrics_module.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(0) + + mock_metrics_module.OPC_TAGS_SUBSCRIBED.labels.assert_called_once_with( + pod_id=raw_opc_manager.pod_id, + server_name=raw_opc_manager.name + ) + mock_metrics_module.OPC_TAGS_SUBSCRIBED.labels.return_value.set.assert_called_once_with(0) + + +@patch('ingestor.managers.opc_manager.metrics') +def test_datachange_notification(metrics, opc_manager_subscribed): data = MagicMock( monitored_item=MagicMock( Value=MagicMock( Value=MagicMock(Value=42), SourceTimestamp=datetime.strptime( - '2021-01-01T00:00:00', '%Y-%m-%dT%H:%M:%S') - ))) + "2021-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S" + ), + ) + ) + ) opc_manager_subscribed.nodes = { - 'ns=3;i=1001': { - 'tag_name': 'Counter', - 'cycle_rule': { - 'cycle_increment': 1.0, - 'cycle_count': 2 - }, - 'topics': ['topic1', 'topic2'] + "ns=3;i=1001": { + "tag_name": "Counter", + "cycle_rule": {"cycle_increment": 1.0, "cycle_count": 2}, + "topics": ["topic1", "topic2"], } } - opc_manager_subscribed.datachange_notification( - 'ns=3;i=1001', None, data) + metrics.OPC_CYCLES_WITHOUT_DATA.reset_mock() + + opc_manager_subscribed.datachange_notification("ns=3;i=1001", None, data) opc_manager_subscribed.data_manager.publish.assert_any_call( - 'topic1', { - 'tag': 'ns=3;i=1001', - 'name': 'Counter', - 'timestamp': '2021-01-01 00:00:00', - 'value': 42 - }) + "topic1", + { + "tag": "ns=3;i=1001", + "name": "Counter", + "timestamp": "2021-01-01 00:00:00", + "value": 42, + }, + ) opc_manager_subscribed.data_manager.publish.assert_any_call( - 'topic2', { - 'tag': 'ns=3;i=1001', - 'name': 'Counter', - 'timestamp': '2021-01-01 00:00:00', - 'value': 42 - }) - assert opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == 0 + "topic2", + { + "tag": "ns=3;i=1001", + "name": "Counter", + "timestamp": "2021-01-01 00:00:00", + "value": 42, + }, + ) + assert opc_manager_subscribed.nodes["ns=3;i=1001"]["cycle_rule"]["cycle_count"] == 0 + + metrics.OPC_CYCLES_WITHOUT_DATA.labels.assert_called_once_with( + pod_id=opc_manager_subscribed.pod_id, + server_name=opc_manager_subscribed.name + ) + metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with(0) def test_check_cycles_no_notification(opc_manager): # Setup: node with cycle_count just below threshold opc_manager.nodes = { - 'ns=3;i=1001': { - 'tag_name': 'Counter', - 'cycle_rule': { - 'cycle_increment': 1.0, - 'cycle_count': 3.0 - } + "ns=3;i=1001": { + "tag_name": "Counter", + "cycle_rule": {"cycle_increment": 1.0, "cycle_count": 3.0}, } } opc_manager.notification_handler.build_and_send_notification = MagicMock() @@ -270,20 +411,18 @@ def test_check_cycles_no_notification(opc_manager): opc_manager.check_cycles() # After one increment, cycle_count = 4.0, still below threshold - assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx( - 4.0) + assert opc_manager.nodes["ns=3;i=1001"]["cycle_rule"][ + "cycle_count" + ] == pytest.approx(4.0) opc_manager.notification_handler.build_and_send_notification.assert_not_called() def test_check_cycles_triggers_notification(opc_manager): # Setup: node with cycle_count just below threshold, increment will cross threshold opc_manager.nodes = { - 'ns=3;i=1001': { - 'tag_name': 'Counter', - 'cycle_rule': { - 'cycle_increment': 2.5, - 'cycle_count': 3.0 - } + "ns=3;i=1001": { + "tag_name": "Counter", + "cycle_rule": {"cycle_increment": 2.5, "cycle_count": 3.0}, } } opc_manager.notification_handler.build_and_send_notification = MagicMock() @@ -291,19 +430,23 @@ def test_check_cycles_triggers_notification(opc_manager): opc_manager.check_cycles() # After increment, cycle_count = 5.5, should trigger notification - assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx( - 5.5) + assert opc_manager.nodes["ns=3;i=1001"]["cycle_rule"][ + "cycle_count" + ] == pytest.approx(5.5) opc_manager.notification_handler.build_and_send_notification.assert_called_once_with( - notification_id='TAG_ns=3;i=1001:Counter_LISTENNING_STOPPED', - message='5.5 cycles without receive from ns=3;i=1001:Counter', + notification_id="TAG_ns=3;i=1001:Counter_LISTENNING_STOPPED", + message="5.5 cycles without receive from ns=3;i=1001:Counter", block="opc_manager", - level=NotificationLevel.WARNING + level=NotificationLevel.WARNING, ) -def test_check_opc_listenning_no_notification(opc_manager): +@patch('ingestor.managers.opc_manager.metrics') +def test_check_opc_listenning_no_notification(metrics, opc_manager): opc_manager.non_receive_count = 3 opc_manager.notification_handler.build_and_send_notification = MagicMock() + metrics.OPC_CYCLES_WITHOUT_DATA.reset_mock() + metrics.OPC_RECONNECTIONS_TOTAL.reset_mock() result = opc_manager.check_opc_listenning() @@ -311,6 +454,13 @@ def test_check_opc_listenning_no_notification(opc_manager): opc_manager.notification_handler.build_and_send_notification.assert_not_called() assert result is False + metrics.OPC_CYCLES_WITHOUT_DATA.labels.assert_called_once_with( + pod_id=opc_manager.pod_id, + server_name=opc_manager.name + ) + metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with(opc_manager.non_receive_count) + metrics.OPC_RECONNECTIONS_TOTAL.labels.assert_not_called() + def test_check_opc_listenning_warning_notification(opc_manager): opc_manager.non_receive_count = 4 @@ -320,17 +470,20 @@ def test_check_opc_listenning_warning_notification(opc_manager): assert opc_manager.non_receive_count == 5 opc_manager.notification_handler.build_and_send_notification.assert_called_once_with( - notification_id=f'OPC_LISTENNING_STOPPED__{opc_manager.name}', - message=f'5 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}', + notification_id=f"OPC_LISTENNING_STOPPED__{opc_manager.name}", + message=f"5 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}", block="opc_manager", - level=NotificationLevel.ERROR + level=NotificationLevel.ERROR, ) assert result is False -def test_check_opc_listenning_error_notification_and_retry(opc_manager): +@patch('ingestor.managers.opc_manager.metrics') +def test_check_opc_listenning_error_notification_and_retry(metrics, opc_manager): opc_manager.non_receive_count = 14 opc_manager.notification_handler.build_and_send_notification = MagicMock() + metrics.OPC_CYCLES_WITHOUT_DATA.reset_mock() + metrics.OPC_RECONNECTIONS_TOTAL.reset_mock() result = opc_manager.check_opc_listenning() @@ -340,16 +493,56 @@ def test_check_opc_listenning_error_notification_and_retry(opc_manager): calls = opc_manager.notification_handler.build_and_send_notification.call_args_list # First call: 5 cycles warning assert calls[0].kwargs == dict( - notification_id=f'OPC_LISTENNING_STOPPED__{opc_manager.name}', - message=f'15 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}', + notification_id=f"OPC_LISTENNING_STOPPED__{opc_manager.name}", + message=f"15 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}", block="opc_manager", - level=NotificationLevel.ERROR + level=NotificationLevel.ERROR, ) # Second call: 15 cycles retry assert calls[1].kwargs == dict( - notification_id=f'OPC_CONNECTION_RETRY__{opc_manager.name}', - message=f'Retrying to connect to server {opc_manager.name}', + notification_id=f"OPC_CONNECTION_RETRY__{opc_manager.name}", + message=f"Retrying to connect to server {opc_manager.name}", block="opc_manager", - level=NotificationLevel.ERROR + level=NotificationLevel.ERROR, ) assert result is True + + metrics.OPC_CYCLES_WITHOUT_DATA.labels.assert_called_once_with( + pod_id=opc_manager.pod_id, + server_name=opc_manager.name + ) + metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with(opc_manager.non_receive_count) + + metrics.OPC_RECONNECTIONS_TOTAL.labels.assert_called_once_with( + pod_id=opc_manager.pod_id, + server_name=opc_manager.name + ) + metrics.OPC_RECONNECTIONS_TOTAL.labels.return_value.inc.assert_called_once() + + +@patch("ingestor.managers.opc_manager.metrics") +def test_init_metrics_calls_correct_metric_methods(mock_metrics): + opc_manager = OpcManager( + name="TestInitConnector", + url="opc.tcp://init.test:4840", + data_manager=MagicMock(), + logger=MagicMock(), + server_uri="opc.tcp://init.test:4840/uri", + notification_handler=MagicMock(), + pod_id="init_pod_localhost", + ) + + mock_metrics.OPC_CONNECTION_STATUS.labels.assert_called_once_with( + pod_id=opc_manager.pod_id, + server_name=opc_manager.name, + server_url=opc_manager.url, + ) + + mock_metrics.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(0) + + mock_metrics.OPC_TAGS_SUBSCRIBED.labels.assert_called_once_with( + pod_id=opc_manager.pod_id, + server_name=opc_manager.name + ) + + mock_metrics.OPC_TAGS_SUBSCRIBED.labels.return_value.set.assert_called_once_with(0) diff --git a/tests/unit/test_metrics.py b/tests/unit/test_metrics.py index 8d9ad7c..8750d42 100644 --- a/tests/unit/test_metrics.py +++ b/tests/unit/test_metrics.py @@ -165,50 +165,6 @@ def test_opc_tags_subscribed(): assert set(metrics.OPC_TAGS_SUBSCRIBED._labelnames) == {"pod_id", "server_name"} -def test_opc_datachange_notifications(): - """Verify the definition of OPC_DATACHANGE_NOTIFICATIONS.""" - assert metrics.OPC_DATACHANGE_NOTIFICATIONS is not None - assert isinstance(metrics.OPC_DATACHANGE_NOTIFICATIONS, Counter) - assert ( - metrics.OPC_DATACHANGE_NOTIFICATIONS._name == "opc_datachange_notifications" - ) # REMOVED _total - assert set(metrics.OPC_DATACHANGE_NOTIFICATIONS._labelnames) == { - "pod_id", - "server_name", - "tag_id", - "tag_name", - } - - -def test_opc_tag_last_value(): - """Verify the definition of OPC_TAG_LAST_VALUE.""" - assert metrics.OPC_TAG_LAST_VALUE is not None - assert isinstance(metrics.OPC_TAG_LAST_VALUE, Gauge) - assert metrics.OPC_TAG_LAST_VALUE._name == "opc_tag_last_value" - assert set(metrics.OPC_TAG_LAST_VALUE._labelnames) == { - "pod_id", - "server_name", - "tag_id", - "tag_name", - } - - -def test_opc_tag_last_read_timestamp(): - """Verify the definition of OPC_TAG_LAST_READ_TIMESTAMP.""" - assert metrics.OPC_TAG_LAST_READ_TIMESTAMP is not None - assert isinstance(metrics.OPC_TAG_LAST_READ_TIMESTAMP, Gauge) - assert ( - metrics.OPC_TAG_LAST_READ_TIMESTAMP._name - == "opc_tag_last_read_timestamp_seconds" - ) - assert set(metrics.OPC_TAG_LAST_READ_TIMESTAMP._labelnames) == { - "pod_id", - "server_name", - "tag_id", - "tag_name", - } - - def test_opc_cycles_without_data(): """Verify the definition of OPC_CYCLES_WITHOUT_DATA.""" assert metrics.OPC_CYCLES_WITHOUT_DATA is not None From 0fe2343891e8e58bfbb27f859b0262282f1192a7 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Thu, 29 May 2025 17:47:53 -0300 Subject: [PATCH 22/28] SIENTIAPDE-1083: add prometheus metrics to ingestor_manager. --- ingestor/managers/ingestor_manager.py | 12 +++++++ tests/unit/managers/test_ingestor_manager.py | 34 ++++++++++++++++---- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index 85d5602..667d928 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -7,6 +7,7 @@ from sientia_do.notifications.models import NotificationLevel from ingestor.managers.data_manager import DataManager from ingestor.managers.opc_manager import OpcManager from ingestor.managers.resource_manager import ResourceManager +import ingestor.metrics as metrics class IngestorManager(): @@ -162,6 +163,8 @@ class IngestorManager(): self.opc_managers[server].disconnect() self.opc_managers.pop(server, None) + metrics.OPC_MANAGERS_ACTIVE.labels(pod_id=self.pod_id).set(len(self.opc_managers)) + def check_opc_servers_integrity(self): """ Checks the integrity of the OPC servers and updates the OPC servers if necessary. @@ -179,6 +182,8 @@ class IngestorManager(): for slot, _config in self.managed_tags.items(): self.managed_tags[slot].pop(server, None) + metrics.OPC_MANAGERS_ACTIVE.labels(pod_id=self.pod_id).set(len(self.opc_managers)) + def declare_active(self): """ Declares the ingestor as active by sending a heartbeat signal to the resource manager. @@ -211,6 +216,7 @@ class IngestorManager(): leases = self.resource_manager.get_all_leases() self.number_of_slots = len(leases) if leases else 0 + metrics.LEASES_TOTAL.set(self.number_of_slots) return self.number_of_slots def get_number_of_slots(self) -> int: @@ -224,6 +230,7 @@ class IngestorManager(): slots = self.resource_manager.get_all_slots() self.number_of_slots = len(slots) if slots else 0 + metrics.SLOTS_TOTAL.set(self.number_of_slots) return self.number_of_slots def get_slot_leases(self, max_slots: int = 1) -> Dict: @@ -255,9 +262,11 @@ class IngestorManager(): if slots is None: continue acquired[str(i)] = slots + metrics.SLOTS_ACQUIRED.labels(pod_id=self.pod_id).inc() if len(acquired) >= max_slots: self.managed_tags.update(acquired) + metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(len(self.managed_tags)) return acquired self.logger.warning( @@ -265,6 +274,7 @@ class IngestorManager(): f"Only {acquired} slots were leased." ) self.managed_tags.update(acquired) + metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(len(self.managed_tags)) return acquired def unsubscribe_slot(self, slot: str): @@ -332,6 +342,7 @@ class IngestorManager(): for lease_id in ids: self.resource_manager.drop_tag_lease(lease_id) + metrics.SLOTS_RELEASED.labels(pod_id=self.pod_id).inc() def manage_server(self, slot: str, server: str, server_config: dict, tags: dict) -> int: """ @@ -390,6 +401,7 @@ class IngestorManager(): tags_to_sub ) except Exception as e: + metrics.OPC_SUBSCRIPTION_ERRORS.labels(pod_id=self.pod_id, server=server, slot=slot).inc() trace = traceback.format_exc() self.notification_handler.build_and_send_notification( notification_id=f'OPC_SUBSCRIPTION_ERROR_{slot}:{server}', diff --git a/tests/unit/managers/test_ingestor_manager.py b/tests/unit/managers/test_ingestor_manager.py index f1bcbf3..a1f2d12 100644 --- a/tests/unit/managers/test_ingestor_manager.py +++ b/tests/unit/managers/test_ingestor_manager.py @@ -111,7 +111,8 @@ def test_initialize_opc_from_config_exception(traceback_mock, opc_manager, inges @patch('ingestor.managers.ingestor_manager.OpcManager') -def test_update_opc_servers(opc_manager, ingestor_manager): +@patch('ingestor.managers.ingestor_manager.metrics') +def test_update_opc_servers(metrics, opc_manager, ingestor_manager): manager1 = MagicMock( config={"config": "config1"}) manager2 = MagicMock( @@ -175,6 +176,9 @@ def test_update_opc_servers(opc_manager, ingestor_manager): assert ingestor_manager.opc_managers['server2'] != mock + metrics.OPC_MANAGERS_ACTIVE.labels.assert_called_once_with(pod_id=ingestor_manager.pod_id) + metrics.OPC_MANAGERS_ACTIVE.labels.return_value.set.assert_called_once_with(len(ingestor_manager.opc_managers)) + def test_declare_active(ingestor_manager): ingestor_manager.resource_manager.ingestor_heartbeat = MagicMock() @@ -196,36 +200,44 @@ def test_get_active_ingestors_empty(ingestor_manager): ingestor_manager.resource_manager.get_all_ingestors.assert_called_once() -def test_get_number_of_leases_success(ingestor_manager): +@patch('ingestor.managers.ingestor_manager.metrics') +def test_get_number_of_leases_success(metrics, ingestor_manager): ingestor_manager.resource_manager.get_all_leases = MagicMock( return_value=["lease1", "lease2"]) result = ingestor_manager.get_number_of_leases() assert result == 2 ingestor_manager.resource_manager.get_all_leases.assert_called_once() + metrics.LEASES_TOTAL.set.assert_called_once_with(2) -def test_get_number_of_leases_empty(ingestor_manager): +@patch('ingestor.managers.ingestor_manager.metrics') +def test_get_number_of_leases_empty(metrics, ingestor_manager): ingestor_manager.resource_manager.get_all_leases = MagicMock( return_value=None) result = ingestor_manager.get_number_of_leases() assert result == 0 ingestor_manager.resource_manager.get_all_leases.assert_called_once() + metrics.LEASES_TOTAL.set.assert_called_once_with(0) -def test_get_number_of_slots_success(ingestor_manager): +@patch('ingestor.managers.ingestor_manager.metrics') +def test_get_number_of_slots_success(metrics, ingestor_manager): ingestor_manager.resource_manager.get_all_slots = MagicMock( return_value=["slot1", "slot2"]) result = ingestor_manager.get_number_of_slots() assert result == 2 ingestor_manager.resource_manager.get_all_slots.assert_called_once() + metrics.SLOTS_TOTAL.set.assert_called_once_with(2) -def test_get_number_of_slots_empty(ingestor_manager): +@patch('ingestor.managers.ingestor_manager.metrics') +def test_get_number_of_slots_empty(metrics, ingestor_manager): ingestor_manager.resource_manager.get_all_slots = MagicMock( return_value=None) result = ingestor_manager.get_number_of_slots() assert result == 0 ingestor_manager.resource_manager.get_all_slots.assert_called_once() + metrics.SLOTS_TOTAL.set.assert_called_once_with(0) def test_get_slot_leases_1_success(ingestor_manager): @@ -339,13 +351,17 @@ def test_update_slot_config(ingestor_manager): assert ingestor_manager.resource_manager.renew_tag_lease.call_count == 3 -def test_drop_slot_leases(ingestor_manager): +@patch('ingestor.managers.ingestor_manager.metrics') +def test_drop_slot_leases(metrics, ingestor_manager): ingestor_manager.resource_manager.drop_tag_lease = MagicMock() ingestor_manager.drop_slot_leases(["1", "2"]) ingestor_manager.resource_manager.drop_tag_lease.assert_any_call("1") ingestor_manager.resource_manager.drop_tag_lease.assert_any_call("2") + metrics.SLOTS_RELEASED.labels.assert_any_call(pod_id=ingestor_manager.pod_id) + metrics.SLOTS_RELEASED.labels.return_value.inc.assert_any_call() + def test_manage_server_no_server(ingestor_manager): ingestor_manager.opc_managers = { @@ -491,7 +507,8 @@ def test_subscribe_to_tags(ingestor_manager): 'server3', None) -def test_check_opc_servers_integrity_all_healthy(ingestor_manager): +@patch('ingestor.managers.ingestor_manager.metrics') +def test_check_opc_servers_integrity_all_healthy(metrics, ingestor_manager): # Setup mock OPC managers opc_manager1 = MagicMock() opc_manager1.check_cycles.return_value = None @@ -523,6 +540,9 @@ def test_check_opc_servers_integrity_all_healthy(ingestor_manager): # Verify that no reinitialization was needed ingestor_manager.initialize_opc_from_config.assert_not_called() + metrics.OPC_MANAGERS_ACTIVE.labels.assert_called_once_with(pod_id=ingestor_manager.pod_id) + metrics.OPC_MANAGERS_ACTIVE.labels.return_value.set.assert_called_once_with(len(ingestor_manager.opc_managers)) + def test_check_opc_servers_integrity_server_lost(ingestor_manager): # Setup mock OPC manager that will be lost From d7ba8a4ab418467e37d2327f7e296d82fc7fb55c Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Fri, 30 May 2025 12:30:51 -0300 Subject: [PATCH 23/28] SIENTIAPDE-1083: add env HTTP_SERVER_PORT for prometheus metrics. --- ingestor/app.py | 5 +++-- values.yaml | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ingestor/app.py b/ingestor/app.py index 20d2a9f..e7c68f8 100644 --- a/ingestor/app.py +++ b/ingestor/app.py @@ -58,8 +58,9 @@ def signal_handler(_signum, _frame): def start_prometheus_server(): try: - start_http_server(8000) - print("Prometheus server started on port 8000.") + port = int(os.getenv("HTTP_SERVER_PORT", 4840)) + start_http_server(port) + print(f"Prometheus server started on port {port}.") metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP except Exception as e: print(f"Failed to start Prometheus server: {e}") diff --git a/values.yaml b/values.yaml index a94bca0..ad2f8f5 100644 --- a/values.yaml +++ b/values.yaml @@ -112,7 +112,7 @@ tolerations: [] affinity: {} service: - enabled: false + enabled: true type: ClusterIP port: 4840 targetPort: 4840 @@ -123,7 +123,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-opc-ingestor.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1049-lidar-com-erros-de-reconexao-no-ingestor-opc" + value: "main" - name: PYTHON_APP value: "ingestor.app" @@ -152,6 +152,8 @@ env: value: "10" - name: LOG_LEVEL value: "DEBUG" + - name: HTTP_SERVER_PORT + value: "4840" ssh: From 7f9d9aeba5e001fd982d40421a5cf9438979cb67 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Fri, 30 May 2025 12:40:47 -0300 Subject: [PATCH 24/28] SIENTIAPDE-1083: fix app test. --- tests/unit/test_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py index dda7701..5705c67 100644 --- a/tests/unit/test_app.py +++ b/tests/unit/test_app.py @@ -128,7 +128,7 @@ def test_main_successful_run_one_loop(mock_app_env, capsys): mock_ingestor_instance.logger.info.assert_any_call("Main loop exit_signaled.") captured = capsys.readouterr() - assert "Prometheus server started on port 8000." in captured.out + assert "Prometheus server started on port 4840." in captured.out def test_main_prometheus_server_fails_to_start(mock_app_env, capsys): From 6dc304d73d313bde2b9a709dd4ecc5d4e9d53a96 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Fri, 30 May 2025 13:21:53 -0300 Subject: [PATCH 25/28] SIENTIAPDE-1083: fix app test 2. --- tests/unit/test_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py index 5705c67..c4af7bb 100644 --- a/tests/unit/test_app.py +++ b/tests/unit/test_app.py @@ -98,7 +98,7 @@ def test_main_successful_run_one_loop(mock_app_env, capsys): app.main() assert excinfo.value.code == 0 - mock_app_env["start_http_server"].assert_called_once_with(8000) + mock_app_env["start_http_server"].assert_called_once_with(4840) app.metrics.APP_UP.labels.assert_any_call(pod_id="test_pod") set_calls = mock_app_env["metrics_APP_UP_labels_set"].call_args_list assert call(1) in set_calls From 112d512a02f21f5278dd39fc4cfdac18c8f84b17 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Mon, 2 Jun 2025 16:42:12 -0300 Subject: [PATCH 26/28] Bump sientia-module version to 0.0.8. --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index ad2f8f5..0bce04a 100644 --- a/values.yaml +++ b/values.yaml @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.0.2" + tag: "0.0.8" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: From 1af7e80e5b5a1e0693a867bb4fc10f1788de00c9 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 9 Jun 2025 08:42:22 -0300 Subject: [PATCH 27/28] SIENTIAPDE-1094 Update settings for language services, adjust simulator port, enhance error handling in ingestor, and refine Redis feeder script --- .vscode/settings.json | 3 ++- docker-compose.yaml | 2 +- ingestor/app.py | 14 +++++++++++--- ingestor/ingestor.py | 14 ++++++++------ requirements.txt | 2 +- simulator/redis-feeder.py | 13 +++++++------ 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index a595a07..dc84355 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,5 +8,6 @@ }, "python.languageServer": "Pylance", "python.analysis.typeCheckingMode": "standard", - "editor.suggestSelection": "first" + "editor.suggestSelection": "first", + "windsurfPyright.disableLanguageServices": true } diff --git a/docker-compose.yaml b/docker-compose.yaml index 2216ee5..1dbc796 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -88,7 +88,7 @@ services: GIT_BRANCH: ${SIMULATOR_GIT_BRANCH} container_name: simulator ports: - - "4840:4840" + - "4841:4840" depends_on: - kafka - redis diff --git a/ingestor/app.py b/ingestor/app.py index e7c68f8..651d4a5 100644 --- a/ingestor/app.py +++ b/ingestor/app.py @@ -16,14 +16,21 @@ POD_ID = os.getenv("HOSTNAME", "localhost") def main(): start_prometheus_server() ingestor = Ingestor() - ingestor.prepare_ingestor() + try: + ingestor.prepare_ingestor() + except Exception as e: + metrics.APP_ERRORS_TOTAL.labels( + pod_id=POD_ID).inc() # Increment errors + print(f"Failed to prepare ingestor: {e}") + exit_signal.set() ingestor.logger.info("Ingestor prepared. Starting main loop.") while not exit_signal.is_set(): start_time = time() # Start loop timer try: ingestor.loop() - metrics.APP_LOOP_COUNT.labels(pod_id=POD_ID).inc() # Increment loop counter + metrics.APP_LOOP_COUNT.labels( + pod_id=POD_ID).inc() # Increment loop counter exit_signal.wait(ingestor.poll_interval) @@ -33,7 +40,8 @@ def main(): except Exception: print("Exception in main loop. Setting exit_signal flag.") traceback.print_exc() - metrics.APP_ERRORS_TOTAL.labels(pod_id=POD_ID).inc() # Increment errors + metrics.APP_ERRORS_TOTAL.labels( + pod_id=POD_ID).inc() # Increment errors exit_signal.set() finally: # Record loop duration diff --git a/ingestor/ingestor.py b/ingestor/ingestor.py index 883af54..6540426 100644 --- a/ingestor/ingestor.py +++ b/ingestor/ingestor.py @@ -49,13 +49,14 @@ class Ingestor: self.notification_handler = NotificationHandler( servers=self.kafka_servers, logger=self.logger, - project_name="OPC_INGESTOR", - pipeline_name="-", - trigger_name="-", - model_name="-", - model="-", + project_name="OPC_INGESTOR" ) + self.notification_handler.base_notification.pipeline = 'OPC_INGESTOR' + self.notification_handler.base_notification.trigger = 'INGESTOR' + self.notification_handler.base_notification.model_name = '-' + self.notification_handler.base_notification.model_id = '-' + self.ingestor_manager = None def shutdown(self): @@ -79,7 +80,8 @@ class Ingestor: logger = getLogger(__name__) logger.setLevel(getenv("LOG_LEVEL", "INFO")) handler = StreamHandler() - formatter = Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + formatter = Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) diff --git a/requirements.txt b/requirements.txt index 40a9a06..8d5d26c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ asyncua==1.1.5 redis -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.1.14 prometheus_client diff --git a/simulator/redis-feeder.py b/simulator/redis-feeder.py index d3dde93..d8e42c0 100644 --- a/simulator/redis-feeder.py +++ b/simulator/redis-feeder.py @@ -1,5 +1,6 @@ -import redis import json +import redis + # Redis connection settings REDIS_HOST = "localhost" @@ -7,19 +8,19 @@ REDIS_PORT = 6379 REDIS_USERNAME = None # "default" REDIS_PASSWORD = None # "bdnZOpcyiL" -OPC_URL = "opc.tcp://sientia-opc-simulator-service.sientia-opc.svc.cluster.local:4840" -OPC_URL = "opc.tcp://localhost:4840" +OPC_URL = "opc.tcp://sientia-opc-simulator-service.sientia-opc.svc.cluster.local:4841" +OPC_URL = "opc.tcp://localhost:4841" # Connect to Redis r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True, username=REDIS_USERNAME, password=REDIS_PASSWORD) # Define the key pattern to target -pattern = "slot:opc_tags:*" +PATTERN = "slot:opc_tags:*" # Step 1: Find and delete matching keys -print("🔍 Searching for keys matching:", pattern) -for key in r.scan_iter(match=pattern): +print("🔍 Searching for keys matching:", PATTERN) +for key in r.scan_iter(match=PATTERN): r.delete(key) print(f"❌ Deleted: {key}") From a83d4ac72806777c2631348d8f5f287f29364c81 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 9 Jun 2025 11:30:43 -0300 Subject: [PATCH 28/28] SIENTIAPDE-1094 Refactor test___init__ to remove unnecessary parameters from notification_handler assertion --- tests/unit/test_ingestor.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/unit/test_ingestor.py b/tests/unit/test_ingestor.py index 54ebdc7..cd39f23 100644 --- a/tests/unit/test_ingestor.py +++ b/tests/unit/test_ingestor.py @@ -47,11 +47,7 @@ def test___init__(notification_handler, init_logger, getenv): notification_handler.assert_called_once_with( servers=["localhost:9092", "localhost:35"], logger=ingestor.logger, - project_name="OPC_INGESTOR", - pipeline_name="-", - trigger_name="-", - model_name="-", - model="-" + project_name="OPC_INGESTOR" )