Update dependencies, improve CI workflow, and enhance code formatting

- Updated the `sientia-dataops-library` dependency version from 1.4.3 to 1.4.6 in `requirements.txt`.
- Modified the GitHub Actions workflow to install development and runtime dependencies separately, improving clarity and organization.
- Added code formatting and linting checks using Ruff, along with type checking using mypy, to ensure code quality.
- Updated `.gitignore` to include additional cache directories and log files.
- Refactored code in various files for consistency in string formatting and improved logging messages.

.github/workflows/quality-gate.yml

@@ -43,11 +43,38 @@ jobs:
           mkdir -p ~/.ssh
           ssh-keyscan github.com >> ~/.ssh/known_hosts
 
-      - name: 📦 Install Dependencies
+      - name: 📦 Install Development Dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install pytest pytest-cov pytest-asyncio
+          pip install -r requirements-dev.txt
+
+      - name: 📦 Install Runtime Dependencies
+        run: |
+          pip install -r ${{ steps.prepare-requirements.outputs.PROCESSED_REQUIREMENTS_FILE }}
+
+      - name: 📝 Code Formatting Check (Ruff)
+        run: |
+          echo "Checking code formatting..."
+          ruff format --check ingestor/ tests/
+        continue-on-error: false
+
+      - name: 🔎 Code Linting (Ruff)
+        run: |
+          echo "Running linting checks..."
+          ruff check ingestor/ tests/
+        continue-on-error: false
+
+      - name: 🏷️ Type Checking (mypy)
+        run: |
+          echo "Running type checks..."
+          mypy ingestor/
+        continue-on-error: true
+
+      - name: 🔒 Security Analysis (Bandit)
+        run: |
+          echo "Running security analysis..."
+          bandit -r ingestor/ -ll -q
+        continue-on-error: true
 
       - name: 🧪 Run Tests with Pytest
         run: |

.gitignore

@@ -177,4 +177,8 @@ cython_debug/
 # VSCode
 .vscode/
 
-git_log
+git_log
+
+.git/
+.ruff_cache/
+.mypy_cache/

encrypt.py

@@ -0,0 +1,112 @@
+import os
+import argparse
+from pathspec import PathSpec
+import yaml
+
+'''
+Usage:
+    python .\encrypt.py path_to_dir output_file --ignore ignore_file --chunk-size 100000
+'''
+
+
+def load_ignore_patterns(ignore_file, include_library):
+    # Ensure the .gitignore file exists
+    if not os.path.exists(ignore_file):
+        raise FileNotFoundError(f"Ignore file not found at {ignore_file}")
+
+    # Load and parse the .gitignore patterns
+    with open(ignore_file, 'r') as file:
+        patterns = file.readlines()
+        if not include_library:
+            patterns.append('**/deploy/library/')
+
+        spec = PathSpec.from_lines('gitwildmatch', patterns)
+        return spec
+
+
+def is_ignored(file_path, spec):
+    """Check if a file should be ignored based on the ignore patterns."""
+    return spec.match_file(file_path) if spec else False
+
+
+def encode_file_tree_to_yaml(directory, ignore_file, include_library):
+    """Encode the file tree into a single YAML file."""
+    ignore_patterns = load_ignore_patterns(
+        ignore_file, include_library) if ignore_file else None
+    file_tree = {}
+
+    for root, dirs, files in os.walk(directory):
+        # Skip ignored directories
+        dirs[:] = [d for d in dirs if not is_ignored(
+            os.path.join(root, d), ignore_patterns)]
+
+        for file in files:
+            file_path = os.path.join(root, file)
+
+            # Skip ignored files
+            if is_ignored(file_path, ignore_patterns):
+                continue
+
+            # Read file content
+            try:
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    content = f.read()
+            except Exception as e:
+                print(f"Error reading file {file_path}: {e}")
+                raise
+
+            # Create nested dictionary structure
+            path_parts = os.path.relpath(file_path, directory).split(os.sep)
+            current_level = file_tree
+
+            # all except the last part (the file name)
+            for part in path_parts[:-1]:
+                current_level = current_level.setdefault(part, {})
+
+            # Add the file and its content
+            current_level[path_parts[-1]] = content
+    return yaml.dump(file_tree, default_flow_style=False)
+
+
+def chunk_and_write_file_tree_to_yaml(yaml_content, output_file, chunk_size=None):
+    """Chunk the YAML content and write it to the output file."""
+
+    chunks = [yaml_content] if chunk_size is None else [
+        yaml_content[i:i + chunk_size] for i in range(0, len(yaml_content), chunk_size)]
+
+    for i, chunk in enumerate(chunks):
+        chunk_file = f"{output_file}_{i}.yaml"
+        # Write the file tree to the output YAML file
+        with open(chunk_file, 'w', encoding='utf-8') as yaml_file:
+            yaml_file.write(chunk)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Encrypts file tree to yaml file")
+    parser.add_argument("input_directory", help="Directory to encode")
+    parser.add_argument("output_yaml_file", help="Output YAML file")
+    parser.add_argument("--ignore", default=None,
+                        help="Path to the ignore file")
+    parser.add_argument("--chunk-size", type=int, default=None,
+                        help="Chunk size for the output YAML file")
+    parser.add_argument("--library", type=bool, default=False,
+                        help="Incude the library in the output YAML file")
+
+    # Parse arguments
+    args = parser.parse_args()
+
+    # Example usage
+    directory_to_encode = args.input_directory
+    ignore_file_path = args.ignore
+    output_yaml_file = args.output_yaml_file
+    include_library = args.library
+
+    content = encode_file_tree_to_yaml(
+        directory_to_encode, ignore_file_path, include_library)
+    chunk_and_write_file_tree_to_yaml(
+        content, output_yaml_file, args.chunk_size)
+
+
+if __name__ == "__main__":
+    main()

ingestor/app.py

@@ -1,9 +1,9 @@
-import os
 import asyncio
+import os
 import signal
 import traceback
 from threading import Event
-from time import sleep, time
+from time import time
 
 from prometheus_client import start_http_server
 
@@ -11,7 +11,7 @@ import ingestor.metrics as metrics
 from ingestor.ingestor import Ingestor
 
 exit_signal = Event()
-POD_ID = os.getenv("HOSTNAME", "localhost")
+POD_ID = os.getenv('HOSTNAME', 'localhost')
 
 
 async def main():
@@ -43,30 +43,27 @@ async def main():
     try:
         await ingestor.prepare_ingestor()
     except Exception as e:
-        metrics.APP_ERRORS_TOTAL.labels(
-            pod_id=POD_ID).inc()  # Increment errors
-        ingestor.logger.error(f"Failed to prepare ingestor: {e}")
+        metrics.APP_ERRORS_TOTAL.labels(pod_id=POD_ID).inc()  # Increment errors
+        ingestor.logger.error(f'Failed to prepare ingestor: {e}')
         exit_signal.set()
-    ingestor.logger.info("Ingestor prepared. Starting main loop.")
+    ingestor.logger.info('Ingestor prepared. Starting main loop.')
 
     while not exit_signal.is_set():
         start_time = time()  # Start loop timer
         try:
             await ingestor.loop()
-            metrics.APP_LOOP_COUNT.labels(
-                pod_id=POD_ID).inc()  # Increment loop counter
+            metrics.APP_LOOP_COUNT.labels(pod_id=POD_ID).inc()  # Increment loop counter
 
             # Use asyncio.sleep instead of exit_signal.wait for better async compatibility
             await asyncio.sleep(ingestor.poll_interval)
 
         except KeyboardInterrupt:  # Handle Ctrl+C gracefully
-            print("KeyboardInterrupt received. Setting exit_signal flag.")
+            print('KeyboardInterrupt received. Setting exit_signal flag.')
             exit_signal.set()
         except Exception:
-            print("Exception in main loop. Setting exit_signal flag.")
+            print('Exception in main loop. Setting exit_signal flag.')
             traceback.print_exc()
-            metrics.APP_ERRORS_TOTAL.labels(
-                pod_id=POD_ID).inc()  # Increment errors
+            metrics.APP_ERRORS_TOTAL.labels(pod_id=POD_ID).inc()  # Increment errors
             exit_signal.set()
         finally:
             # Record loop duration
@@ -76,7 +73,7 @@ async def main():
     await ingestor.shutdown()
     metrics.APP_UP.labels(pod_id=POD_ID).set(0)  # Mark app as DOWN
 
-    ingestor.logger.info("Main loop exit_signaled.")
+    ingestor.logger.info('Main loop exit_signaled.')
 
     # Give Prometheus a chance to scrape one last time before exiting (optional)
     await asyncio.sleep(5)
@@ -96,7 +93,7 @@ def signal_handler(_signum, _frame):
         _signum: The signal number received
         _frame: The current stack frame (unused)
     """
-    print(f"Received signal {_signum}. Setting exit_signal flag.")
+    print(f'Received signal {_signum}. Setting exit_signal flag.')
     exit_signal.set()
 
 
@@ -115,12 +112,12 @@ def start_prometheus_server():
         Exception: If the server fails to start, the application will exit
     """
     try:
-        port = int(os.getenv("HTTP_METRICS_PORT", 9090))
+        port = int(os.getenv('HTTP_METRICS_PORT', 9090))
         start_http_server(port)
-        print(f"Prometheus server started on port {port}.")
+        print(f'Prometheus server started on port {port}.')
         metrics.APP_UP.labels(pod_id=POD_ID).set(1)  # Mark app as UP
     except Exception as e:
-        print(f"Failed to start Prometheus server: {e}")
+        print(f'Failed to start Prometheus server: {e}')
         os._exit(1)
 
 
@@ -139,13 +136,13 @@ def run_async_main():
         asyncio.set_event_loop(loop)
         loop.run_until_complete(main())
     except KeyboardInterrupt:
-        print("KeyboardInterrupt received in main thread.")
+        print('KeyboardInterrupt received in main thread.')
         exit_signal.set()
     finally:
         loop.close()
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
     signal.signal(signal.SIGHUP, signal_handler)

ingestor/ingestor.py

@@ -1,14 +1,12 @@
-import asyncio
-from os import getenv
 from copy import deepcopy
-from typing import Dict, Any
+from os import getenv
+from typing import Any
 
 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from sientia_do.observability.logger import get_logger
 
-from ingestor.managers.ingestor_manager import IngestorManager
-
 import ingestor.metrics as metrics
+from ingestor.managers.ingestor_manager import IngestorManager
 
 
 class Ingestor:
@@ -73,36 +71,36 @@ class Ingestor:
         - Metrics collection and monitoring
         """
 
-        kafka_servers = getenv("KAFKA_SERVERS", "localhost:9092")
-        export_to_kafka = getenv("EXPORT_TO_KAFKA", "false")
+        kafka_servers = getenv('KAFKA_SERVERS', 'localhost:9092')
+        export_to_kafka = getenv('EXPORT_TO_KAFKA', 'false')
 
-        if export_to_kafka and export_to_kafka == "true":
+        if export_to_kafka and export_to_kafka == 'true':
             export_to_kafka = True
         else:
             export_to_kafka = False
 
         self.export_to_kafka = export_to_kafka
-        self.redis_host = getenv("REDIS_HOST", "localhost")
-        self.redis_port = int(getenv("REDIS_PORT", "6379"))
-        self.redis_username = getenv("REDIS_USERNAME", None)
-        self.redis_password = getenv("REDIS_PASSWORD", None)
-        self.lease_ttl = int(getenv("LEASE_TTL", "10"))
-        self.heartbeat_ttl = int(getenv("HEARTBEAT_TTL", "20"))
-        self.pod_id = getenv("HOSTNAME", "localhost")
-        self.poll_interval = int(getenv("POLL_INTERVAL", "5"))
-        mongo_url = getenv("MONGODB_URL", "localhost:27017")
-        mongo_username = getenv("MONGODB_USERNAME", "sientia")
-        mongo_password = getenv("MONGODB_PASSWORD", "sientia")
-        self.mongo_database = getenv("MONGODB_DATABASE", "sientia")
-        self.mongo_connection_string = f"mongodb://{mongo_username}:{mongo_password}@{mongo_url}"
+        self.redis_host = getenv('REDIS_HOST', 'localhost')
+        self.redis_port = int(getenv('REDIS_PORT', '6379'))
+        self.redis_username = getenv('REDIS_USERNAME', None)
+        self.redis_password = getenv('REDIS_PASSWORD', None)
+        self.lease_ttl = int(getenv('LEASE_TTL', '10'))
+        self.heartbeat_ttl = int(getenv('HEARTBEAT_TTL', '20'))
+        self.pod_id = getenv('HOSTNAME', 'localhost')
+        self.poll_interval = int(getenv('POLL_INTERVAL', '5'))
+        mongo_url = getenv('MONGODB_URL', 'localhost:27017')
+        mongo_username = getenv('MONGODB_USERNAME', 'sientia')
+        mongo_password = getenv('MONGODB_PASSWORD', 'sientia')
+        self.mongo_database = getenv('MONGODB_DATABASE', 'sientia')
+        self.mongo_connection_string = f'mongodb://{mongo_username}:{mongo_password}@{mongo_url}'
 
-        self.kafka_servers = kafka_servers.split(",")
+        self.kafka_servers = kafka_servers.split(',')
         self.logger = get_logger(__name__)
         self.notification_handler = NotificationHandler(
             connection_string=self.mongo_connection_string,
             database=self.mongo_database,
             logger=self.logger,
-            project_name="opc_ingestor"
+            project_name='opc_ingestor',
         )
 
         self.metadata = {
@@ -148,7 +146,7 @@ class Ingestor:
         """
 
         if not acquired:
-            self.logger.warning("No slots available")
+            self.logger.warning('No slots available')
 
         else:
             # Subscribe to acquired slots
@@ -198,7 +196,7 @@ class Ingestor:
 
         # Get slot lease
         acquired = self.ingestor_manager.get_slot_leases()
-        self.logger.info(f"Acquired slots: {acquired}")
+        self.logger.info(f'Acquired slots: {acquired}')
 
         await self.handle_acquired_tags(acquired)
 
@@ -229,9 +227,7 @@ class Ingestor:
             # Get slot lease
             self.ingestor_manager.get_slot_leases(1)
 
-    async def manage_leases(
-        self, available_slots: int, lacking_ingestors: int, slot_diff: int
-    ):
+    async def manage_leases(self, available_slots: int, lacking_ingestors: int, slot_diff: int):
         """
         Manages the allocation and deallocation of slot leases for ingestors.
 
@@ -257,14 +253,13 @@ class Ingestor:
 
         if available_slots > 0 and lacking_ingestors > 0:
             # Some ingestors are inactive, so there are "available_slots" slots available
-            self.logger.info(f"Slots available: {available_slots}")
+            self.logger.info(f'Slots available: {available_slots}')
 
             # Get slot lease
             self.ingestor_manager.get_slot_leases(available_slots)
 
         elif lacking_ingestors <= 0 and slot_diff > 0:
-
-            self.logger.info(f"Extra slots available: {slot_diff}")
+            self.logger.info(f'Extra slots available: {slot_diff}')
             # There's enough slots for all ingestors, but this ingestor has more than one slot
             # So we need to drop the extra leases
 
@@ -281,7 +276,7 @@ class Ingestor:
                 len(self.ingestor_manager.managed_tags)
             )
 
-    async def update_ingestor_manager(self, old_managed_tags: Dict[str, Any]):
+    async def update_ingestor_manager(self, old_managed_tags: dict[str, Any]):
         """
         Updates the ingestor manager with new managed tags and handles configuration changes.
 
@@ -302,37 +297,39 @@ class Ingestor:
             - Updates metrics to reflect current state
         """
 
-        self.logger.debug(
-            f"Current managed tags: {self.ingestor_manager.managed_tags}")
+        self.logger.debug(f'Current managed tags: {self.ingestor_manager.managed_tags}')
 
         await self.ingestor_manager.update_opc_servers()
         new_managed_tags = deepcopy(self.ingestor_manager.managed_tags)
 
         self.logger.debug(
-            f"Comparing new managed tags {new_managed_tags} with old managed tags {old_managed_tags}"
+            f'Comparing new managed tags {new_managed_tags} with old managed tags {old_managed_tags}'
         )
 
         keys = set(new_managed_tags) | set(old_managed_tags)
 
-        changes = {k: (new_managed_tags.get(k), old_managed_tags.get(k))
-                   for k in keys if new_managed_tags.get(k) != old_managed_tags.get(k)}
+        changes = {
+            k: (new_managed_tags.get(k), old_managed_tags.get(k))
+            for k in keys
+            if new_managed_tags.get(k) != old_managed_tags.get(k)
+        }
 
-        self.logger.debug(f"Changes: {changes}")
+        self.logger.debug(f'Changes: {changes}')
 
         for slot, config in new_managed_tags.items():
             if slot not in old_managed_tags:
-                self.logger.info(f"Subscribing to new slot {slot}")
+                self.logger.info(f'Subscribing to new slot {slot}')
                 await self.ingestor_manager.subscribe_to_tags({slot: config})
                 continue
 
             if config != old_managed_tags[slot]:
-                self.logger.info(f"Resubscribing to slot {slot}")
+                self.logger.info(f'Resubscribing to slot {slot}')
                 await self.ingestor_manager.unsubscribe_slot(slot)
                 await self.ingestor_manager.subscribe_to_tags({slot: config})
 
         for slot in old_managed_tags.keys():
             if slot not in new_managed_tags:
-                self.logger.info(f"Unsubscribing from slot {slot}")
+                self.logger.info(f'Unsubscribing from slot {slot}')
                 await self.ingestor_manager.unsubscribe_slot(slot)
 
         # Ensure the gauge is updated after any potential changes here
@@ -366,7 +363,7 @@ class Ingestor:
 
         self.ingestor_manager.declare_active()
 
-        self.logger.info("Polling for slot updates...")
+        self.logger.info('Polling for slot updates...')
         # Get active ingestors
 
         current_managed_tags = deepcopy(self.ingestor_manager.managed_tags)
@@ -380,14 +377,14 @@ class Ingestor:
         metrics.ACTIVE_INGESTORS.set(number_of_ingestors)
 
         # Handle no slots
-        self.logger.info("Managing no slots...")
+        self.logger.info('Managing no slots...')
         self.manage_no_slots(number_of_slots)
 
         available_slots = number_of_slots - number_of_leases
         lacking_ingestors = number_of_slots - number_of_ingestors
         slot_diff = len(self.ingestor_manager.managed_tags) - 1
 
-        self.logger.info("Managing leases...")
+        self.logger.info('Managing leases...')
         await self.manage_leases(available_slots, lacking_ingestors, slot_diff)
 
         # Update managed slots gauge
@@ -396,23 +393,23 @@ class Ingestor:
         )
 
         self.logger.debug(
-            f"Active ingestors: {ingestors}, "
-            f"Number of slots: {number_of_slots}, "
-            f"Number of leases: {number_of_leases}, "
-            f"Managed tags: {self.ingestor_manager.managed_tags}, "
-            f"Managed servers: {self.ingestor_manager.opc_managers}"
+            f'Active ingestors: {ingestors}, '
+            f'Number of slots: {number_of_slots}, '
+            f'Number of leases: {number_of_leases}, '
+            f'Managed tags: {self.ingestor_manager.managed_tags}, '
+            f'Managed servers: {self.ingestor_manager.opc_managers}'
         )
         if not self.ingestor_manager.managed_tags:
             # No slots acquired
-            self.logger.info("No slots acquired in this loop")
+            self.logger.info('No slots acquired in this loop')
 
         # Update opc servers
-        self.logger.info("Updating slot config...")
+        self.logger.info('Updating slot config...')
         self.ingestor_manager.update_slot_config()
 
         # Check OPC cycles
-        self.logger.info("Checking OPC servers integrity...")
+        self.logger.info('Checking OPC servers integrity...')
         self.ingestor_manager.check_opc_servers_integrity()
 
-        self.logger.info("Updating managed tags...")
+        self.logger.info('Updating managed tags...')
         await self.update_ingestor_manager(current_managed_tags)

ingestor/managers/data_manager.py

@@ -1,16 +1,18 @@
 import json
+import os
+import traceback
 from time import sleep
-from pymongo import MongoClient
+
 from kafka import KafkaProducer
 from kafka.errors import NoBrokersAvailable
+from pymongo import MongoClient
 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from sientia_do.notifications.models import NotificationLevel
-from sientia_do.temporal.constants import now
-from sientia_do.temporal.activities.base import BaseActivity
 from sientia_do.observability.logger import Logger
-import traceback
+from sientia_do.temporal.activities.base import BaseActivity
+from sientia_do.temporal.constants import now
+
 import ingestor.metrics as metrics
-import os
 
 
 class DataManager(BaseActivity):
@@ -85,49 +87,43 @@ class DataManager(BaseActivity):
             - KAFKA_CONNECTION_STATUS: Set to 1 on successful connection, 0 on failure
         """
 
-        self.pod_id = os.getenv("HOSTNAME", "localhost")
+        self.pod_id = os.getenv('HOSTNAME', 'localhost')
         self.kafka_producer = None
         self.export_to_kafka = export_to_kafka
 
         if self.export_to_kafka:
             for i in range(0, 3):
                 logger.info(
-                    f"Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}"
+                    f'Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}'
                 )
                 try:
                     self.kafka_producer = KafkaProducer(
                         bootstrap_servers=kafka_servers,
                         value_serializer=lambda v: json.dumps(v).encode(
-                            "utf-8"
+                            'utf-8'
                         ),  # Serialize JSON messages
-                        key_serializer=lambda k: str(
-                            k).encode("utf-8") if k else None,
+                        key_serializer=lambda k: str(k).encode('utf-8') if k else None,
                     )
                     # Kafka connected
-                    metrics.KAFKA_CONNECTION_STATUS.labels(
-                        pod_id=self.pod_id).set(1)
+                    metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1)
                     break
                 except NoBrokersAvailable:
-                    logger.error(
-                        f"Kafka servers {kafka_servers} are not available. Retrying..."
-                    )
+                    logger.error(f'Kafka servers {kafka_servers} are not available. Retrying...')
                     sleep(5)
             else:
                 # Kafka not connected
-                metrics.KAFKA_CONNECTION_STATUS.labels(
-                    pod_id=self.pod_id).set(0)
+                metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
                 logger.error(
-                    f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts."
+                    f'Failed to connect to Kafka servers {kafka_servers} after 3 attempts.'
                 )
                 raise NoBrokersAvailable(
-                    f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts."
+                    f'Failed to connect to Kafka servers {kafka_servers} after 3 attempts.'
                 )
 
-            logger.info(
-                f"DataManager initialized with Kafka servers: {kafka_servers}")
+            logger.info(f'DataManager initialized with Kafka servers: {kafka_servers}')
 
         logger.info(
-            f"Trying to initializing DataManager with MongoDB servers: {mongo_connection_string}"
+            f'Trying to initializing DataManager with MongoDB servers: {mongo_connection_string}'
         )
 
         self.connection_string = mongo_connection_string
@@ -140,13 +136,11 @@ class DataManager(BaseActivity):
 
         self.mongo_db = self.mongo_client[self.database]
 
-        logger.info(
-            f"DataManager initialized with MongoDB servers: {self.connection_string}"
-        )
+        logger.info(f'DataManager initialized with MongoDB servers: {self.connection_string}')
 
-        BaseActivity.__init__(self, logger=logger,
-                              notification_handler=notification_handler,
-                              set_error_counter=True)
+        BaseActivity.__init__(
+            self, logger=logger, notification_handler=notification_handler, set_error_counter=True
+        )
 
     def shutdown(self):
         """
@@ -165,22 +159,19 @@ class DataManager(BaseActivity):
                 self.kafka_producer.flush(timeout=10)
                 self.kafka_producer.close()
                 # Mark as disconnected
-                metrics.KAFKA_CONNECTION_STATUS.labels(
-                    pod_id=self.pod_id).set(0)
+                metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
             except Exception as e:
-                self.logger.error(f"Error closing Kafka producer: {e}")
+                self.logger.error(f'Error closing Kafka producer: {e}')
         else:
-            self.logger.warning(
-                "Kafka producer is already closed or not initialized.")
+            self.logger.warning('Kafka producer is already closed or not initialized.')
 
         if self.mongo_client:
             try:
                 self.mongo_client.close()
             except Exception as e:
-                self.logger.error(f"Error closing MongoDB client: {e}")
+                self.logger.error(f'Error closing MongoDB client: {e}')
         else:
-            self.logger.warning(
-                "MongoDB client is already closed or not initialized.")
+            self.logger.warning('MongoDB client is already closed or not initialized.')
 
     def __del__(self):
         self.shutdown()
@@ -197,7 +188,7 @@ class DataManager(BaseActivity):
             msg: Kafka message object containing delivery details
         """
         self.logger.debug(
-            f"Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}"
+            f'Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}'
         )
 
     def delivery_error(self, err: str):
@@ -210,7 +201,7 @@ class DataManager(BaseActivity):
         Args:
             err: Error information from the failed delivery attempt
         """
-        self.logger.error(f"Delivery failed for record : {err}")
+        self.logger.error(f'Delivery failed for record : {err}')
 
     def publish(self, topic: str, data: dict) -> None:
         """
@@ -229,26 +220,22 @@ class DataManager(BaseActivity):
 
         if self.export_to_kafka:
             try:
-
-                self.logger.debug(
-                    f"Publishing message to topic {topic}: {data}")
+                self.logger.debug(f'Publishing message to topic {topic}: {data}')
                 self.kafka_producer.send(topic=topic, value=data).add_callback(
                     self.delivery_report
                 ).add_errback(self.delivery_error)
 
                 self.kafka_producer.flush(timeout=10)
-                metrics.KAFKA_MESSAGES_SENT.labels(
-                    pod_id=self.pod_id, topic=topic).inc()
+                metrics.KAFKA_MESSAGES_SENT.labels(pod_id=self.pod_id, topic=topic).inc()
 
             except Exception as e:
-                metrics.KAFKA_MESSAGES_ERRORS.labels(
-                    pod_id=self.pod_id, topic=topic).inc()
+                metrics.KAFKA_MESSAGES_ERRORS.labels(pod_id=self.pod_id, topic=topic).inc()
                 trace = traceback.format_exc()
                 self.send_notification(
                     metadata=self.metadata,
-                    notification_id=f"KAFKA_PRODUCER_ERROR_{topic}",
-                    message=f"Error publishing message to topic {topic}: {e}",
-                    block="kafka_producer",
+                    notification_id=f'KAFKA_PRODUCER_ERROR_{topic}',
+                    message=f'Error publishing message to topic {topic}: {e}',
+                    block='kafka_producer',
                     level=NotificationLevel.ERROR,
                     attachment_content=trace,
                 )
@@ -260,25 +247,22 @@ class DataManager(BaseActivity):
             collection.insert_one(
                 {
                     **data,
-                    "inserted_at": now(),
+                    'inserted_at': now(),
                 }
             )
-            self.logger.debug(
-                f"Message inserted into MongoDB collection {topic}: {data}")
+            self.logger.debug(f'Message inserted into MongoDB collection {topic}: {data}')
 
             metrics.TAG_WRITTEN_COUNT.labels(
-                pod_id=self.pod_id,
-                tag_name=data["name"],
-                collection_name=topic
+                pod_id=self.pod_id, tag_name=data['name'], collection_name=topic
             ).inc()
 
         except Exception as e:
             trace = traceback.format_exc()
             self.send_notification(
                 metadata=self.metadata,
-                notification_id=f"MONGO_PRODUCER_ERROR_{topic}",
-                message=f"Error inserting message to MongoDB: {e}",
-                block="mongo_producer",
+                notification_id=f'MONGO_PRODUCER_ERROR_{topic}',
+                message=f'Error inserting message to MongoDB: {e}',
+                block='mongo_producer',
                 level=NotificationLevel.ERROR,
                 attachment_content=trace,
             )

ingestor/managers/ingestor_manager.py

@@ -1,15 +1,16 @@
 import asyncio
 import traceback
-from typing import Dict, List
 from copy import deepcopy
+
 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from sientia_do.notifications.models import NotificationLevel
-from sientia_do.temporal.activities.base import BaseActivity
 from sientia_do.observability.logger import Logger
+from sientia_do.temporal.activities.base import BaseActivity
+
+import ingestor.metrics as metrics
 from ingestor.managers.data_manager import DataManager
 from ingestor.managers.opc_manager import OpcManager
 from ingestor.managers.resource_manager import ResourceManager
-import ingestor.metrics as metrics
 
 
 class IngestorManager(BaseActivity):
@@ -57,14 +58,20 @@ class IngestorManager(BaseActivity):
         metadata (dict): Application metadata
     """
 
-    def __init__(self,
-                 kafka_servers: str, redis_data: dict,
-                 lease_ttl: int, heartbeat_ttl: int,
-                 poll_interval: int, mongo_connection_string: str, mongo_database: str,
-                 metadata: dict,
-                 logger: Logger, notification_handler: NotificationHandler,
-                 export_to_kafka: bool = False):
-
+    def __init__(
+        self,
+        kafka_servers: str,
+        redis_data: dict,
+        lease_ttl: int,
+        heartbeat_ttl: int,
+        poll_interval: int,
+        mongo_connection_string: str,
+        mongo_database: str,
+        metadata: dict,
+        logger: Logger,
+        notification_handler: NotificationHandler,
+        export_to_kafka: bool = False,
+    ):
         redis_host = redis_data.get('host')
         redis_port = redis_data.get('port')
         redis_username = redis_data.get('username', None)
@@ -77,7 +84,7 @@ class IngestorManager(BaseActivity):
             export_to_kafka=export_to_kafka,
             metadata=metadata,
             logger=logger,
-            notification_handler=notification_handler
+            notification_handler=notification_handler,
         )
         self.opc_managers = {}
         self.resource_manager = ResourceManager(
@@ -98,9 +105,9 @@ class IngestorManager(BaseActivity):
 
         self.metadata = metadata
 
-        BaseActivity.__init__(self, logger=logger,
-                              notification_handler=notification_handler,
-                              set_error_counter=True)
+        BaseActivity.__init__(
+            self, logger=logger, notification_handler=notification_handler, set_error_counter=True
+        )
 
     async def initialize_opc_from_config(self, server_config: dict) -> OpcManager | None:
         """
@@ -130,8 +137,7 @@ class IngestorManager(BaseActivity):
         """
 
         try:
-            self.logger.info(
-                f"Initializing OpcManager at {server_config['url']}")
+            self.logger.info(f'Initializing OpcManager at {server_config["url"]}')
             manager = OpcManager(
                 name=server_config['name'],
                 url=server_config['url'],
@@ -142,7 +148,7 @@ class IngestorManager(BaseActivity):
                 metadata=self.metadata,
                 cert_path=server_config.get('cert_path'),
                 private_key_path=server_config.get('private_key_path'),
-                server_cert_path=server_config.get('server_cert_path')
+                server_cert_path=server_config.get('server_cert_path'),
             )
 
             manager.config = server_config
@@ -153,9 +159,9 @@ class IngestorManager(BaseActivity):
                 metadata=self.metadata,
                 notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}',
                 message=f'Error initializing OPC manager: {e}',
-                block="opc_manager",
+                block='opc_manager',
                 level=NotificationLevel.ERROR,
-                attachment_content=trace
+                attachment_content=trace,
             )
             self.logger.error(trace)
 
@@ -216,49 +222,41 @@ class IngestorManager(BaseActivity):
                 server_config.pop('tags', None)
                 server_instance = self.opc_managers.get(server, None)
                 if server_instance is None:
-                    self.logger.info(
-                        f"Initializing OPC manager for server {server}"
-                    )
-                    server_instance = await self.initialize_opc_from_config(
-                        server_config
-                    )
+                    self.logger.info(f'Initializing OPC manager for server {server}')
+                    server_instance = await self.initialize_opc_from_config(server_config)
 
                 elif server_instance.config != server_config:
-                    self.logger.warning(
-                        f"Reinitializing OPC manager for server {server}"
-                    )
+                    self.logger.warning(f'Reinitializing OPC manager for server {server}')
                     server_instance.disconnect()
                     del self.opc_managers[server]
-                    server_instance = await self.initialize_opc_from_config(
-                        server_config
-                    )
+                    server_instance = await self.initialize_opc_from_config(server_config)
                 else:
                     self.logger.debug(
-                        f"OPC manager for server {server} is already initialized and up to date"
+                        f'OPC manager for server {server} is already initialized and up to date'
                     )
 
                 if server_instance is not None:
                     self.opc_managers[server] = server_instance
                 else:
                     self.logger.warning(
-                        f"Failed to initialize OPC manager for server {server}, "
-                        f"removing server from managed tags."
+                        f'Failed to initialize OPC manager for server {server}, '
+                        f'removing server from managed tags.'
                     )
-                    _a = [self.managed_tags[slot].pop(server, None)
-                          for slot, _value in current_managed_tags.items()]
+                    _a = [
+                        self.managed_tags[slot].pop(server, None)
+                        for slot, _value in current_managed_tags.items()
+                    ]
 
         servers = list(self.opc_managers.keys())
         for server in servers:
             if server not in registered_servers:
                 self.logger.warning(
-                    f"Server {server} not found in managed tags. "
-                    f"Desconnecting from server."
+                    f'Server {server} not found in managed tags. Desconnecting from server.'
                 )
                 await self.opc_managers[server].disconnect()
                 self.opc_managers.pop(server, None)
 
-        metrics.OPC_MANAGERS_ACTIVE.labels(
-            pod_id=self.pod_id).set(len(self.opc_managers))
+        metrics.OPC_MANAGERS_ACTIVE.labels(pod_id=self.pod_id).set(len(self.opc_managers))
 
     def check_opc_servers_integrity(self):
         """
@@ -280,16 +278,12 @@ class IngestorManager(BaseActivity):
 
             is_lost = opc_manager.check_opc_listenning()
             if is_lost:
-                self.logger.warning(
-                    f"OPC server {server} is lost. "
-                    f"Server will be disconnected."
-                )
+                self.logger.warning(f'OPC server {server} is lost. Server will be disconnected.')
 
                 for slot, _config in self.managed_tags.items():
                     self.managed_tags[slot].pop(server, None)
 
-        metrics.OPC_MANAGERS_ACTIVE.labels(
-            pod_id=self.pod_id).set(len(self.opc_managers))
+        metrics.OPC_MANAGERS_ACTIVE.labels(pod_id=self.pod_id).set(len(self.opc_managers))
 
     def declare_active(self):
         """
@@ -306,7 +300,7 @@ class IngestorManager(BaseActivity):
 
         self.resource_manager.ingestor_heartbeat()
 
-    def get_active_ingestors(self) -> List[str]:
+    def get_active_ingestors(self) -> list[str]:
         """
         Retrieve a list of active ingestors.
 
@@ -363,7 +357,7 @@ class IngestorManager(BaseActivity):
         metrics.SLOTS_TOTAL.set(self.number_of_slots)
         return self.number_of_slots
 
-    def get_slot_leases(self, max_slots: int = 1) -> Dict:
+    def get_slot_leases(self, max_slots: int = 1) -> dict:
         """
         Acquires a specified number of resource slots by leasing them from the resource manager.
 
@@ -377,7 +371,7 @@ class IngestorManager(BaseActivity):
             max_slots (int): The maximum number of slots to lease. Defaults to 1.
 
         Returns:
-            Dict: A dictionary where the keys are the slot identifiers (as strings) 
+            Dict: A dictionary where the keys are the slot identifiers (as strings)
                   and the values are the leased slot details.
 
         Behavior:
@@ -394,7 +388,7 @@ class IngestorManager(BaseActivity):
         acquired = {}
         for i in range(1, self.number_of_slots + 1):
             if self.resource_manager.lease_tag(str(i)):
-                self.logger.info(f"Leased slot {i}")
+                self.logger.info(f'Leased slot {i}')
                 slots = self.resource_manager.get_tag_slot(str(i))
                 if slots is None:
                     continue
@@ -403,17 +397,14 @@ class IngestorManager(BaseActivity):
 
                 if len(acquired) >= max_slots:
                     self.managed_tags.update(acquired)
-                    metrics.SLOTS_MANAGED.labels(
-                        pod_id=self.pod_id).set(len(self.managed_tags))
+                    metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(len(self.managed_tags))
                     return acquired
 
         self.logger.warning(
-            f"Unable to acquire {max_slots} slots. "
-            f"Only {acquired} slots were leased."
+            f'Unable to acquire {max_slots} slots. Only {acquired} slots were leased.'
         )
         self.managed_tags.update(acquired)
-        metrics.SLOTS_MANAGED.labels(
-            pod_id=self.pod_id).set(len(self.managed_tags))
+        metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(len(self.managed_tags))
         return acquired
 
     async def unsubscribe_slot(self, slot: str):
@@ -441,7 +432,7 @@ class IngestorManager(BaseActivity):
 
     def update_slot_config(self):
         """
-        Updates the configuration of managed slots by renewing their leases, 
+        Updates the configuration of managed slots by renewing their leases,
         fetching the latest configurations, and handling any changes or removals.
 
         This method performs the following steps:
@@ -458,7 +449,7 @@ class IngestorManager(BaseActivity):
             - Updates OPC server subscriptions based on the current state of managed slots.
 
         Raises:
-            None explicitly, but relies on the behavior of `resource_manager` and 
+            None explicitly, but relies on the behavior of `resource_manager` and
             other dependencies for error handling.
 
         Logging:
@@ -480,17 +471,17 @@ class IngestorManager(BaseActivity):
         for slot in removed_slots:
             self.managed_tags.pop(slot, None)
 
-    def drop_slot_leases(self, ids: List[str]) -> None:
+    def drop_slot_leases(self, ids: list[str]) -> None:
         """
         Releases the leases associated with the specified slot IDs.
 
-        This method iterates through a list of slot IDs and calls the 
-        `drop_tag_lease` method of the `resource_manager` to release 
+        This method iterates through a list of slot IDs and calls the
+        `drop_tag_lease` method of the `resource_manager` to release
         the lease for each ID. It's used during load balancing and
         graceful shutdown scenarios.
 
         Args:
-            ids (List[str]): A list of slot IDs for which the leases 
+            ids (List[str]): A list of slot IDs for which the leases
                              should be released.
 
         Returns:
@@ -509,7 +500,7 @@ class IngestorManager(BaseActivity):
         Manages the subscription of tags to a specified OPC server and slot.
 
         This method ensures that the specified server and slot have an active subscription
-        for the provided tags. If the server or slot is not properly configured, or if 
+        for the provided tags. If the server or slot is not properly configured, or if
         subscription fails, appropriate error handling is performed.
 
         Args:
@@ -527,12 +518,12 @@ class IngestorManager(BaseActivity):
 
         Logs:
             - Logs informational messages about the subscription process.
-            - Logs errors if the server is not found, subscription creation fails, or 
+            - Logs errors if the server is not found, subscription creation fails, or
               tag subscription fails.
             - Logs a warning if a subscription is removed due to failure.
 
         Raises:
-            Exception: Any unexpected exceptions during subscription creation or tag 
+            Exception: Any unexpected exceptions during subscription creation or tag
             subscription are logged but not propagated.
 
         Side Effects:
@@ -541,77 +532,63 @@ class IngestorManager(BaseActivity):
             - Updates error metrics and notifications
         """
 
-        self.logger.info(
-            f"Subscribing to tags from {slot}:{server}"
-        )
+        self.logger.info(f'Subscribing to tags from {slot}:{server}')
         tags_to_sub = server_config.get('tags')
         if server not in self.opc_managers:
-            self.logger.error(
-                f"Server {server} not found in opc_managers."
-            )
+            self.logger.error(f'Server {server} not found in opc_managers.')
             return 1
         if slot not in self.opc_managers[server].subscriptions:
             try:
-                await self.opc_managers[server].create_subscription(
-                    slot
-                )
+                await self.opc_managers[server].create_subscription(slot)
             except Exception as e:
-                self.logger.error(
-                    f"Failed to create subscription for slot {slot}: {e}"
-                )
+                self.logger.error(f'Failed to create subscription for slot {slot}: {e}')
                 return 2
         try:
-            self.logger.info(
-                tags_to_sub
-            )
+            self.logger.info(tags_to_sub)
             await self.opc_managers[server].subscribe(
                 slot, deepcopy(tags_to_sub), self.poll_interval
             )
-            self.logger.info(
-                tags_to_sub
-            )
+            self.logger.info(tags_to_sub)
         except Exception as e:
             metrics.OPC_SUBSCRIPTION_ERRORS.labels(
-                pod_id=self.pod_id, server=server, slot=slot).inc()
+                pod_id=self.pod_id, server=server, slot=slot
+            ).inc()
             trace = traceback.format_exc()
             self.send_notification(
                 metadata=self.metadata,
                 notification_id=f'OPC_SUBSCRIPTION_ERROR_{slot}:{server}',
                 message=f'Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}',
-                block="opc_manager",
+                block='opc_manager',
                 level=NotificationLevel.ERROR,
-                attachment_content=trace
+                attachment_content=trace,
             )
             self.logger.error(trace)
 
-            self.logger.warning(
-                "Removing subscription from server "
-                f"{server} for slot {slot}"
-            )
+            self.logger.warning(f'Removing subscription from server {server} for slot {slot}')
             await self.opc_managers[server].unsubscribe(slot)
             return 2
         return 0
 
-    async def subscribe_to_tags(self, tags: Dict) -> None:
+    async def subscribe_to_tags(self, tags: dict) -> None:
         """
         Subscribes to a set of tags and manages their configurations.
 
         This method processes a dictionary of tags, iterating through each slot and server
-        configuration. It attempts to manage the server configurations and removes any 
+        configuration. It attempts to manage the server configurations and removes any
         servers that return a specific response code.
 
         Args:
-            tags (Dict): A dictionary containing tag configurations. The structure is 
+            tags (Dict): A dictionary containing tag configurations. The structure is
                          expected to be {slot: {server: server_config}}.
 
         Side Effects:
             - Logs the provided tags for debugging purposes.
-            - Updates the `managed_tags` attribute by removing servers that meet the 
+            - Updates the `managed_tags` attribute by removing servers that meet the
               removal criteria.
             - Establishes OPC subscriptions for all configured tags.
 
         Removal Criteria:
-            - If the `manage_server` method returns a response code of 2 for a given 
+            - If the `manage_server` method returns a response code of 2 for a given
               slot and server, that server is removed from the `managed_tags` attribute.
 
         The method ensures that only successfully configured servers remain in the
@@ -622,9 +599,7 @@ class IngestorManager(BaseActivity):
         self.logger.info(tags)
         for slot, slot_config in tags.items():
             for server, server_config in slot_config.items():
-                response = await self.manage_server(
-                    slot, server, server_config, tags
-                )
+                response = await self.manage_server(slot, server, server_config, tags)
                 if response == 2:
                     to_remove.append([slot, server])
 

ingestor/managers/opc_manager.py

@@ -1,15 +1,16 @@
 import json
-import asyncio
 from pathlib import Path
-from asyncua.crypto.security_policies import SecurityPolicyBasic256
+
 from asyncua import Client
-from sientia_do.notifications.models import NotificationLevel
+from asyncua.crypto.security_policies import SecurityPolicyBasic256
 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
-from sientia_do.temporal.activities.base import BaseActivity
+from sientia_do.notifications.models import NotificationLevel
 from sientia_do.observability.logger import Logger
-from sientia_do.temporal.constants import OPC_TIMEZONE, DATETIME_FORMAT_WITH_TZ
-from ingestor.managers.data_manager import DataManager
+from sientia_do.temporal.activities.base import BaseActivity
+from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, OPC_TIMEZONE
+
 import ingestor.metrics as metrics
+from ingestor.managers.data_manager import DataManager
 
 
 class OpcManager(BaseActivity):
@@ -54,9 +55,19 @@ class OpcManager(BaseActivity):
         metadata (dict): Application metadata
     """
 
-    def __init__(self, name: str, url: str, data_manager: DataManager, logger: Logger,
-                 server_uri: str, notification_handler: NotificationHandler, metadata: dict,
-                 cert_path: str = None, private_key_path: str = None, server_cert_path: str = None):
+    def __init__(
+        self,
+        name: str,
+        url: str,
+        data_manager: DataManager,
+        logger: Logger,
+        server_uri: str,
+        notification_handler: NotificationHandler,
+        metadata: dict,
+        cert_path: str = None,
+        private_key_path: str = None,
+        server_cert_path: str = None,
+    ):
         self.url = url
         self.name = name
         self.server_uri = server_uri
@@ -71,14 +82,14 @@ class OpcManager(BaseActivity):
         self.data_manager = data_manager
         self.metadata = metadata
 
-        BaseActivity.__init__(self, logger=logger,
-                              notification_handler=notification_handler,
-                              set_error_counter=True)
+        BaseActivity.__init__(
+            self, logger=logger, notification_handler=notification_handler, set_error_counter=True
+        )
 
         metrics.OPC_CONNECTION_STATUS.labels(
-            pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0)
-        metrics.OPC_TAGS_SUBSCRIBED.labels(
-            pod_id=self.pod_id, server_name=self.name).set(0)
+            pod_id=self.pod_id, server_name=self.name, server_url=self.url
+        ).set(0)
+        metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(0)
 
     def __str__(self):
         """
@@ -87,8 +98,10 @@ class OpcManager(BaseActivity):
         Returns:
             str: Human-readable representation showing server details and current state.
         """
-        return f"OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n" \
-            f"nodes={self.nodes}, subscriptions={self.subscriptions}"
+        return (
+            f'OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n'
+            f'nodes={self.nodes}, subscriptions={self.subscriptions}'
+        )
 
     async def shutdown(self):
         """
@@ -103,10 +116,9 @@ class OpcManager(BaseActivity):
         and ensure clean disconnection from OPC servers.
         """
         try:
-
             await self.disconnect()
         except Exception as e:
-            self.logger.error(f"Error during cleanup: {e}")
+            self.logger.error(f'Error during cleanup: {e}')
 
     async def set_security(self):
         """
@@ -133,11 +145,11 @@ class OpcManager(BaseActivity):
 
         if not all([self.cert_path, self.private_key_path]):
             raise ValueError(
-                "Certificate and private key paths must be provided for secure connection.")
+                'Certificate and private key paths must be provided for secure connection.'
+            )
         cert = Path(self.cert_path)
         private_key = Path(self.private_key_path)
-        server_cert = Path(
-            self.server_cert_path) if self.server_cert_path else None
+        server_cert = Path(self.server_cert_path) if self.server_cert_path else None
 
         await self.client.set_application_uri(self.server_uri)
         self.logger.info('Setting security...')
@@ -145,7 +157,7 @@ class OpcManager(BaseActivity):
             SecurityPolicyBasic256,
             certificate=str(cert),
             private_key=str(private_key),
-            server_certificate=str(server_cert)
+            server_certificate=str(server_cert),
         )
         await self.client.set_secure_channel_timeout(10000000)
         await self.client.set_session_timeout(10000000)
@@ -172,8 +184,7 @@ class OpcManager(BaseActivity):
             - OPC_CONNECTION_STATUS: Set to 1 on successful connection
         """
 
-        metrics.OPC_CONNECTIONS_TOTAL.labels(
-            pod_id=self.pod_id, server_name=self.name).inc()
+        metrics.OPC_CONNECTIONS_TOTAL.labels(pod_id=self.pod_id, server_name=self.name).inc()
         try:
             self.client = Client(self.url, watchdog_intervall=3600000)
             self.client.name = self.pod_id
@@ -182,14 +193,15 @@ class OpcManager(BaseActivity):
             self.logger.info(f'Starting connection to {self.name}...')
             await self.client.connect()
             metrics.OPC_CONNECTION_STATUS.labels(
-                pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(1)
+                pod_id=self.pod_id, server_name=self.name, server_url=self.url
+            ).set(1)
             self.logger.info(f'Connection to {self.name} successful.')
         except Exception as e:
             metrics.OPC_CONNECTION_STATUS.labels(
-                pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0)
-            metrics.OPC_CONNECTIONS_FAILED.labels(
-                pod_id=self.pod_id, server_name=self.name).inc()
-            self.logger.error(f"Failed to connect to {self.name}: {e}")
+                pod_id=self.pod_id, server_name=self.name, server_url=self.url
+            ).set(0)
+            metrics.OPC_CONNECTIONS_FAILED.labels(pod_id=self.pod_id, server_name=self.name).inc()
+            self.logger.error(f'Failed to connect to {self.name}: {e}')
             raise
 
     async def create_subscription(self, name: str, period: int = 500):
@@ -214,16 +226,16 @@ class OpcManager(BaseActivity):
         """
 
         if not self.client:
-            raise ValueError("Client not connected. Call connect first.")
+            raise ValueError('Client not connected. Call connect first.')
         try:
             p = period if period is not None else 500
             self.subscriptions[name] = await self.client.create_subscription(p, self)
             self.logger.info(f'Subscription {name} created on {self.name}.')
             metrics.OPC_SUBSCRIPTIONS_CREATED.labels(
-                pod_id=self.pod_id, server_name=self.name, slot_name=name).inc()
+                pod_id=self.pod_id, server_name=self.name, slot_name=name
+            ).inc()
         except Exception as e:
-            self.logger.error(
-                f"Failed to create subscription {name} on {self.name}: {e}")
+            self.logger.error(f'Failed to create subscription {name} on {self.name}: {e}')
             raise
 
     async def subscribe(self, subscription: str, nodes: dict, collect_period: int):
@@ -252,23 +264,23 @@ class OpcManager(BaseActivity):
         """
 
         if not self.subscriptions.get(subscription):
-            raise ValueError(
-                "Subscription not created. Call create_subscription first.")
+            raise ValueError('Subscription not created. Call create_subscription first.')
 
-        self.logger.info(f"Subscribing to {subscription} on {self.name}...")
-        self.logger.info(f"Subscribing to nodes: {nodes}")
+        self.logger.info(f'Subscribing to {subscription} on {self.name}...')
+        self.logger.info(f'Subscribing to nodes: {nodes}')
         addr_nodes = [self.client.get_node(n) for n in nodes]
-        self.logger.debug(f"Addr nodes: {addr_nodes}")
+        self.logger.debug(f'Addr nodes: {addr_nodes}')
         self.nodes.update(nodes)
-        self.logger.debug(f"Nodes: {self.nodes}")
-        metrics.OPC_TAGS_SUBSCRIBED.labels(
-            pod_id=self.pod_id, server_name=self.name).set(len(self.nodes))
+        self.logger.debug(f'Nodes: {self.nodes}')
+        metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(
+            len(self.nodes)
+        )
         self.collect_period = collect_period
 
         for node, config in self.nodes.items():
             self.nodes[node]['cycle_rule'] = {
-                'cycle_increment': collect_period*1000/float(config['frequency']),
-                'cycle_count': 0
+                'cycle_increment': collect_period * 1000 / float(config['frequency']),
+                'cycle_count': 0,
             }
 
         await self.subscriptions[subscription].subscribe_data_change(addr_nodes)
@@ -288,18 +300,17 @@ class OpcManager(BaseActivity):
             - An info message upon successful unsubscription.
 
         Behavior:
-            - If the subscription exists, it is deleted and removed from the 
+            - If the subscription exists, it is deleted and removed from the
               subscriptions dictionary.
             - If the subscription does not exist, no action is taken.
         """
 
         if not self.subscriptions.get(subscription):
-            self.logger.warning(
-                f"Subscription '{subscription}' not found. Cannot unsubscribe.")
+            self.logger.warning(f"Subscription '{subscription}' not found. Cannot unsubscribe.")
             return
         await self.subscriptions[subscription].delete()
         del self.subscriptions[subscription]
-        self.logger.info(f"Unsubscribed from {subscription}.")
+        self.logger.info(f'Unsubscribed from {subscription}.')
 
     async def disconnect(self):
         """
@@ -322,28 +333,27 @@ class OpcManager(BaseActivity):
 
         self.logger.warning('Disconnecting from OPC server')
         if self.client is None:
-            self.logger.warning("Client already disconnected.")
+            self.logger.warning('Client already disconnected.')
             return
         try:
             for sub in self.subscriptions:
                 await self.subscriptions[sub].delete()
-            self.logger.warning("Deleted all subscriptions.")
+            self.logger.warning('Deleted all subscriptions.')
         except Exception as sub_error:
-            self.logger.error(f"Failed to clean up subscription: {sub_error}")
+            self.logger.error(f'Failed to clean up subscription: {sub_error}')
 
         try:
             await self.client.disconnect()
         except Exception as conn_error:
-            self.logger.error(
-                f"Failed to disconnect from OPC UA server: {conn_error}")
+            self.logger.error(f'Failed to disconnect from OPC UA server: {conn_error}')
         finally:
             del self.client
             self.client = None
             metrics.OPC_CONNECTION_STATUS.labels(
-                pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0)
-            metrics.OPC_TAGS_SUBSCRIBED.labels(
-                pod_id=self.pod_id, server_name=self.name).set(0)
-            self.logger.warning("Disconnected from OPC UA server.")
+                pod_id=self.pod_id, server_name=self.name, server_url=self.url
+            ).set(0)
+            metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(0)
+            self.logger.warning('Disconnected from OPC UA server.')
 
     async def datachange_notification(self, node, _val, data):
         """
@@ -371,35 +381,33 @@ class OpcManager(BaseActivity):
         monitored_item = data.monitored_item
         value = monitored_item.Value.Value.Value
         # source_timestamp
-        source_timestamp = monitored_item.Value.SourceTimestamp.replace(
-            tzinfo=OPC_TIMEZONE)
+        source_timestamp = monitored_item.Value.SourceTimestamp.replace(tzinfo=OPC_TIMEZONE)
         tag = str(node)
 
         self.logger.debug(
-            f"Data change notification received for tag:"
-            f"{tag} after {self.nodes[tag]['cycle_rule']['cycle_count']} cycles")
+            f'Data change notification received for tag:'
+            f'{tag} after {self.nodes[tag]["cycle_rule"]["cycle_count"]} cycles'
+        )
 
         self.nodes[tag]['cycle_rule']['cycle_count'] = 0
         self.non_receive_count = 0
-        metrics.OPC_CYCLES_WITHOUT_DATA.labels(
-            pod_id=self.pod_id, server_name=self.name).set(0)
+        metrics.OPC_CYCLES_WITHOUT_DATA.labels(pod_id=self.pod_id, server_name=self.name).set(0)
 
         data = {
             'tag': tag,
             'name': self.nodes[str(node)]['tag_name'],
             'timestamp': source_timestamp.strftime(DATETIME_FORMAT_WITH_TZ),
-            'value': value
+            'value': value,
         }
 
-        _a = [self.data_manager.publish(e, data)
-              for e in self.nodes[tag]['topics']]
+        _a = [self.data_manager.publish(e, data) for e in self.nodes[tag]['topics']]
 
     def check_cycles(self):
         """
         Checks the cycle counts for all monitored nodes and sends
         notifications if thresholds are exceeded.
 
-        This method iterates through all monitored nodes and updates their cycle counts based on 
+        This method iterates through all monitored nodes and updates their cycle counts based on
         configured increments. If a node's cycle count exceeds a threshold (5 cycles), it triggers
         a warning notification.
 
@@ -408,8 +416,7 @@ class OpcManager(BaseActivity):
             - Sends warning notifications for nodes exceeding cycle thresholds
         """
         for node, config in self.nodes.items():
-            self.nodes[node]['cycle_rule']['cycle_count'] += config[
-                'cycle_rule']['cycle_increment']
+            self.nodes[node]['cycle_rule']['cycle_count'] += config['cycle_rule']['cycle_increment']
             if self.nodes[node]['cycle_rule']['cycle_count'] >= 5:
                 name = config['tag_name']
                 cycles = self.nodes[node]['cycle_rule']['cycle_count']
@@ -417,8 +424,8 @@ class OpcManager(BaseActivity):
                     metadata=self.metadata,
                     notification_id=f'TAG_{node}:{name}_LISTENNING_STOPPED',
                     message=f'{cycles} cycles without receive from {node}:{name}',
-                    block="opc_manager",
-                    level=NotificationLevel.WARNING
+                    block='opc_manager',
+                    level=NotificationLevel.WARNING,
                 )
 
     def check_opc_listenning(self) -> bool:
@@ -441,26 +448,26 @@ class OpcManager(BaseActivity):
         """
 
         self.non_receive_count += 1
-        metrics.OPC_CYCLES_WITHOUT_DATA.labels(
-            pod_id=self.pod_id, server_name=self.name).set(self.non_receive_count)
+        metrics.OPC_CYCLES_WITHOUT_DATA.labels(pod_id=self.pod_id, server_name=self.name).set(
+            self.non_receive_count
+        )
         if self.non_receive_count >= 5:
             self.send_notification(
                 metadata=self.metadata,
                 notification_id=f'OPC_LISTENNING_STOPPED__{self.name}',
                 message=f'{self.non_receive_count} cycles without '
                 f'receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}',
-                block="opc_manager",
-                level=NotificationLevel.ERROR
+                block='opc_manager',
+                level=NotificationLevel.ERROR,
             )
         if self.non_receive_count >= 15:
-            metrics.OPC_RECONNECTIONS_TOTAL.labels(
-                pod_id=self.pod_id, server_name=self.name).inc()
+            metrics.OPC_RECONNECTIONS_TOTAL.labels(pod_id=self.pod_id, server_name=self.name).inc()
             self.send_notification(
                 metadata=self.metadata,
                 notification_id=f'OPC_CONNECTION_RETRY__{self.name}',
                 message=f'Retrying to connect to server {self.name}',
-                block="opc_manager",
-                level=NotificationLevel.ERROR
+                block='opc_manager',
+                level=NotificationLevel.ERROR,
             )
             return True
         return False

ingestor/managers/resource_manager.py

@@ -1,13 +1,14 @@
 import json
-from typing import List
-from redis import Redis
 from time import time
-import ingestor.metrics as metrics
+
+from redis import Redis
 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from sientia_do.notifications.models import NotificationLevel
 from sientia_do.observability.logger import Logger
 from sientia_do.temporal.activities.base import BaseActivity
 
+import ingestor.metrics as metrics
+
 
 class ResourceManager(BaseActivity):
     """
@@ -80,9 +81,9 @@ class ResourceManager(BaseActivity):
         Metrics:
             - REDIS_CONNECTION_STATUS: Set to 1 on successful connection, 0 on failure
         """
-        BaseActivity.__init__(self, logger=logger,
-                              notification_handler=notification_handler,
-                              set_error_counter=True)
+        BaseActivity.__init__(
+            self, logger=logger, notification_handler=notification_handler, set_error_counter=True
+        )
         try:
             self.redis = Redis(
                 host=host,
@@ -94,7 +95,7 @@ class ResourceManager(BaseActivity):
             self.redis.ping()
             metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1)
         except Exception as e:
-            self.logger.error(f"Failed to connect to Redis: {e}")
+            self.logger.error(f'Failed to connect to Redis: {e}')
             metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
             raise
 
@@ -146,9 +147,9 @@ class ResourceManager(BaseActivity):
             ).inc()
             self.send_notification(
                 metadata=self.metadata,
-                notification_id=f"REDIS_OPERATION_ERROR_{operation_name}",
+                notification_id=f'REDIS_OPERATION_ERROR_{operation_name}',
                 message=f"Error in Redis operation '{operation_name}': {e}",
-                block="redis_manager",
+                block='redis_manager',
                 level=NotificationLevel.ERROR,
             )
             raise
@@ -172,7 +173,7 @@ class ResourceManager(BaseActivity):
             - REDIS_OPERATIONS_DURATION: Records timing for get operations
         """
 
-        history = self._execute_redis_op("get", self.redis.get, key)
+        history = self._execute_redis_op('get', self.redis.get, key)
         return json.loads(history) if history else None
 
     def get_tag_slot(self, id: str) -> dict:
@@ -193,7 +194,7 @@ class ResourceManager(BaseActivity):
         and delegates to the get() method for the actual Redis operation.
         """
 
-        return self.get(f"slot:opc_tags:{id}")
+        return self.get(f'slot:opc_tags:{id}')
 
     def ingestor_heartbeat(self) -> None:
         """
@@ -215,9 +216,9 @@ class ResourceManager(BaseActivity):
         """
 
         self._execute_redis_op(
-            "set",
+            'set',
             self.redis.set,
-            f"heartbeat:ingestor:{self.pod_id}",
+            f'heartbeat:ingestor:{self.pod_id}',
             1,
             ex=self.heartbeat_ttl,
         )
@@ -249,9 +250,9 @@ class ResourceManager(BaseActivity):
         """
 
         return self._execute_redis_op(
-            "set_nx",
+            'set_nx',
             self.redis.set,
-            f"lease:opc_tags:{tag_id}",
+            f'lease:opc_tags:{tag_id}',
             self.pod_id,
             nx=True,
             ex=self.lease_ttl,
@@ -282,12 +283,10 @@ class ResourceManager(BaseActivity):
             - REDIS_OPERATIONS_DURATION: Records timing for renewal operations
         """
 
-        current = self._execute_redis_op(
-            "get", self.redis.get, f"lease:opc_tags:{tag_id}"
-        )
+        current = self._execute_redis_op('get', self.redis.get, f'lease:opc_tags:{tag_id}')
         if current == self.pod_id:
             self._execute_redis_op(
-                "expire", self.redis.expire, f"lease:opc_tags:{tag_id}", self.lease_ttl
+                'expire', self.redis.expire, f'lease:opc_tags:{tag_id}', self.lease_ttl
             )
             return True
         return False
@@ -313,10 +312,9 @@ class ResourceManager(BaseActivity):
             - REDIS_OPERATIONS_DURATION: Records timing for lease dropping operations
         """
 
-        self._execute_redis_op("delete", self.redis.delete,
-                               f"lease:opc_tags:{tag_id}")
+        self._execute_redis_op('delete', self.redis.delete, f'lease:opc_tags:{tag_id}')
 
-    def get_all_ingestors(self) -> List[str]:
+    def get_all_ingestors(self) -> list[str]:
         """
         Retrieves all active ingestors from Redis.
 
@@ -338,9 +336,9 @@ class ResourceManager(BaseActivity):
             - REDIS_OPERATIONS_DURATION: Records timing for ingestor discovery
         """
 
-        return self._execute_redis_op("keys", self.redis.keys, "heartbeat:ingestor:*")
+        return self._execute_redis_op('keys', self.redis.keys, 'heartbeat:ingestor:*')
 
-    def get_all_slots(self) -> List[str]:
+    def get_all_slots(self) -> list[str]:
         """
         Retrieves all available slots from Redis.
 
@@ -362,9 +360,9 @@ class ResourceManager(BaseActivity):
             - REDIS_OPERATIONS_DURATION: Records timing for slot discovery
         """
 
-        return self._execute_redis_op("keys", self.redis.keys, "slot:opc_tags:*")
+        return self._execute_redis_op('keys', self.redis.keys, 'slot:opc_tags:*')
 
-    def get_all_leases(self) -> List[str]:
+    def get_all_leases(self) -> list[str]:
         """
         Retrieves all active leases from Redis.
 
@@ -386,4 +384,4 @@ class ResourceManager(BaseActivity):
             - REDIS_OPERATIONS_DURATION: Records timing for lease discovery
         """
 
-        return self._execute_redis_op("keys", self.redis.keys, "lease:opc_tags:*")
+        return self._execute_redis_op('keys', self.redis.keys, 'lease:opc_tags:*')

ingestor/metrics.py

@@ -17,157 +17,157 @@ labels for multi-dimensional analysis and alerting.
 from prometheus_client import Counter, Gauge, Histogram
 
 # Metric label definitions for consistent labeling across all metrics
-POD_ID_LABEL = ["pod_id"]
-SERVER_LABELS = ["pod_id", "server_name", "server_url"]
-KAFKA_LABELS = ["pod_id", "topic"]
-REDIS_LABELS = ["pod_id", "operation"]
-NOTIFICATION_LABELS = ["pod_id", "level", "block"]
+POD_ID_LABEL = ['pod_id']
+SERVER_LABELS = ['pod_id', 'server_name', 'server_url']
+KAFKA_LABELS = ['pod_id', 'topic']
+REDIS_LABELS = ['pod_id', 'operation']
+NOTIFICATION_LABELS = ['pod_id', 'level', 'block']
 
-MAIN_LABELS = ["pod_id"]
+MAIN_LABELS = ['pod_id']
 
 # --- Reliability Metrics ---
 TAG_WRITTEN_COUNT = Counter(
-    "ingestor_tag_written_count",
-    "Number of writing process to the collection",
-    [*MAIN_LABELS, "tag_name", "collection_name"],
+    'ingestor_tag_written_count',
+    'Number of writing process to the collection',
+    [*MAIN_LABELS, 'tag_name', 'collection_name'],
 )
 
 # --- General Application Metrics ---
 APP_LOOP_COUNT = Counter(
-    "app_main_loop_total",
-    "Total number of times the application main loop has run",
+    'app_main_loop_total',
+    'Total number of times the application main loop has run',
     POD_ID_LABEL,
 )
 APP_LOOP_DURATION = Histogram(
-    "app_main_loop_duration_seconds",
-    "Duration of the application main loop in seconds",
+    'app_main_loop_duration_seconds',
+    'Duration of the application main loop in seconds',
     POD_ID_LABEL,
 )
 APP_ERRORS_TOTAL = Counter(
-    "app_errors_total",
-    "Total number of unhandled errors in the main loop",
+    'app_errors_total',
+    'Total number of unhandled errors in the main loop',
     POD_ID_LABEL,
 )
 APP_UP = Gauge(
-    "app_up",
-    "Indicates if the application is running (1) or shutting down (0)",
+    'app_up',
+    'Indicates if the application is running (1) or shutting down (0)',
     POD_ID_LABEL,
 )
 
 # --- Ingestor Manager Metrics ---
 ACTIVE_INGESTORS = Gauge(
-    "ingestor_active_total",
-    "Number of active ingestors reported by Redis",
+    'ingestor_active_total',
+    'Number of active ingestors reported by Redis',
 )
 SLOTS_TOTAL = Gauge(
-    "ingestor_slots_total",
-    "Total number of slots configured in Redis",
+    'ingestor_slots_total',
+    'Total number of slots configured in Redis',
 )
 LEASES_TOTAL = Gauge(
-    "ingestor_leases_total",
-    "Total number of leases (allocated slots) in Redis",
+    'ingestor_leases_total',
+    'Total number of leases (allocated slots) in Redis',
 )
 SLOTS_MANAGED = Gauge(
-    "ingestor_slots_managed_current",
-    "Number of slots currently managed by this ingestor instance",
+    'ingestor_slots_managed_current',
+    'Number of slots currently managed by this ingestor instance',
     POD_ID_LABEL,
 )
 SLOTS_ACQUIRED = Counter(
-    "ingestor_slots_acquired_total",
-    "Total number of slots acquired by this instance",
+    'ingestor_slots_acquired_total',
+    'Total number of slots acquired by this instance',
     POD_ID_LABEL,
 )
 SLOTS_RELEASED = Counter(
-    "ingestor_slots_released_total",
-    "Total number of slots released by this instance",
+    'ingestor_slots_released_total',
+    'Total number of slots released by this instance',
     POD_ID_LABEL,
 )
 OPC_MANAGERS_ACTIVE = Gauge(
-    "ingestor_opc_managers_active",
-    "Number of active OPC Managers in this instance",
+    'ingestor_opc_managers_active',
+    'Number of active OPC Managers in this instance',
     POD_ID_LABEL,
 )
 OPC_SUBSCRIPTION_ERRORS = Counter(
-    "ingestor_opc_subscription_errors_total",
-    "Errors when trying to subscribe to OPC tags",
-    ["pod_id", "server", "slot"],
+    'ingestor_opc_subscription_errors_total',
+    'Errors when trying to subscribe to OPC tags',
+    ['pod_id', 'server', 'slot'],
 )
 
 # --- OPC Manager Metrics ---
 OPC_CONNECTIONS_TOTAL = Counter(
-    "opc_connections_initiated_total",
-    "Total connection attempts to OPC servers",
-    ["pod_id", "server_name"],
+    'opc_connections_initiated_total',
+    'Total connection attempts to OPC servers',
+    ['pod_id', 'server_name'],
 )
 OPC_CONNECTIONS_FAILED = Counter(
-    "opc_connections_failed_total",
-    "Total failed connection attempts to OPC servers",
-    ["pod_id", "server_name"],
+    'opc_connections_failed_total',
+    'Total failed connection attempts to OPC servers',
+    ['pod_id', 'server_name'],
 )
 OPC_CONNECTION_STATUS = Gauge(
-    "opc_connection_status",
-    "Connection status with the OPC server (1=connected, 0=disconnected)",
+    'opc_connection_status',
+    'Connection status with the OPC server (1=connected, 0=disconnected)',
     SERVER_LABELS,
 )
 OPC_SUBSCRIPTIONS_CREATED = Counter(
-    "opc_subscriptions_created_total",
-    "Total OPC subscriptions created",
-    ["pod_id", "server_name", "slot_name"],
+    'opc_subscriptions_created_total',
+    'Total OPC subscriptions created',
+    ['pod_id', 'server_name', 'slot_name'],
 )
 OPC_TAGS_SUBSCRIBED = Gauge(
-    "opc_tags_subscribed_current",
-    "Current number of OPC tags subscribed on a server",
-    ["pod_id", "server_name"],
+    'opc_tags_subscribed_current',
+    'Current number of OPC tags subscribed on a server',
+    ['pod_id', 'server_name'],
 )
 OPC_CYCLES_WITHOUT_DATA = Gauge(
-    "opc_cycles_without_data",
-    "Current number of cycles without receiving data from a server",
-    ["pod_id", "server_name"],
+    'opc_cycles_without_data',
+    'Current number of cycles without receiving data from a server',
+    ['pod_id', 'server_name'],
 )
 OPC_RECONNECTIONS_TOTAL = Counter(
-    "opc_reconnections_tried_total",
-    "Reconnection attempts to an OPC server after a loss",
-    ["pod_id", "server_name"],
+    'opc_reconnections_tried_total',
+    'Reconnection attempts to an OPC server after a loss',
+    ['pod_id', 'server_name'],
 )
 
 # --- Data Manager (Kafka) Metrics ---
 KAFKA_MESSAGES_SENT = Counter(
-    "kafka_messages_sent_total", "Total messages sent to Kafka", KAFKA_LABELS
+    'kafka_messages_sent_total', 'Total messages sent to Kafka', KAFKA_LABELS
 )
 KAFKA_MESSAGES_ERRORS = Counter(
-    "kafka_messages_errors_total",
-    "Total errors sending messages to Kafka",
+    'kafka_messages_errors_total',
+    'Total errors sending messages to Kafka',
     KAFKA_LABELS,
 )
 KAFKA_CONNECTION_STATUS = Gauge(
-    "kafka_connection_status",
-    "Connection status with Kafka (1=connected, 0=disconnected)",
+    'kafka_connection_status',
+    'Connection status with Kafka (1=connected, 0=disconnected)',
     POD_ID_LABEL,
 )
 
 # --- Resource Manager (Redis) Metrics ---
 REDIS_OPERATIONS_TOTAL = Counter(
-    "redis_operations_total", "Total number of Redis operations performed", REDIS_LABELS
+    'redis_operations_total', 'Total number of Redis operations performed', REDIS_LABELS
 )
 REDIS_OPERATIONS_ERRORS = Counter(
-    "redis_operations_errors_total",
-    "Total number of errors in Redis operations",
+    'redis_operations_errors_total',
+    'Total number of errors in Redis operations',
     REDIS_LABELS,
 )
 REDIS_OPERATIONS_DURATION = Histogram(
-    "redis_operations_duration_seconds",
-    "Duration of Redis operations in seconds",
+    'redis_operations_duration_seconds',
+    'Duration of Redis operations in seconds',
     REDIS_LABELS,
 )
 REDIS_CONNECTION_STATUS = Gauge(
-    "redis_connection_status",
-    "Connection status with Redis (1=connected, 0=disconnected)",
+    'redis_connection_status',
+    'Connection status with Redis (1=connected, 0=disconnected)',
     POD_ID_LABEL,
 )
 
 # --- Notification Metrics ---
 NOTIFICATIONS_SENT = Counter(
-    "notifications_sent_total",
-    "Total number of notifications sent",
+    'notifications_sent_total',
+    'Total number of notifications sent',
     NOTIFICATION_LABELS,
 )

pyproject.toml

@@ -0,0 +1,159 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "laborious"
+version = "0.0.0"
+description = "Sientia DataOps Laborious - ML Model Orchestration System"
+readme = "README.md"
+requires-python = ">=3.11"
+authors = [
+    {name = "Aignosi", email = "dev@aignosi.com"}
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+exclude = [
+    ".git",
+    ".venv",
+    "venv",
+    "__pycache__",
+    "*.pyc",
+    ".pytest_cache",
+    "htmlcov",
+]
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # pyflakes
+    "I",   # isort
+    "B",   # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "UP",  # pyupgrade
+    "N",   # pep8-naming
+    "YTT", # flake8-2020
+    "S",   # flake8-bandit
+    "BLE", # flake8-blind-except
+    "A",   # flake8-builtins
+    "C90", # mccabe complexity
+]
+
+ignore = [
+    "BLE001",  # ignore blind except, we need to send notifications with any error
+    "E501",  # line too long (handled by formatter)
+    "S101",  # use of assert (needed for tests)
+    "S105",  # possible hardcoded password (false positives)
+    "S106",  # possible hardcoded password (false positives)
+    "N802",  # function name should be lowercase (temporal decorators)
+    "N806",  # variable in function should be lowercase
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = [
+    "S101",  # assert allowed in tests
+    "S105",  # hardcoded passwords ok in tests
+    "S106",  # hardcoded passwords ok in tests
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 15
+
+[tool.ruff.format]
+quote-style = "single"
+indent-style = "space"
+line-ending = "auto"
+
+[tool.mypy]
+python_version = "3.11"
+warn_return_any = false
+warn_unused_configs = true
+disallow_untyped_defs = false
+disallow_incomplete_defs = false
+check_untyped_defs = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = false
+warn_no_return = true
+strict_equality = true
+ignore_missing_imports = true
+
+# Ignore missing imports for external packages
+[[tool.mypy.overrides]]
+module = "temporalio.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "sientia_do.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "mlflow.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "prometheus_client.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "sientia.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "pandas.*"
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-v",
+    "--strict-markers",
+    "--cov=model_manager",
+    "--cov-report=term-missing",
+    "--cov-report=html",
+    "--cov-report=xml",
+]
+markers = [
+    "asyncio: marks tests as async",
+    "integration: marks tests as integration tests",
+    "unit: marks tests as unit tests",
+]
+
+[tool.coverage.run]
+source = ["model_manager"]
+omit = [
+    "*/tests/*",
+    "*/venv/*",
+    "*/__pycache__/*",
+    "*/site-packages/*",
+]
+branch = true
+
+[tool.coverage.report]
+precision = 2
+show_missing = true
+skip_covered = false
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "def __str__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "class .*\\bProtocol\\):",
+    "@(abc\\.)?abstractmethod",
+]
+
+[tool.coverage.html]
+directory = "htmlcov"
+
+[tool.bandit]
+exclude_dirs = ["tests", "venv", ".venv"]
+skips = ["B101", "B601"]  # Skip assert and shell injection in controlled environments

requirements-dev.txt

@@ -0,0 +1,19 @@
+# Development and Testing Dependencies
+# These packages are only needed for development, testing, and code quality checks
+# Install with: pip install -r requirements-dev.txt
+
+# Code Quality & Linting
+ruff>=0.1.0              # Fast Python linter and formatter (replaces flake8, black, isort)
+mypy>=1.7.0              # Static type checker
+bandit>=1.7.5            # Security vulnerability scanner
+pandas-stubs>=2.0.0      # Type stubs for pandas
+types-requests>=2.31.0   # Type stubs for requests
+
+# Testing
+pytest>=7.4.0            # Testing framework
+pytest-cov>=4.1.0        # Coverage plugin for pytest
+pytest-asyncio>=0.21.0   # Async test support (already in main requirements)
+
+# Development Tools
+ipython>=8.12.0          # Enhanced Python shell
+ipdb>=0.13.13            # IPython debugger

requirements.txt

@@ -1,5 +1,5 @@
 asyncua==1.1.5
 redis
-git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.3
+git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.6
 prometheus_client
 pymongo

tests/unit/managers/test_data_manager.py

@@ -1,33 +1,34 @@
 from unittest.mock import ANY, MagicMock, patch
-from pytest import fixture
+
 from kafka.errors import NoBrokersAvailable
+from pytest import fixture
 from sientia_do.notifications.models import NotificationLevel
+
 from ingestor.managers.data_manager import DataManager
 
 metadata = {
-    "metadata": {
-        "model_id": "test_model",
-        "model_name": "test_model",
-        "workflow_name": "test_workflow",
-        "schema_name": "test_schedule",
-        "pod_id": "localhost",
+    'metadata': {
+        'model_id': 'test_model',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+        'schema_name': 'test_schedule',
+        'pod_id': 'localhost',
     },
 }
 
 
 @fixture
-@patch("ingestor.managers.data_manager.KafkaProducer")
-@patch("ingestor.managers.data_manager.MongoClient")
+@patch('ingestor.managers.data_manager.KafkaProducer')
+@patch('ingestor.managers.data_manager.MongoClient')
 def data_manager(mongo, kafka):
-
     data_manager = DataManager(
-        kafka_servers="localhost:9092",
-        mongo_connection_string="mongodb://localhost:27017",
-        mongo_database="sientia",
+        kafka_servers='localhost:9092',
+        mongo_connection_string='mongodb://localhost:27017',
+        mongo_database='sientia',
         export_to_kafka=True,
         logger=MagicMock(),
         notification_handler=MagicMock(),
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
     )
 
     data_manager.send_notification = MagicMock()
@@ -35,112 +36,106 @@ def data_manager(mongo, kafka):
     return data_manager
 
 
-@patch("ingestor.managers.data_manager.KafkaProducer")
-@patch("ingestor.managers.data_manager.MongoClient")
+@patch('ingestor.managers.data_manager.KafkaProducer')
+@patch('ingestor.managers.data_manager.MongoClient')
 def test___init___success(mongo, kafka):
     logger_mock = MagicMock()
 
     data_manager = DataManager(
-        metadata=metadata["metadata"],
-        kafka_servers="localhost:9092",
-        mongo_connection_string="mongodb://localhost:27017",
-        mongo_database="sientia",
+        metadata=metadata['metadata'],
+        kafka_servers='localhost:9092',
+        mongo_connection_string='mongodb://localhost:27017',
+        mongo_database='sientia',
         export_to_kafka=True,
         logger=logger_mock,
-        notification_handler=MagicMock()
+        notification_handler=MagicMock(),
     )
 
     kafka.assert_called_once_with(
-        bootstrap_servers="localhost:9092",
-        value_serializer=ANY,
-        key_serializer=ANY
+        bootstrap_servers='localhost:9092', value_serializer=ANY, key_serializer=ANY
     )
     assert data_manager.kafka_producer is not None
     logger_mock.info.assert_any_call(
-        "Trying (0) to initializing DataManager with Kafka servers: localhost:9092"
-    )
-    logger_mock.info.assert_any_call(
-        "DataManager initialized with Kafka servers: localhost:9092"
+        'Trying (0) to initializing DataManager with Kafka servers: localhost:9092'
     )
+    logger_mock.info.assert_any_call('DataManager initialized with Kafka servers: localhost:9092')
     logger_mock.error.assert_not_called()
     assert logger_mock.info.call_count == 4
 
 
-@patch("ingestor.managers.data_manager.KafkaProducer")
-@patch("ingestor.managers.data_manager.MongoClient")
+@patch('ingestor.managers.data_manager.KafkaProducer')
+@patch('ingestor.managers.data_manager.MongoClient')
 def test___init___second_attempt(mongo, kafka):
     kafka.side_effect = [NoBrokersAvailable, MagicMock()]
     logger_mock = MagicMock()
 
     data_manager = DataManager(
-        kafka_servers="localhost:9092",
-        mongo_connection_string="mongodb://localhost:27017",
-        mongo_database="sientia",
+        kafka_servers='localhost:9092',
+        mongo_connection_string='mongodb://localhost:27017',
+        mongo_database='sientia',
         export_to_kafka=True,
         logger=logger_mock,
         notification_handler=MagicMock(),
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
     )
 
     kafka.assert_any_call(
-        bootstrap_servers="localhost:9092",
-        value_serializer=ANY,
-        key_serializer=ANY
+        bootstrap_servers='localhost:9092', value_serializer=ANY, key_serializer=ANY
     )
     assert kafka.call_count == 2
     assert data_manager.kafka_producer is not None
     logger_mock.info.assert_any_call(
-        "Trying (0) to initializing DataManager with Kafka servers: localhost:9092"
+        'Trying (0) to initializing DataManager with Kafka servers: localhost:9092'
     )
     logger_mock.info.assert_any_call(
-        "Trying (1) to initializing DataManager with Kafka servers: localhost:9092"
-    )
-    logger_mock.info.assert_any_call(
-        "DataManager initialized with Kafka servers: localhost:9092"
+        'Trying (1) to initializing DataManager with Kafka servers: localhost:9092'
     )
+    logger_mock.info.assert_any_call('DataManager initialized with Kafka servers: localhost:9092')
     logger_mock.error.assert_called_once_with(
-        "Kafka servers localhost:9092 are not available. Retrying..."
+        'Kafka servers localhost:9092 are not available. Retrying...'
     )
     assert logger_mock.info.call_count == 5
 
 
-@patch("ingestor.managers.data_manager.KafkaProducer")
-@patch("ingestor.managers.data_manager.MongoClient")
+@patch('ingestor.managers.data_manager.KafkaProducer')
+@patch('ingestor.managers.data_manager.MongoClient')
 def test___init___failure_max_attempts(mongo, kafka):
     kafka.side_effect = NoBrokersAvailable
     logger_mock = MagicMock()
 
     try:
         DataManager(
-            kafka_servers="localhost:9092",
-            mongo_connection_string="mongodb://localhost:27017",
-            mongo_database="sientia",
+            kafka_servers='localhost:9092',
+            mongo_connection_string='mongodb://localhost:27017',
+            mongo_database='sientia',
             export_to_kafka=True,
             logger=logger_mock,
             notification_handler=MagicMock(),
-            metadata=metadata["metadata"],
+            metadata=metadata['metadata'],
         )
     except NoBrokersAvailable as e:
-        assert str(
-            e) == "NoBrokersAvailable: Failed to connect to Kafka servers localhost:9092 after 3 attempts."
+        assert (
+            str(e)
+            == 'NoBrokersAvailable: Failed to connect to Kafka servers localhost:9092 after 3 attempts.'
+        )
 
         assert kafka.call_count == 3
         logger_mock.info.assert_any_call(
-            "Trying (0) to initializing DataManager with Kafka servers: localhost:9092"
+            'Trying (0) to initializing DataManager with Kafka servers: localhost:9092'
         )
         logger_mock.info.assert_any_call(
-            "Trying (1) to initializing DataManager with Kafka servers: localhost:9092"
+            'Trying (1) to initializing DataManager with Kafka servers: localhost:9092'
         )
         logger_mock.info.assert_any_call(
-            "Trying (2) to initializing DataManager with Kafka servers: localhost:9092"
+            'Trying (2) to initializing DataManager with Kafka servers: localhost:9092'
         )
         logger_mock.error.assert_called_with(
-            "Failed to connect to Kafka servers localhost:9092 after 3 attempts."
+            'Failed to connect to Kafka servers localhost:9092 after 3 attempts.'
         )
         assert logger_mock.info.call_count == 3
 
     else:
-        assert False, "Expected NoBrokersAvailable exception was not raised."
+        assert False, 'Expected NoBrokersAvailable exception was not raised.'
 
 
 def test_shutdown_has_producer(data_manager):
@@ -161,7 +156,7 @@ def test_shutdown_no_producer(data_manager):
     data_manager.shutdown()
 
     data_manager.logger.warning.assert_any_call(
-        "Kafka producer is already closed or not initialized."
+        'Kafka producer is already closed or not initialized.'
     )
 
 
@@ -171,30 +166,24 @@ def test_shutdown_no_mongo_client(data_manager):
     data_manager.shutdown()
 
     data_manager.logger.warning.assert_any_call(
-        "MongoDB client is already closed or not initialized."
+        'MongoDB client is already closed or not initialized.'
     )
 
 
 def test_shutdown_exception(data_manager):
-    data_manager.kafka_producer.flush = MagicMock(
-        side_effect=Exception("Test error"))
+    data_manager.kafka_producer.flush = MagicMock(side_effect=Exception('Test error'))
     data_manager.kafka_producer.close = MagicMock()
 
     data_manager.shutdown()
-    data_manager.logger.error.assert_called_once_with(
-        "Error closing Kafka producer: Test error"
-    )
+    data_manager.logger.error.assert_called_once_with('Error closing Kafka producer: Test error')
 
 
 def test_shutdown_exception_mongo(data_manager):
-    data_manager.mongo_client.close = MagicMock(
-        side_effect=Exception("Test error"))
+    data_manager.mongo_client.close = MagicMock(side_effect=Exception('Test error'))
 
     data_manager.shutdown()
 
-    data_manager.logger.error.assert_called_once_with(
-        "Error closing MongoDB client: Test error"
-    )
+    data_manager.logger.error.assert_called_once_with('Error closing MongoDB client: Test error')
 
 
 def test___del__(data_manager):
@@ -205,29 +194,27 @@ def test___del__(data_manager):
 
 def test_delivery_report(data_manager):
     msg = MagicMock()
-    msg.topic = "test_topic"
+    msg.topic = 'test_topic'
     msg.partition = 0
     msg.offset = 1
 
     data_manager.delivery_report(msg)
 
     data_manager.logger.debug.assert_called_once_with(
-        f"Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}"
+        f'Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}'
     )
 
 
 def test_delivery_error(data_manager):
-    err = "Test error"
+    err = 'Test error'
     data_manager.delivery_error(err)
 
-    data_manager.logger.error.assert_called_once_with(
-        f"Delivery failed for record : {err}"
-    )
+    data_manager.logger.error.assert_called_once_with(f'Delivery failed for record : {err}')
 
 
 def test_publish(data_manager):
-    topic = "test_topic"
-    data = {"key": "value"}
+    topic = 'test_topic'
+    data = {'key': 'value'}
 
     # Mock the send method of the Kafka producer
     send_mock = MagicMock()
@@ -237,9 +224,7 @@ def test_publish(data_manager):
     data_manager.publish(topic, data)
 
     # Check if the send method was called with the correct arguments
-    send_mock.assert_called_once_with(
-        topic=topic, value=data
-    )
+    send_mock.assert_called_once_with(topic=topic, value=data)
 
     send_mock.return_value.add_callback.assert_called_once()
 
@@ -248,57 +233,53 @@ def test_publish(data_manager):
 
 def test_publish_no_kafka(data_manager):
     data_manager.export_to_kafka = False
-    topic = "test_topic"
-    data = {"key": "value"}
+    topic = 'test_topic'
+    data = {'key': 'value'}
 
     data_manager.publish(topic, data)
 
     data_manager.kafka_producer.send.assert_not_called()
 
 
-@patch("ingestor.managers.data_manager.traceback")
+@patch('ingestor.managers.data_manager.traceback')
 def test_publish_error(traceback, data_manager):
-    topic = "test_topic"
-    data = {
-        "key": "value",
-        "name": "test_tag"
-    }
+    topic = 'test_topic'
+    data = {'key': 'value', 'name': 'test_tag'}
 
     # Mock the send method of the Kafka producer to raise an exception
-    send_mock = MagicMock(side_effect=Exception("Test error"))
+    send_mock = MagicMock(side_effect=Exception('Test error'))
     data_manager.kafka_producer.send = send_mock
 
     # Call the publish method
     data_manager.publish(topic, data)
 
     # Check if the send method was called with the correct arguments
-    send_mock.assert_called_once_with(
-        topic=topic, value=data
-    )
+    send_mock.assert_called_once_with(topic=topic, value=data)
 
     # Check if the error was logged
     data_manager.send_notification.assert_called_once_with(
-        notification_id=f"KAFKA_PRODUCER_ERROR_{topic}",
-        message=f"Error publishing message to topic {topic}: Test error",
-        block="kafka_producer",
+        notification_id=f'KAFKA_PRODUCER_ERROR_{topic}',
+        message=f'Error publishing message to topic {topic}: Test error',
+        block='kafka_producer',
         level=NotificationLevel.ERROR,
         attachment_content=traceback.format_exc.return_value,
-        metadata=metadata["metadata"]
+        metadata=metadata['metadata'],
     )
 
 
 def test_publish_error_mongo(data_manager):
     data_manager.export_to_kafka = False
     data_manager.mongo_db.__getitem__.return_value.insert_one = MagicMock(
-        side_effect=Exception("Test error"))
+        side_effect=Exception('Test error')
+    )
 
-    data_manager.publish("test_topic", {"key": "value"})
+    data_manager.publish('test_topic', {'key': 'value'})
 
     data_manager.send_notification.assert_called_once_with(
-        notification_id="MONGO_PRODUCER_ERROR_test_topic",
-        message="Error inserting message to MongoDB: Test error",
-        block="mongo_producer",
+        notification_id='MONGO_PRODUCER_ERROR_test_topic',
+        message='Error inserting message to MongoDB: Test error',
+        block='mongo_producer',
         level=NotificationLevel.ERROR,
         attachment_content=ANY,
-        metadata=metadata["metadata"]
+        metadata=metadata['metadata'],
     )

tests/unit/managers/test_ingestor_manager.py

@@ -1,14 +1,16 @@
 from unittest.mock import AsyncMock, MagicMock, patch
+
 from pytest import fixture, mark
 from sientia_do.notifications.models import NotificationLevel
+
 from ingestor.managers.ingestor_manager import IngestorManager
 
 metadata = {
-    "metadata": {
-        "model_id": "test_model",
-        "model_name": "test_model",
-        "workflow_name": "test_workflow",
-        "schema_name": "test_schedule",
+    'metadata': {
+        'model_id': 'test_model',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+        'schema_name': 'test_schedule',
     },
 }
 
@@ -18,20 +20,17 @@ metadata = {
 @patch('ingestor.managers.ingestor_manager.ResourceManager')
 def ingestor_manager(data_manager_mock, resource_manager_mock):
     ingestor = IngestorManager(
-        kafka_servers="localhost:9092",
-        redis_data={
-            "host": "localhost",
-            "port": 6379
-        },
+        kafka_servers='localhost:9092',
+        redis_data={'host': 'localhost', 'port': 6379},
         lease_ttl=60,
         heartbeat_ttl=60,
         poll_interval=5,
-        mongo_connection_string="mongodb://localhost:27017",
-        mongo_database="sientia",
+        mongo_connection_string='mongodb://localhost:27017',
+        mongo_database='sientia',
         export_to_kafka=False,
         logger=MagicMock(),
         notification_handler=MagicMock(),
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
     )
 
     ingestor.send_notification = MagicMock()
@@ -43,41 +42,39 @@ def ingestor_manager(data_manager_mock, resource_manager_mock):
 @patch('ingestor.managers.ingestor_manager.DataManager')
 @patch('ingestor.managers.ingestor_manager.ResourceManager')
 @patch('ingestor.managers.ingestor_manager.NotificationHandler')
-def test___init__(notification_handler_mock, resource_manager_mock, data_manager_mock, opc_manager_mock):
-
+def test___init__(
+    notification_handler_mock, resource_manager_mock, data_manager_mock, opc_manager_mock
+):
     ingestor = IngestorManager(
-        kafka_servers="localhost:9092",
-        redis_data={
-            "host": "localhost",
-            "port": 6379
-        },
+        kafka_servers='localhost:9092',
+        redis_data={'host': 'localhost', 'port': 6379},
         lease_ttl=60,
         heartbeat_ttl=60,
         poll_interval=5,
-        mongo_connection_string="mongodb://localhost:27017",
-        mongo_database="sientia",
+        mongo_connection_string='mongodb://localhost:27017',
+        mongo_database='sientia',
         export_to_kafka=False,
         logger=MagicMock(),
         notification_handler=MagicMock(),
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
     )
 
     opc_manager_mock.assert_not_called()
     data_manager_mock.assert_called_once_with(
-        kafka_servers="localhost:9092",
-        mongo_connection_string="mongodb://localhost:27017",
-        mongo_database="sientia",
+        kafka_servers='localhost:9092',
+        mongo_connection_string='mongodb://localhost:27017',
+        mongo_database='sientia',
         export_to_kafka=False,
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
         logger=ingestor.logger,
         notification_handler=ingestor.notification_handler,
     )
     resource_manager_mock.assert_called_once_with(
-        host="localhost",
+        host='localhost',
         port=6379,
         lease_ttl=60,
         heartbeat_ttl=60,
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
         logger=ingestor.logger,
         notification_handler=ingestor.notification_handler,
         username=None,
@@ -101,12 +98,11 @@ async def test_initialize_opc_from_config(opc_manager, ingestor_manager):
         'cert_path': '/path/to/cert',
         'private_key_path': '/path/to/private_key',
         'server_cert_path': '/path/to/server_cert',
-        'pod_id': 'test_pod'
+        'pod_id': 'test_pod',
     }
 
     opc_manager.return_value = MagicMock(connect=AsyncMock())
-    result = await ingestor_manager.initialize_opc_from_config(
-        server_config)
+    result = await ingestor_manager.initialize_opc_from_config(server_config)
 
     opc_manager.assert_called_once_with(
         name=server_config['name'],
@@ -118,7 +114,7 @@ async def test_initialize_opc_from_config(opc_manager, ingestor_manager):
         cert_path=server_config['cert_path'],
         private_key_path=server_config['private_key_path'],
         server_cert_path=server_config['server_cert_path'],
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
     )
 
     assert result == opc_manager.return_value
@@ -135,25 +131,24 @@ async def test_initialize_opc_from_config_exception(traceback_mock, opc_manager,
         'server_uri': 'http://opcua-server.simulator',
         'cert_path': '/path/to/cert',
         'private_key_path': '/path/to/private_key',
-        'server_cert_path': '/path/to/server_cert'
+        'server_cert_path': '/path/to/server_cert',
     }
 
     ingestor_manager.logger.error = MagicMock()
-    opc_manager.side_effect = Exception("Initialization error")
+    opc_manager.side_effect = Exception('Initialization error')
 
-    result = await ingestor_manager.initialize_opc_from_config(
-        server_config)
+    result = await ingestor_manager.initialize_opc_from_config(server_config)
 
     assert result is None
 
     traceback_mock.format_exc.assert_called_once()
     ingestor_manager.send_notification.assert_called_once_with(
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
         notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}',
         message='Error initializing OPC manager: Initialization error',
-        block="opc_manager",
+        block='opc_manager',
         level=NotificationLevel.ERROR,
-        attachment_content=traceback_mock.format_exc.return_value
+        attachment_content=traceback_mock.format_exc.return_value,
     )
 
 
@@ -161,46 +156,33 @@ async def test_initialize_opc_from_config_exception(traceback_mock, opc_manager,
 @patch('ingestor.managers.ingestor_manager.OpcManager')
 @patch('ingestor.managers.ingestor_manager.metrics')
 async def test_update_opc_servers(metrics, opc_manager, ingestor_manager):
-    manager1 = MagicMock(
-        config={"config": "config1"}
-    )
-    manager2 = MagicMock(
-        config={"config": "config2"}
-    )
-    manager3 = MagicMock(
-        config={"config": "config3"}
-    )
+    manager1 = MagicMock(config={'config': 'config1'})
+    manager2 = MagicMock(config={'config': 'config2'})
+    manager3 = MagicMock(config={'config': 'config3'})
 
     async def mock_initialize_from_config(config):
-        if config == {"config": "config1"}:
+        if config == {'config': 'config1'}:
             return manager1
-        elif config == {"config": "config2"}:
+        elif config == {'config': 'config2'}:
             return manager2
-        elif config == {"config": "config3"}:
+        elif config == {'config': 'config3'}:
             return manager3
         else:
             return None
 
-    ingestor_manager.initialize_opc_from_config = AsyncMock(
-        side_effect=mock_initialize_from_config
-    )
+    ingestor_manager.initialize_opc_from_config = AsyncMock(side_effect=mock_initialize_from_config)
 
     ingestor_manager.managed_tags = {
-        "slot1": {
-            "server1": {"config": "config1"},
-            "server2": {"config": "config2"},
-            'server5': {"config": "config5"}
+        'slot1': {
+            'server1': {'config': 'config1'},
+            'server2': {'config': 'config2'},
+            'server5': {'config': 'config5'},
         },
-        "slot2": {
-            "server3": {"config": "config3"},
-            "server1": {"config": "config1"}
-        }
+        'slot2': {'server3': {'config': 'config3'}, 'server1': {'config': 'config1'}},
     }
 
-    mock = MagicMock(
-        config={"config": "old_config2"})
-    ingestor_manager.opc_managers['server3'] = AsyncMock(
-        config={"config": "config3"})
+    mock = MagicMock(config={'config': 'old_config2'})
+    ingestor_manager.opc_managers['server3'] = AsyncMock(config={'config': 'config3'})
     ingestor_manager.opc_managers['server2'] = mock
     ingestor_manager.opc_managers['server4'] = AsyncMock()
 
@@ -208,29 +190,23 @@ async def test_update_opc_servers(metrics, opc_manager, ingestor_manager):
 
     assert len(ingestor_manager.opc_managers) == 3
 
-    ingestor_manager.initialize_opc_from_config.assert_any_call(
-        {"config": "config1"})
-    ingestor_manager.initialize_opc_from_config.assert_any_call(
-        {"config": "config2"})
-    ingestor_manager.initialize_opc_from_config.assert_any_call(
-        {"config": "config5"})
+    ingestor_manager.initialize_opc_from_config.assert_any_call({'config': 'config1'})
+    ingestor_manager.initialize_opc_from_config.assert_any_call({'config': 'config2'})
+    ingestor_manager.initialize_opc_from_config.assert_any_call({'config': 'config5'})
     assert ingestor_manager.initialize_opc_from_config.call_count == 3
 
-    assert ingestor_manager.opc_managers['server1'].config == {
-        "config": "config1"}
-    assert ingestor_manager.opc_managers['server2'].config == {
-        "config": "config2"}
-    assert ingestor_manager.opc_managers['server3'].config == {
-        "config": "config3"}
+    assert ingestor_manager.opc_managers['server1'].config == {'config': 'config1'}
+    assert ingestor_manager.opc_managers['server2'].config == {'config': 'config2'}
+    assert ingestor_manager.opc_managers['server3'].config == {'config': 'config3'}
     assert 'server4' not in ingestor_manager.opc_managers
     assert 'server5' not in ingestor_manager.opc_managers
 
     assert ingestor_manager.opc_managers['server2'] != mock
 
-    metrics.OPC_MANAGERS_ACTIVE.labels.assert_called_once_with(
-        pod_id=ingestor_manager.pod_id)
+    metrics.OPC_MANAGERS_ACTIVE.labels.assert_called_once_with(pod_id=ingestor_manager.pod_id)
     metrics.OPC_MANAGERS_ACTIVE.labels.return_value.set.assert_called_once_with(
-        len(ingestor_manager.opc_managers))
+        len(ingestor_manager.opc_managers)
+    )
 
 
 def test_declare_active(ingestor_manager):
@@ -246,8 +222,7 @@ def test_get_active_ingestors(ingestor_manager):
 
 
 def test_get_active_ingestors_empty(ingestor_manager):
-    ingestor_manager.resource_manager.get_all_ingestors = MagicMock(
-        return_value=None)
+    ingestor_manager.resource_manager.get_all_ingestors = MagicMock(return_value=None)
     result = ingestor_manager.get_active_ingestors()
     assert result == []
     ingestor_manager.resource_manager.get_all_ingestors.assert_called_once()
@@ -255,8 +230,7 @@ def test_get_active_ingestors_empty(ingestor_manager):
 
 @patch('ingestor.managers.ingestor_manager.metrics')
 def test_get_number_of_leases_success(metrics, ingestor_manager):
-    ingestor_manager.resource_manager.get_all_leases = MagicMock(
-        return_value=["lease1", "lease2"])
+    ingestor_manager.resource_manager.get_all_leases = MagicMock(return_value=['lease1', 'lease2'])
     result = ingestor_manager.get_number_of_leases()
     assert result == 2
     ingestor_manager.resource_manager.get_all_leases.assert_called_once()
@@ -265,8 +239,7 @@ def test_get_number_of_leases_success(metrics, ingestor_manager):
 
 @patch('ingestor.managers.ingestor_manager.metrics')
 def test_get_number_of_leases_empty(metrics, ingestor_manager):
-    ingestor_manager.resource_manager.get_all_leases = MagicMock(
-        return_value=None)
+    ingestor_manager.resource_manager.get_all_leases = MagicMock(return_value=None)
     result = ingestor_manager.get_number_of_leases()
     assert result == 0
     ingestor_manager.resource_manager.get_all_leases.assert_called_once()
@@ -275,8 +248,7 @@ def test_get_number_of_leases_empty(metrics, ingestor_manager):
 
 @patch('ingestor.managers.ingestor_manager.metrics')
 def test_get_number_of_slots_success(metrics, ingestor_manager):
-    ingestor_manager.resource_manager.get_all_slots = MagicMock(
-        return_value=["slot1", "slot2"])
+    ingestor_manager.resource_manager.get_all_slots = MagicMock(return_value=['slot1', 'slot2'])
     result = ingestor_manager.get_number_of_slots()
     assert result == 2
     ingestor_manager.resource_manager.get_all_slots.assert_called_once()
@@ -285,8 +257,7 @@ def test_get_number_of_slots_success(metrics, ingestor_manager):
 
 @patch('ingestor.managers.ingestor_manager.metrics')
 def test_get_number_of_slots_empty(metrics, ingestor_manager):
-    ingestor_manager.resource_manager.get_all_slots = MagicMock(
-        return_value=None)
+    ingestor_manager.resource_manager.get_all_slots = MagicMock(return_value=None)
     result = ingestor_manager.get_number_of_slots()
     assert result == 0
     ingestor_manager.resource_manager.get_all_slots.assert_called_once()
@@ -294,39 +265,32 @@ def test_get_number_of_slots_empty(metrics, ingestor_manager):
 
 
 def test_get_slot_leases_1_success(ingestor_manager):
-    ingestor_manager.resource_manager.lease_tag = MagicMock(
-        return_value=True)
-    ingestor_manager.resource_manager.get_tag_slot = MagicMock(
-        return_value={"tags": ["tag1"]})
+    ingestor_manager.resource_manager.lease_tag = MagicMock(return_value=True)
+    ingestor_manager.resource_manager.get_tag_slot = MagicMock(return_value={'tags': ['tag1']})
 
     ingestor_manager.number_of_slots = 1
     result = ingestor_manager.get_slot_leases()
 
-    assert result == {
-        "1": {"tags": ["tag1"]}
-    }
+    assert result == {'1': {'tags': ['tag1']}}
 
 
 def test_get_slot_leases_2_success(ingestor_manager):
-    ingestor_manager.resource_manager.lease_tag = MagicMock(
-        side_effect=[True, True])
+    ingestor_manager.resource_manager.lease_tag = MagicMock(side_effect=[True, True])
     ingestor_manager.resource_manager.get_tag_slot = MagicMock(
-        side_effect=[{"tags": ["tag1"]}, {"tags": ["tag2"]}])
+        side_effect=[{'tags': ['tag1']}, {'tags': ['tag2']}]
+    )
 
     ingestor_manager.number_of_slots = 2
     result = ingestor_manager.get_slot_leases(max_slots=2)
 
-    assert result == {
-        "1": {"tags": ["tag1"]},
-        "2": {"tags": ["tag2"]}
-    }
+    assert result == {'1': {'tags': ['tag1']}, '2': {'tags': ['tag2']}}
 
 
 def test_get_slot_leases_2_1_none(ingestor_manager):
-    ingestor_manager.resource_manager.lease_tag = MagicMock(
-        side_effect=[True, True])
+    ingestor_manager.resource_manager.lease_tag = MagicMock(side_effect=[True, True])
     ingestor_manager.resource_manager.get_tag_slot = MagicMock(
-        side_effect=[None, {"tags": ["tag1"]}])
+        side_effect=[None, {'tags': ['tag1']}]
+    )
 
     ingestor_manager.number_of_slots = 1
     result = ingestor_manager.get_slot_leases(max_slots=1)
@@ -335,10 +299,8 @@ def test_get_slot_leases_2_1_none(ingestor_manager):
 
 
 def test_get_slot_leases_1_failure(ingestor_manager):
-    ingestor_manager.resource_manager.lease_tag = MagicMock(
-        return_value=False)
-    ingestor_manager.resource_manager.get_tag_slot = MagicMock(
-        return_value={"tags": ["tag1"]})
+    ingestor_manager.resource_manager.lease_tag = MagicMock(return_value=False)
+    ingestor_manager.resource_manager.get_tag_slot = MagicMock(return_value={'tags': ['tag1']})
 
     result = ingestor_manager.get_slot_leases()
     ingestor_manager.resource_manager.get_tag_slot.assert_not_called()
@@ -349,42 +311,30 @@ def test_get_slot_leases_1_failure(ingestor_manager):
 @mark.asyncio
 async def test_unsubscribe_slot(ingestor_manager):
     ingestor_manager.managed_tags = {
-        "slot1": {
-            "server1": {"tags": "config1"},
-            "server2": {"tags": "config2"}
-        },
-        "slot2": {
-            "server3": {"tags": "config3"},
-            "server1": {"tags": "config1"}
-        }
+        'slot1': {'server1': {'tags': 'config1'}, 'server2': {'tags': 'config2'}},
+        'slot2': {'server3': {'tags': 'config3'}, 'server1': {'tags': 'config1'}},
     }
     ingestor_manager.opc_managers = {
-        "server1": AsyncMock(),
-        "server2": AsyncMock(),
-        "server3": AsyncMock()
+        'server1': AsyncMock(),
+        'server2': AsyncMock(),
+        'server3': AsyncMock(),
     }
-    await ingestor_manager.unsubscribe_slot("slot1")
+    await ingestor_manager.unsubscribe_slot('slot1')
 
-    ingestor_manager.opc_managers["server1"].unsubscribe.assert_called_once_with(
-        "slot1")
-    ingestor_manager.opc_managers["server2"].unsubscribe.assert_called_once_with(
-        "slot1")
-    ingestor_manager.opc_managers["server3"].unsubscribe.assert_not_called()
+    ingestor_manager.opc_managers['server1'].unsubscribe.assert_called_once_with('slot1')
+    ingestor_manager.opc_managers['server2'].unsubscribe.assert_called_once_with('slot1')
+    ingestor_manager.opc_managers['server3'].unsubscribe.assert_not_called()
 
 
 def test_update_slot_config(ingestor_manager):
     ingestor_manager.managed_tags = {
-        "slot1": {"config": "old_config"},
-        "slot2": {"config": "new_config"},
-        "slot3": {"config": "old_config"}
+        'slot1': {'config': 'old_config'},
+        'slot2': {'config': 'new_config'},
+        'slot3': {'config': 'old_config'},
     }
 
     ingestor_manager.resource_manager.get_tag_slot = MagicMock(
-        side_effect=[
-            {"config": "updated_config"},
-            {"config": "new_config"},
-            None
-        ]
+        side_effect=[{'config': 'updated_config'}, {'config': 'new_config'}, None]
     )
 
     ingestor_manager.update_opc_servers = MagicMock()
@@ -393,179 +343,130 @@ def test_update_slot_config(ingestor_manager):
 
     ingestor_manager.update_slot_config()
 
-    assert ingestor_manager.managed_tags["slot1"] == {
-        "config": "updated_config"}
-    assert ingestor_manager.managed_tags["slot2"] == {
-        "config": "new_config"}
-    assert "slot3" not in ingestor_manager.managed_tags
+    assert ingestor_manager.managed_tags['slot1'] == {'config': 'updated_config'}
+    assert ingestor_manager.managed_tags['slot2'] == {'config': 'new_config'}
+    assert 'slot3' not in ingestor_manager.managed_tags
 
-    ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot1")
-    ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot3")
-    ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot2")
+    ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot1')
+    ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot3')
+    ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot2')
     assert ingestor_manager.resource_manager.renew_tag_lease.call_count == 3
 
 
 @patch('ingestor.managers.ingestor_manager.metrics')
 def test_drop_slot_leases(metrics, ingestor_manager):
     ingestor_manager.resource_manager.drop_tag_lease = MagicMock()
-    ingestor_manager.drop_slot_leases(["1", "2"])
+    ingestor_manager.drop_slot_leases(['1', '2'])
 
-    ingestor_manager.resource_manager.drop_tag_lease.assert_any_call("1")
-    ingestor_manager.resource_manager.drop_tag_lease.assert_any_call("2")
+    ingestor_manager.resource_manager.drop_tag_lease.assert_any_call('1')
+    ingestor_manager.resource_manager.drop_tag_lease.assert_any_call('2')
 
-    metrics.SLOTS_RELEASED.labels.assert_any_call(
-        pod_id=ingestor_manager.pod_id)
+    metrics.SLOTS_RELEASED.labels.assert_any_call(pod_id=ingestor_manager.pod_id)
     metrics.SLOTS_RELEASED.labels.return_value.inc.assert_any_call()
 
 
 @mark.asyncio
 async def test_manage_server_no_server(ingestor_manager):
-    ingestor_manager.opc_managers = {
-        "server1": MagicMock(),
-        "server2": MagicMock()
-    }
-    server_config = {
-        'tags': 'config1'
-    }
+    ingestor_manager.opc_managers = {'server1': MagicMock(), 'server2': MagicMock()}
+    server_config = {'tags': 'config1'}
 
-    result = await ingestor_manager.manage_server(
-        'slot1', 'server3', server_config, server_config)
+    result = await ingestor_manager.manage_server('slot1', 'server3', server_config, server_config)
 
     assert result == 1
-    ingestor_manager.opc_managers["server1"].create_subscription.assert_not_called(
-    )
-    ingestor_manager.opc_managers["server1"].subscribe.assert_not_called()
+    ingestor_manager.opc_managers['server1'].create_subscription.assert_not_called()
+    ingestor_manager.opc_managers['server1'].subscribe.assert_not_called()
 
 
 @mark.asyncio
 async def test_manage_server_create_subscription_failure(ingestor_manager):
-    ingestor_manager.opc_managers = {
-        "server1": MagicMock(),
-        "server2": MagicMock()
-    }
-    ingestor_manager.subscriptions = {
-        "server1": MagicMock()
-    }
-    server_config = {
-        'tags': 'config1'
-    }
+    ingestor_manager.opc_managers = {'server1': MagicMock(), 'server2': MagicMock()}
+    ingestor_manager.subscriptions = {'server1': MagicMock()}
+    server_config = {'tags': 'config1'}
 
-    ingestor_manager.opc_managers["server1"].create_subscription.side_effect = Exception(
-        "Subscription error")
+    ingestor_manager.opc_managers['server1'].create_subscription.side_effect = Exception(
+        'Subscription error'
+    )
 
-    result = await ingestor_manager.manage_server(
-        'slot1', 'server1', server_config, server_config)
+    result = await ingestor_manager.manage_server('slot1', 'server1', server_config, server_config)
 
     assert result == 2
-    ingestor_manager.opc_managers["server1"].create_subscription.assert_called_once_with(
-        'slot1')
-    ingestor_manager.opc_managers["server1"].subscribe.assert_not_called()
+    ingestor_manager.opc_managers['server1'].create_subscription.assert_called_once_with('slot1')
+    ingestor_manager.opc_managers['server1'].subscribe.assert_not_called()
 
 
 @mark.asyncio
 async def test_manage_server(ingestor_manager):
-    ingestor_manager.opc_managers = {
-        "server1": AsyncMock(),
-        "server2": AsyncMock()
-    }
-    ingestor_manager.subscriptions = {
-        "server1": AsyncMock()
-    }
-    server_config = {
-        'tags': 'config1'
-    }
+    ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()}
+    ingestor_manager.subscriptions = {'server1': AsyncMock()}
+    server_config = {'tags': 'config1'}
 
-    result = await ingestor_manager.manage_server(
-        'slot1', 'server1', server_config, server_config)
+    result = await ingestor_manager.manage_server('slot1', 'server1', server_config, server_config)
 
     assert result == 0
-    ingestor_manager.opc_managers["server1"].create_subscription.assert_called_once_with(
-        'slot1')
-    ingestor_manager.opc_managers["server1"].subscribe.assert_called_once_with(
-        'slot1', 'config1', ingestor_manager.poll_interval)
+    ingestor_manager.opc_managers['server1'].create_subscription.assert_called_once_with('slot1')
+    ingestor_manager.opc_managers['server1'].subscribe.assert_called_once_with(
+        'slot1', 'config1', ingestor_manager.poll_interval
+    )
 
 
 @patch('ingestor.managers.ingestor_manager.traceback')
 @mark.asyncio
 async def test_manage_server_subscribe_failure(traceback_mock, ingestor_manager):
-    ingestor_manager.opc_managers = {
-        "server1": AsyncMock(),
-        "server2": AsyncMock()
-    }
-    ingestor_manager.subscriptions = {
-        "server1": AsyncMock()
-    }
-    server_config = {
-        'tags': 'config1'
-    }
+    ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()}
+    ingestor_manager.subscriptions = {'server1': AsyncMock()}
+    server_config = {'tags': 'config1'}
 
-    ingestor_manager.opc_managers["server1"].subscribe.side_effect = Exception(
-        "Subscription error")
+    ingestor_manager.opc_managers['server1'].subscribe.side_effect = Exception('Subscription error')
 
-    result = await ingestor_manager.manage_server(
-        'slot1', 'server1', server_config, server_config)
+    result = await ingestor_manager.manage_server('slot1', 'server1', server_config, server_config)
 
     assert result == 2
-    ingestor_manager.opc_managers["server1"].create_subscription.assert_called_once_with(
-        'slot1')
-    ingestor_manager.opc_managers["server1"].subscribe.assert_called_once_with(
-        'slot1', 'config1', ingestor_manager.poll_interval)
-    ingestor_manager.opc_managers["server1"].unsubscribe.assert_called_once_with(
-        'slot1')
+    ingestor_manager.opc_managers['server1'].create_subscription.assert_called_once_with('slot1')
+    ingestor_manager.opc_managers['server1'].subscribe.assert_called_once_with(
+        'slot1', 'config1', ingestor_manager.poll_interval
+    )
+    ingestor_manager.opc_managers['server1'].unsubscribe.assert_called_once_with('slot1')
 
     traceback_mock.format_exc.assert_called_once()
 
     ingestor_manager.send_notification.assert_called_once_with(
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
         notification_id='OPC_SUBSCRIPTION_ERROR_slot1:server1',
-        message='Failed to subscribe to tags from slot1:server1\n{\'tags\': \'config1\'}: Subscription error',
-        block="opc_manager",
+        message="Failed to subscribe to tags from slot1:server1\n{'tags': 'config1'}: Subscription error",
+        block='opc_manager',
         level=NotificationLevel.ERROR,
-        attachment_content=traceback_mock.format_exc.return_value
+        attachment_content=traceback_mock.format_exc.return_value,
     )
 
     ingestor_manager.logger.warning.assert_any_call(
-        "Removing subscription from server server1 for slot slot1"
+        'Removing subscription from server server1 for slot slot1'
     )
 
 
 @mark.asyncio
 async def test_subscribe_to_tags(ingestor_manager):
-    ingestor_manager.manage_server = AsyncMock(
-        side_effect=[0, 1, 2])
-    ingestor_manager.managed_tags = {
-        "slot1": MagicMock(),
-        "slot2": MagicMock()
-    }
+    ingestor_manager.manage_server = AsyncMock(side_effect=[0, 1, 2])
+    ingestor_manager.managed_tags = {'slot1': MagicMock(), 'slot2': MagicMock()}
 
-    ingestor_manager.opc_managers = {
-        "server1": AsyncMock(),
-        "server2": AsyncMock()
-    }
-    ingestor_manager.subscriptions = {
-        "server1": AsyncMock()
-    }
+    ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()}
+    ingestor_manager.subscriptions = {'server1': AsyncMock()}
     tags = {
         'slot1': {
-            "server1": {"tags": "config1"},
-            "server2": {"tags": "config2"},
-            'server3': {"tags": "config3"},
+            'server1': {'tags': 'config1'},
+            'server2': {'tags': 'config2'},
+            'server3': {'tags': 'config3'},
         }
     }
 
     await ingestor_manager.subscribe_to_tags(tags)
 
-    ingestor_manager.manage_server.assert_any_call(
-        'slot1', 'server1', {"tags": "config1"}, tags)
-    ingestor_manager.manage_server.assert_any_call(
-        'slot1', 'server2', {"tags": "config2"}, tags)
-    ingestor_manager.manage_server.assert_any_call(
-        'slot1', 'server3', {"tags": "config3"}, tags)
+    ingestor_manager.manage_server.assert_any_call('slot1', 'server1', {'tags': 'config1'}, tags)
+    ingestor_manager.manage_server.assert_any_call('slot1', 'server2', {'tags': 'config2'}, tags)
+    ingestor_manager.manage_server.assert_any_call('slot1', 'server3', {'tags': 'config3'}, tags)
 
     assert ingestor_manager.manage_server.call_count == 3
 
-    ingestor_manager.managed_tags['slot1'].pop.assert_called_once_with(
-        'server3', None)
+    ingestor_manager.managed_tags['slot1'].pop.assert_called_once_with('server3', None)
 
 
 @patch('ingestor.managers.ingestor_manager.metrics')
@@ -574,17 +475,14 @@ def test_check_opc_servers_integrity_all_healthy(metrics, ingestor_manager):
     opc_manager1 = MagicMock()
     opc_manager1.check_cycles.return_value = None
     opc_manager1.check_opc_listenning.return_value = False
-    opc_manager1.config = {"config": "config1"}
+    opc_manager1.config = {'config': 'config1'}
 
     opc_manager2 = MagicMock()
     opc_manager2.check_cycles.return_value = None
     opc_manager2.check_opc_listenning.return_value = False
-    opc_manager2.config = {"config": "config2"}
+    opc_manager2.config = {'config': 'config2'}
 
-    ingestor_manager.opc_managers = {
-        "server1": opc_manager1,
-        "server2": opc_manager2
-    }
+    ingestor_manager.opc_managers = {'server1': opc_manager1, 'server2': opc_manager2}
 
     # Mock the initialize_opc_from_config method
     ingestor_manager.initialize_opc_from_config = MagicMock()
@@ -601,10 +499,10 @@ def test_check_opc_servers_integrity_all_healthy(metrics, ingestor_manager):
     # Verify that no reinitialization was needed
     ingestor_manager.initialize_opc_from_config.assert_not_called()
 
-    metrics.OPC_MANAGERS_ACTIVE.labels.assert_called_once_with(
-        pod_id=ingestor_manager.pod_id)
+    metrics.OPC_MANAGERS_ACTIVE.labels.assert_called_once_with(pod_id=ingestor_manager.pod_id)
     metrics.OPC_MANAGERS_ACTIVE.labels.return_value.set.assert_called_once_with(
-        len(ingestor_manager.opc_managers))
+        len(ingestor_manager.opc_managers)
+    )
 
 
 def test_check_opc_servers_integrity_server_lost(ingestor_manager):
@@ -612,16 +510,13 @@ def test_check_opc_servers_integrity_server_lost(ingestor_manager):
     opc_manager = MagicMock()
     opc_manager.check_cycles.return_value = None
     opc_manager.check_opc_listenning.return_value = True  # Server is lost
-    opc_manager.config = {"config": "config1"}
+    opc_manager.config = {'config': 'config1'}
 
-    ingestor_manager.opc_managers = {
-        "server1": opc_manager
-    }
+    ingestor_manager.opc_managers = {'server1': opc_manager}
 
     # Mock the initialize_opc_from_config method to return a new manager
     new_manager = MagicMock()
-    ingestor_manager.initialize_opc_from_config = MagicMock(
-        return_value=new_manager)
+    ingestor_manager.initialize_opc_from_config = MagicMock(return_value=new_manager)
 
     # Call the method
     ingestor_manager.check_opc_servers_integrity()
@@ -632,26 +527,18 @@ def test_check_opc_servers_integrity_server_lost_with_tags(ingestor_manager):
     opc_manager = MagicMock()
     opc_manager.check_cycles.return_value = None
     opc_manager.check_opc_listenning.return_value = True  # Server is lost
-    opc_manager.config = {"config": "config1"}
+    opc_manager.config = {'config': 'config1'}
 
-    ingestor_manager.opc_managers = {
-        "server1": opc_manager
-    }
+    ingestor_manager.opc_managers = {'server1': opc_manager}
 
     # Setup managed tags
     ingestor_manager.managed_tags = {
-        "slot1": {
-            "server1": {
-                "config": "config1",
-                "tags": {"tag1": "value1"}
-            }
-        }
+        'slot1': {'server1': {'config': 'config1', 'tags': {'tag1': 'value1'}}}
     }
 
     # Mock the initialize_opc_from_config method to return a new manager
     new_manager = MagicMock()
-    ingestor_manager.initialize_opc_from_config = MagicMock(
-        return_value=new_manager)
+    ingestor_manager.initialize_opc_from_config = MagicMock(return_value=new_manager)
 
     # Call the method
     ingestor_manager.check_opc_servers_integrity()

tests/unit/managers/test_opc_manager.py

@@ -1,66 +1,68 @@
 import json
 from datetime import datetime
 from unittest.mock import AsyncMock, MagicMock, call, patch
-from pytest import fixture, mark
-from asyncua.crypto.security_policies import SecurityPolicyBasic256
+
 import pytest
-from ingestor.managers.opc_manager import OpcManager
+from asyncua.crypto.security_policies import SecurityPolicyBasic256
+from pytest import fixture, mark
 from sientia_do.notifications.models import NotificationLevel
 
+from ingestor.managers.opc_manager import OpcManager
+
 tags = {
-    "ns=3;i=1001": {
-        "aggregation_function": "LTS",
-        "frequency": 1000,
-        "max_value": 100,
-        "min_value": 0,
-        "tag_name": "Counter",
+    'ns=3;i=1001': {
+        'aggregation_function': 'LTS',
+        'frequency': 1000,
+        'max_value': 100,
+        'min_value': 0,
+        'tag_name': 'Counter',
     },
-    "ns=3;i=1003": {
-        "aggregation_function": "AVG",
-        "frequency": 1000,
-        "max_value": 100,
-        "min_value": 0,
-        "tag_name": "Random",
+    'ns=3;i=1003': {
+        'aggregation_function': 'AVG',
+        'frequency': 1000,
+        'max_value': 100,
+        'min_value': 0,
+        'tag_name': 'Random',
     },
-    "ns=3;i=1004": {
-        "aggregation_function": "MDN",
-        "frequency": 1000,
-        "max_value": 100,
-        "min_value": 0,
-        "tag_name": "Sawtooth",
+    'ns=3;i=1004': {
+        'aggregation_function': 'MDN',
+        'frequency': 1000,
+        'max_value': 100,
+        'min_value': 0,
+        'tag_name': 'Sawtooth',
     },
 }
 
 metadata = {
-    "metadata": {
-        "model_id": "test_model",
-        "model_name": "test_model",
-        "workflow_name": "test_workflow",
-        "schema_name": "test_schedule",
+    'metadata': {
+        'model_id': 'test_model',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+        'schema_name': 'test_schedule',
     },
 }
 
 
 @fixture
-@patch("ingestor.managers.opc_manager.metrics")
+@patch('ingestor.managers.opc_manager.metrics')
 def raw_opc_manager(mock_metrics):
     return OpcManager(
-        name="TestConnector",
-        url="opc.tcp://localhost:4840",
+        name='TestConnector',
+        url='opc.tcp://localhost:4840',
         data_manager=MagicMock(),
         logger=MagicMock(),
-        server_uri="opc.tcp://localhost:4840",
+        server_uri='opc.tcp://localhost:4840',
         notification_handler=MagicMock(),
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
     )
 
 
 @fixture
 def opc_manager(raw_opc_manager):
     raw_opc_manager.client = AsyncMock()
-    raw_opc_manager.cert_path = "cert.pem"
-    raw_opc_manager.private_key_path = "private_key.pem"
-    raw_opc_manager.server_cert_path = "server_cert.pem"
+    raw_opc_manager.cert_path = 'cert.pem'
+    raw_opc_manager.private_key_path = 'private_key.pem'
+    raw_opc_manager.server_cert_path = 'server_cert.pem'
     raw_opc_manager.send_notification = MagicMock()
 
     return raw_opc_manager
@@ -68,7 +70,7 @@ def opc_manager(raw_opc_manager):
 
 @fixture
 def opc_manager_subscribed(opc_manager):
-    opc_manager.subscriptions["sub1"] = AsyncMock()
+    opc_manager.subscriptions['sub1'] = AsyncMock()
 
     return opc_manager
 
@@ -76,7 +78,7 @@ def opc_manager_subscribed(opc_manager):
 def test___str__(opc_manager):
     assert (
         str(opc_manager)
-        == "OpcManager(name=TestConnector, url=opc.tcp://localhost:4840, server_uri=opc.tcp://localhost:4840)\nnodes={}, subscriptions={}"
+        == 'OpcManager(name=TestConnector, url=opc.tcp://localhost:4840, server_uri=opc.tcp://localhost:4840)\nnodes={}, subscriptions={}'
     )
 
 
@@ -91,21 +93,18 @@ async def test_shutdown_success(opc_manager):
 
 @mark.asyncio
 async def test_shutdown_error(opc_manager):
-    opc_manager.disconnect = AsyncMock(side_effect=Exception("Test error"))
+    opc_manager.disconnect = AsyncMock(side_effect=Exception('Test error'))
 
     await opc_manager.shutdown()
 
-    opc_manager.logger.error.assert_called_once_with(
-        "Error during cleanup: Test error")
+    opc_manager.logger.error.assert_called_once_with('Error during cleanup: Test error')
 
 
 @mark.asyncio
 async def test_set_security_success(opc_manager):
     await opc_manager.set_security()
 
-    opc_manager.client.set_application_uri.assert_called_once_with(
-        opc_manager.server_uri
-    )
+    opc_manager.client.set_application_uri.assert_called_once_with(opc_manager.server_uri)
 
     opc_manager.client.set_security.assert_called_once_with(
         SecurityPolicyBasic256,
@@ -114,8 +113,7 @@ async def test_set_security_success(opc_manager):
         server_certificate=opc_manager.server_cert_path,
     )
 
-    opc_manager.client.set_secure_channel_timeout.assert_called_once_with(
-        10000000)
+    opc_manager.client.set_secure_channel_timeout.assert_called_once_with(10000000)
     opc_manager.client.set_session_timeout.assert_called_once_with(10000000)
 
 
@@ -127,19 +125,16 @@ async def test_set_security_no_cert(opc_manager):
     try:
         await opc_manager.set_security()
     except ValueError as e:
-        assert (
-            str(e)
-            == "Certificate and private key paths must be provided for secure connection."
-        )
+        assert str(e) == 'Certificate and private key paths must be provided for secure connection.'
     else:
-        assert False, "ValueError not raised"
+        assert False, 'ValueError not raised'
 
     assert opc_manager.client.set_security.call_count == 0
 
 
 @mark.asyncio
-@patch("ingestor.managers.opc_manager.metrics")
-@patch("ingestor.managers.opc_manager.Client")
+@patch('ingestor.managers.opc_manager.metrics')
+@patch('ingestor.managers.opc_manager.Client')
 async def test_connect_no_security(client, mock_metrics, raw_opc_manager):
     raw_opc_manager.set_security = AsyncMock()
     client.return_value = AsyncMock()
@@ -150,26 +145,24 @@ async def test_connect_no_security(client, mock_metrics, raw_opc_manager):
     raw_opc_manager.client.connect.assert_called_once()
     raw_opc_manager.set_security.assert_not_called()
     mock_metrics.OPC_CONNECTIONS_TOTAL.labels.assert_called_once_with(
-        pod_id=raw_opc_manager.pod_id,
-        server_name=raw_opc_manager.name
+        pod_id=raw_opc_manager.pod_id, server_name=raw_opc_manager.name
     )
     mock_metrics.OPC_CONNECTIONS_TOTAL.labels.return_value.inc.assert_called_once()
     mock_metrics.OPC_CONNECTION_STATUS.labels.assert_called_once_with(
         pod_id=raw_opc_manager.pod_id,
         server_name=raw_opc_manager.name,
-        server_url=raw_opc_manager.url
+        server_url=raw_opc_manager.url,
     )
-    mock_metrics.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(
-        1)
+    mock_metrics.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(1)
     mock_metrics.OPC_CONNECTIONS_FAILED.labels.assert_not_called()
 
 
 @mark.asyncio
-@patch("ingestor.managers.opc_manager.Client")
+@patch('ingestor.managers.opc_manager.Client')
 async def test_connect_with_security(client, raw_opc_manager):
-    raw_opc_manager.cert_path = "cert.pem"
-    raw_opc_manager.private_key_path = "private_key.pem"
-    raw_opc_manager.server_cert_path = "server_cert.pem"
+    raw_opc_manager.cert_path = 'cert.pem'
+    raw_opc_manager.private_key_path = 'private_key.pem'
+    raw_opc_manager.server_cert_path = 'server_cert.pem'
     raw_opc_manager.set_security = AsyncMock()
     client.return_value = AsyncMock()
 
@@ -181,15 +174,14 @@ async def test_connect_with_security(client, raw_opc_manager):
 
 
 @mark.asyncio
-@patch("ingestor.managers.opc_manager.Client")
-@patch("ingestor.managers.opc_manager.metrics")
+@patch('ingestor.managers.opc_manager.Client')
+@patch('ingestor.managers.opc_manager.metrics')
 async def test_connect_exception_handling_and_metrics(
     mock_metrics_module, mock_opc_client_class, raw_opc_manager
 ):
     mock_client_instance = mock_opc_client_class.return_value
-    simulated_error_message = "Erro de conexão simulado"
-    mock_client_instance.connect.side_effect = Exception(
-        simulated_error_message)
+    simulated_error_message = 'Erro de conexão simulado'
+    mock_client_instance.connect.side_effect = Exception(simulated_error_message)
 
     opc_manager_instance = raw_opc_manager
     opc_manager_instance.cert_path = None
@@ -207,9 +199,7 @@ async def test_connect_exception_handling_and_metrics(
         server_name=opc_manager_instance.name,
         server_url=opc_manager_instance.url,
     )
-    mock_metrics_module.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(
-        0
-    )
+    mock_metrics_module.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(0)
 
     mock_metrics_module.OPC_CONNECTIONS_FAILED.labels.assert_called_once_with(
         pod_id=opc_manager_instance.pod_id, server_name=opc_manager_instance.name
@@ -217,50 +207,48 @@ async def test_connect_exception_handling_and_metrics(
     mock_metrics_module.OPC_CONNECTIONS_FAILED.labels.return_value.inc.assert_called_once()
 
     opc_manager_instance.logger.error.assert_called_once_with(
-        f"Failed to connect to {opc_manager_instance.name}: {simulated_error_message}"
+        f'Failed to connect to {opc_manager_instance.name}: {simulated_error_message}'
     )
 
 
 @mark.asyncio
 async def test_create_subscription_no_client(raw_opc_manager):
     try:
-        await raw_opc_manager.create_subscription("sub1")
+        await raw_opc_manager.create_subscription('sub1')
     except ValueError as e:
-        assert str(e) == "Client not connected. Call connect first."
+        assert str(e) == 'Client not connected. Call connect first.'
     else:
-        assert False, "ValueError not raised"
+        assert False, 'ValueError not raised'
 
 
 @mark.asyncio
 async def test_create_subscription_success_has_period(opc_manager):
-    await opc_manager.create_subscription("sub1", 1000)
+    await opc_manager.create_subscription('sub1', 1000)
 
-    opc_manager.client.create_subscription.assert_called_once_with(
-        1000, opc_manager)
-    assert opc_manager.subscriptions["sub1"] is not None
+    opc_manager.client.create_subscription.assert_called_once_with(1000, opc_manager)
+    assert opc_manager.subscriptions['sub1'] is not None
 
 
 @mark.asyncio
 async def test_create_subscription_success_no_period(opc_manager):
-    await opc_manager.create_subscription("sub1", None)
+    await opc_manager.create_subscription('sub1', None)
 
-    opc_manager.client.create_subscription.assert_called_once_with(
-        500, opc_manager)
-    assert opc_manager.subscriptions["sub1"] is not None
+    opc_manager.client.create_subscription.assert_called_once_with(500, opc_manager)
+    assert opc_manager.subscriptions['sub1'] is not None
 
 
-@patch("ingestor.managers.opc_manager.metrics")
+@patch('ingestor.managers.opc_manager.metrics')
 @mark.asyncio
 async def test_create_subscription_with_metrics(metrics, opc_manager):
-    await opc_manager.create_subscription("sub1", 1000)
+    await opc_manager.create_subscription('sub1', 1000)
 
     metrics.OPC_SUBSCRIPTIONS_CREATED.labels.assert_called_once_with(
-        pod_id=opc_manager.pod_id, server_name=opc_manager.name, slot_name="sub1"
+        pod_id=opc_manager.pod_id, server_name=opc_manager.name, slot_name='sub1'
     )
     metrics.OPC_SUBSCRIPTIONS_CREATED.labels.return_value.inc.assert_called_once()
 
 
-@patch("ingestor.managers.opc_manager.metrics")
+@patch('ingestor.managers.opc_manager.metrics')
 @mark.asyncio
 async def test_create_subscription_exception_during_client_call(
     mock_metrics_module, raw_opc_manager
@@ -268,71 +256,66 @@ async def test_create_subscription_exception_during_client_call(
     opc_manager_instance = raw_opc_manager
     opc_manager_instance.client = AsyncMock()
 
-    subscription_name = "test_sub_client_error"
+    subscription_name = 'test_sub_client_error'
     simulated_period = 750
-    simulated_error_message = "Falha ao criar subscrição no cliente OPC"
+    simulated_error_message = 'Falha ao criar subscrição no cliente OPC'
 
-    opc_manager_instance.client.create_subscription.side_effect = Exception(
-        simulated_error_message
-    )
+    opc_manager_instance.client.create_subscription.side_effect = Exception(simulated_error_message)
 
     with pytest.raises(Exception, match=simulated_error_message):
-        await opc_manager_instance.create_subscription(
-            subscription_name, period=simulated_period
-        )
+        await opc_manager_instance.create_subscription(subscription_name, period=simulated_period)
 
     opc_manager_instance.client.create_subscription.assert_called_once_with(
         simulated_period, opc_manager_instance
     )
 
     opc_manager_instance.logger.error.assert_called_once_with(
-        f"Failed to create subscription {subscription_name} on {opc_manager_instance.name}: {simulated_error_message}"
+        f'Failed to create subscription {subscription_name} on {opc_manager_instance.name}: {simulated_error_message}'
     )
 
     mock_metrics_module.OPC_SUBSCRIPTIONS_CREATED.labels.assert_not_called()
 
 
-@patch("ingestor.managers.opc_manager.metrics")
+@patch('ingestor.managers.opc_manager.metrics')
 @mark.asyncio
 async def test_subscribe_no_subscription(metrics, opc_manager):
     try:
-        await opc_manager.subscribe("sub1", tags, 1000)
+        await opc_manager.subscribe('sub1', tags, 1000)
     except ValueError as e:
-        assert str(
-            e) == "Subscription not created. Call create_subscription first."
+        assert str(e) == 'Subscription not created. Call create_subscription first.'
     else:
-        assert False, "ValueError not raised"
+        assert False, 'ValueError not raised'
     metrics.OPC_TAGS_SUBSCRIBED.labels.assert_not_called()
 
 
 @mark.asyncio
 async def test_subscribe_success(opc_manager_subscribed):
     opc_manager_subscribed.client.get_node = MagicMock()
-    opc_manager_subscribed.nodes = {"ns=3;i=1001": "data"}
+    opc_manager_subscribed.nodes = {'ns=3;i=1001': 'data'}
 
-    await opc_manager_subscribed.subscribe("sub1", tags, 1000)
+    await opc_manager_subscribed.subscribe('sub1', tags, 1000)
 
     assert opc_manager_subscribed.nodes == tags
-    opc_manager_subscribed.subscriptions["sub1"].subscribe_data_change.assert_called_once_with(
+    opc_manager_subscribed.subscriptions['sub1'].subscribe_data_change.assert_called_once_with(
         [opc_manager_subscribed.client.get_node(n) for n in tags]
     )
 
 
 @mark.asyncio
 async def test_unsubscribe_no_subscription(opc_manager):
-    await opc_manager.unsubscribe("sub1")
+    await opc_manager.unsubscribe('sub1')
 
     opc_manager.logger.warning.assert_called_once_with(
         "Subscription 'sub1' not found. Cannot unsubscribe."
     )
-    assert opc_manager.subscriptions.get("sub1") is None
+    assert opc_manager.subscriptions.get('sub1') is None
 
 
 @mark.asyncio
 async def test_unsubscribe_success(opc_manager_subscribed):
-    await opc_manager_subscribed.unsubscribe("sub1")
+    await opc_manager_subscribed.unsubscribe('sub1')
 
-    opc_manager_subscribed.subscriptions.get("sub1") is None
+    opc_manager_subscribed.subscriptions.get('sub1') is None
 
 
 @mark.asyncio
@@ -341,7 +324,7 @@ async def test_disconnect_success(opc_manager_subscribed):
 
     await opc_manager_subscribed.disconnect()
 
-    opc_manager_subscribed.subscriptions["sub1"].delete.assert_called_once()
+    opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
     assert opc_manager_subscribed.client is None
 
 
@@ -352,42 +335,38 @@ async def test_disconnect_no_client(opc_manager_subscribed):
 
     opc_manager_subscribed.logger.warning.assert_has_calls(
         [
-            call("Client already disconnected."),
+            call('Client already disconnected.'),
         ]
     )
 
 
 @mark.asyncio
 async def test_disconnect_error_unsubscribe(opc_manager_subscribed):
-    opc_manager_subscribed.client = MagicMock(
-        disconnect=AsyncMock()
-    )
-    opc_manager_subscribed.subscriptions["sub1"] = MagicMock(
-        delete=AsyncMock(side_effect=Exception("Test error"))
+    opc_manager_subscribed.client = MagicMock(disconnect=AsyncMock())
+    opc_manager_subscribed.subscriptions['sub1'] = MagicMock(
+        delete=AsyncMock(side_effect=Exception('Test error'))
     )
 
     await opc_manager_subscribed.disconnect()
 
-    opc_manager_subscribed.subscriptions["sub1"].delete.assert_called_once()
+    opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
     opc_manager_subscribed.client = None
     opc_manager_subscribed.logger.error.assert_called_once_with(
-        "Failed to clean up subscription: Test error"
+        'Failed to clean up subscription: Test error'
     )
 
 
 @mark.asyncio
 async def test_disconnect_error(opc_manager_subscribed):
     opc_manager_subscribed.client = MagicMock()
-    opc_manager_subscribed.client.disconnect = MagicMock(
-        side_effect=Exception("Test error")
-    )
+    opc_manager_subscribed.client.disconnect = MagicMock(side_effect=Exception('Test error'))
 
     await opc_manager_subscribed.disconnect()
 
-    opc_manager_subscribed.subscriptions["sub1"].delete.assert_called_once()
+    opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
     opc_manager_subscribed.client = None
     opc_manager_subscribed.logger.error.assert_called_once_with(
-        "Failed to disconnect from OPC UA server: Test error"
+        'Failed to disconnect from OPC UA server: Test error'
     )
 
 
@@ -400,24 +379,21 @@ async def test_disconnect_metrics_on_successful_path(mock_metrics_module, raw_op
     raw_opc_manager.client = MagicMock()
     mock_sub1 = MagicMock()
     mock_sub2 = MagicMock()
-    raw_opc_manager.subscriptions = {"sub1": mock_sub1, "sub2": mock_sub2}
+    raw_opc_manager.subscriptions = {'sub1': mock_sub1, 'sub2': mock_sub2}
 
     await raw_opc_manager.disconnect()
 
     mock_metrics_module.OPC_CONNECTION_STATUS.labels.assert_called_once_with(
         pod_id=raw_opc_manager.pod_id,
         server_name=raw_opc_manager.name,
-        server_url=raw_opc_manager.url
+        server_url=raw_opc_manager.url,
     )
-    mock_metrics_module.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(
-        0)
+    mock_metrics_module.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(0)
 
     mock_metrics_module.OPC_TAGS_SUBSCRIBED.labels.assert_called_once_with(
-        pod_id=raw_opc_manager.pod_id,
-        server_name=raw_opc_manager.name
+        pod_id=raw_opc_manager.pod_id, server_name=raw_opc_manager.name
     )
-    mock_metrics_module.OPC_TAGS_SUBSCRIBED.labels.return_value.set.assert_called_once_with(
-        0)
+    mock_metrics_module.OPC_TAGS_SUBSCRIBED.labels.return_value.set.assert_called_once_with(0)
 
 
 @patch('ingestor.managers.opc_manager.metrics')
@@ -427,76 +403,70 @@ async def test_datachange_notification(metrics, opc_manager_subscribed):
         monitored_item=MagicMock(
             Value=MagicMock(
                 Value=MagicMock(Value=42),
-                SourceTimestamp=datetime.strptime(
-                    "2021-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S"
-                ),
+                SourceTimestamp=datetime.strptime('2021-01-01T00:00:00', '%Y-%m-%dT%H:%M:%S'),
             )
         )
     )
     opc_manager_subscribed.nodes = {
-        "ns=3;i=1001": {
-            "tag_name": "Counter",
-            "cycle_rule": {"cycle_increment": 1.0, "cycle_count": 2},
-            "topics": ["topic1", "topic2"],
+        'ns=3;i=1001': {
+            'tag_name': 'Counter',
+            'cycle_rule': {'cycle_increment': 1.0, 'cycle_count': 2},
+            'topics': ['topic1', 'topic2'],
         }
     }
 
     metrics.OPC_CYCLES_WITHOUT_DATA.reset_mock()
 
-    await opc_manager_subscribed.datachange_notification("ns=3;i=1001", None, data)
+    await opc_manager_subscribed.datachange_notification('ns=3;i=1001', None, data)
 
     opc_manager_subscribed.data_manager.publish.assert_any_call(
-        "topic1",
+        'topic1',
         {
-            "tag": "ns=3;i=1001",
-            "name": "Counter",
-            "timestamp": "2021-01-01 00:00:00-0300",
-            "value": 42,
+            'tag': 'ns=3;i=1001',
+            'name': 'Counter',
+            'timestamp': '2021-01-01 00:00:00-0300',
+            'value': 42,
         },
     )
     opc_manager_subscribed.data_manager.publish.assert_any_call(
-        "topic2",
+        'topic2',
         {
-            "tag": "ns=3;i=1001",
-            "name": "Counter",
-            "timestamp": "2021-01-01 00:00:00-0300",
-            "value": 42,
+            'tag': 'ns=3;i=1001',
+            'name': 'Counter',
+            'timestamp': '2021-01-01 00:00:00-0300',
+            'value': 42,
         },
     )
-    assert opc_manager_subscribed.nodes["ns=3;i=1001"]["cycle_rule"]["cycle_count"] == 0
+    assert opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == 0
 
     metrics.OPC_CYCLES_WITHOUT_DATA.labels.assert_called_once_with(
-        pod_id=opc_manager_subscribed.pod_id,
-        server_name=opc_manager_subscribed.name
+        pod_id=opc_manager_subscribed.pod_id, server_name=opc_manager_subscribed.name
     )
-    metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with(
-        0)
+    metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with(0)
 
 
 def test_check_cycles_no_notification(opc_manager):
     # Setup: node with cycle_count just below threshold
     opc_manager.nodes = {
-        "ns=3;i=1001": {
-            "tag_name": "Counter",
-            "cycle_rule": {"cycle_increment": 1.0, "cycle_count": 3.0},
+        'ns=3;i=1001': {
+            'tag_name': 'Counter',
+            'cycle_rule': {'cycle_increment': 1.0, 'cycle_count': 3.0},
         }
     }
 
     opc_manager.check_cycles()
 
     # After one increment, cycle_count = 4.0, still below threshold
-    assert opc_manager.nodes["ns=3;i=1001"]["cycle_rule"][
-        "cycle_count"
-    ] == pytest.approx(4.0)
+    assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(4.0)
     opc_manager.send_notification.assert_not_called()
 
 
 def test_check_cycles_triggers_notification(opc_manager):
     # Setup: node with cycle_count just below threshold, increment will cross threshold
     opc_manager.nodes = {
-        "ns=3;i=1001": {
-            "tag_name": "Counter",
-            "cycle_rule": {"cycle_increment": 2.5, "cycle_count": 3.0},
+        'ns=3;i=1001': {
+            'tag_name': 'Counter',
+            'cycle_rule': {'cycle_increment': 2.5, 'cycle_count': 3.0},
         }
     }
     opc_manager.notification_handler.build_and_send_notification = MagicMock()
@@ -504,15 +474,13 @@ def test_check_cycles_triggers_notification(opc_manager):
     opc_manager.check_cycles()
 
     # After increment, cycle_count = 5.5, should trigger notification
-    assert opc_manager.nodes["ns=3;i=1001"]["cycle_rule"][
-        "cycle_count"
-    ] == pytest.approx(5.5)
+    assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(5.5)
     opc_manager.send_notification.assert_called_once_with(
-        notification_id="TAG_ns=3;i=1001:Counter_LISTENNING_STOPPED",
-        message="5.5 cycles without receive from ns=3;i=1001:Counter",
-        block="opc_manager",
+        notification_id='TAG_ns=3;i=1001:Counter_LISTENNING_STOPPED',
+        message='5.5 cycles without receive from ns=3;i=1001:Counter',
+        block='opc_manager',
         level=NotificationLevel.WARNING,
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
     )
 
 
@@ -530,11 +498,11 @@ def test_check_opc_listenning_no_notification(metrics, opc_manager):
     assert result is False
 
     metrics.OPC_CYCLES_WITHOUT_DATA.labels.assert_called_once_with(
-        pod_id=opc_manager.pod_id,
-        server_name=opc_manager.name
+        pod_id=opc_manager.pod_id, server_name=opc_manager.name
     )
     metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with(
-        opc_manager.non_receive_count)
+        opc_manager.non_receive_count
+    )
     metrics.OPC_RECONNECTIONS_TOTAL.labels.assert_not_called()
 
 
@@ -545,11 +513,11 @@ def test_check_opc_listenning_warning_notification(opc_manager):
 
     assert opc_manager.non_receive_count == 5
     opc_manager.send_notification.assert_called_once_with(
-        notification_id=f"OPC_LISTENNING_STOPPED__{opc_manager.name}",
-        message=f"5 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}",
-        block="opc_manager",
+        notification_id=f'OPC_LISTENNING_STOPPED__{opc_manager.name}',
+        message=f'5 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}',
+        block='opc_manager',
         level=NotificationLevel.ERROR,
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
     )
     assert result is False
 
@@ -569,47 +537,45 @@ def test_check_opc_listenning_error_notification_and_retry(metrics, opc_manager)
     calls = opc_manager.send_notification.call_args_list
     # First call: 5 cycles warning
     assert calls[0].kwargs == {
-        "notification_id": f"OPC_LISTENNING_STOPPED__{opc_manager.name}",
-        "message": f"15 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}",
-        "block": "opc_manager",
-        "level": NotificationLevel.ERROR,
-        "metadata": metadata["metadata"],
+        'notification_id': f'OPC_LISTENNING_STOPPED__{opc_manager.name}',
+        'message': f'15 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}',
+        'block': 'opc_manager',
+        'level': NotificationLevel.ERROR,
+        'metadata': metadata['metadata'],
     }
     # Second call: 15 cycles retry
     assert calls[1].kwargs == {
-        "notification_id": f"OPC_CONNECTION_RETRY__{opc_manager.name}",
-        "message": f"Retrying to connect to server {opc_manager.name}",
-        "block": "opc_manager",
-        "level": NotificationLevel.ERROR,
-        "metadata": metadata["metadata"],
+        'notification_id': f'OPC_CONNECTION_RETRY__{opc_manager.name}',
+        'message': f'Retrying to connect to server {opc_manager.name}',
+        'block': 'opc_manager',
+        'level': NotificationLevel.ERROR,
+        'metadata': metadata['metadata'],
     }
     assert result is True
 
     metrics.OPC_CYCLES_WITHOUT_DATA.labels.assert_called_once_with(
-        pod_id=opc_manager.pod_id,
-        server_name=opc_manager.name
+        pod_id=opc_manager.pod_id, server_name=opc_manager.name
     )
     metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with(
-        opc_manager.non_receive_count)
+        opc_manager.non_receive_count
+    )
 
     metrics.OPC_RECONNECTIONS_TOTAL.labels.assert_called_once_with(
-        pod_id=opc_manager.pod_id,
-        server_name=opc_manager.name
+        pod_id=opc_manager.pod_id, server_name=opc_manager.name
     )
     metrics.OPC_RECONNECTIONS_TOTAL.labels.return_value.inc.assert_called_once()
 
 
-@patch("ingestor.managers.opc_manager.metrics")
+@patch('ingestor.managers.opc_manager.metrics')
 def test_init_metrics_calls_correct_metric_methods(metrics):
-
     opc_manager = OpcManager(
-        name="TestInitConnector",
-        url="opc.tcp://init.test:4840",
+        name='TestInitConnector',
+        url='opc.tcp://init.test:4840',
         data_manager=MagicMock(),
         logger=MagicMock(),
-        server_uri="opc.tcp://init.test:4840/uri",
+        server_uri='opc.tcp://init.test:4840/uri',
         notification_handler=MagicMock(),
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
     )
 
     metrics.OPC_CONNECTION_STATUS.labels.assert_called_with(
@@ -618,13 +584,10 @@ def test_init_metrics_calls_correct_metric_methods(metrics):
         server_url=opc_manager.url,
     )
 
-    metrics.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(
-        0)
+    metrics.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(0)
 
     metrics.OPC_TAGS_SUBSCRIBED.labels.assert_called_once_with(
-        pod_id=opc_manager.pod_id,
-        server_name=opc_manager.name
+        pod_id=opc_manager.pod_id, server_name=opc_manager.name
     )
 
-    metrics.OPC_TAGS_SUBSCRIBED.labels.return_value.set.assert_called_once_with(
-        0)
+    metrics.OPC_TAGS_SUBSCRIBED.labels.return_value.set.assert_called_once_with(0)

tests/unit/managers/test_resource_manager.py

@@ -1,28 +1,29 @@
 from unittest.mock import MagicMock, patch
+
 from pytest import fixture, raises
 from sientia_do.notifications.models import NotificationLevel
+
 from ingestor.managers.resource_manager import ResourceManager
 
 metadata = {
-    "metadata": {
-        "model_id": "test_model",
-        "model_name": "test_model",
-        "workflow_name": "test_workflow",
-        "schema_name": "test_schedule",
+    'metadata': {
+        'model_id': 'test_model',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+        'schema_name': 'test_schedule',
     },
 }
 
 
 @fixture
-@patch("ingestor.managers.resource_manager.Redis")
+@patch('ingestor.managers.resource_manager.Redis')
 def resource_manager(redis):
-
     resource_manager = ResourceManager(
-        host="localhost",
+        host='localhost',
         port=6379,
         lease_ttl=10,
         heartbeat_ttl=10,
-        metadata=metadata["metadata"],
+        metadata=metadata['metadata'],
         logger=MagicMock(),
         notification_handler=MagicMock(),
     )
@@ -34,152 +35,147 @@ def resource_manager(redis):
 
 def test_get_success(resource_manager):
     resource_manager.redis.get.return_value = '{"key": "value"}'
-    result = resource_manager.get("key")
-    assert result == {"key": "value"}
-    resource_manager.redis.get.assert_called_once_with("key")
+    result = resource_manager.get('key')
+    assert result == {'key': 'value'}
+    resource_manager.redis.get.assert_called_once_with('key')
 
 
 def test_get_failure(resource_manager):
     resource_manager.redis.get.return_value = None
-    result = resource_manager.get("key")
+    result = resource_manager.get('key')
     assert result is None
-    resource_manager.redis.get.assert_called_once_with("key")
+    resource_manager.redis.get.assert_called_once_with('key')
 
 
 def test_get_tag_slot(resource_manager):
-    resource_manager.get = MagicMock(return_value={"tag": "slot"})
-    result = resource_manager.get_tag_slot("id")
-    assert result == {"tag": "slot"}
-    resource_manager.get.assert_called_once_with("slot:opc_tags:id")
+    resource_manager.get = MagicMock(return_value={'tag': 'slot'})
+    result = resource_manager.get_tag_slot('id')
+    assert result == {'tag': 'slot'}
+    resource_manager.get.assert_called_once_with('slot:opc_tags:id')
 
 
 def test_ingestor_heartbeat(resource_manager):
     resource_manager.redis.set.return_value = True
     resource_manager.ingestor_heartbeat()
-    resource_manager.redis.set.assert_called_once_with(
-        "heartbeat:ingestor:localhost", 1, ex=10
-    )
+    resource_manager.redis.set.assert_called_once_with('heartbeat:ingestor:localhost', 1, ex=10)
 
 
 def test_lease_tag(resource_manager):
     resource_manager.redis.set.return_value = True
-    output = resource_manager.lease_tag("tag_id")
+    output = resource_manager.lease_tag('tag_id')
     assert output is True
     resource_manager.redis.set.assert_called_once_with(
-        "lease:opc_tags:tag_id", "localhost", nx=True, ex=10
+        'lease:opc_tags:tag_id', 'localhost', nx=True, ex=10
     )
 
 
 def test_renew_tag_lease_success(resource_manager):
-    resource_manager.redis.get.return_value = "localhost"
+    resource_manager.redis.get.return_value = 'localhost'
     resource_manager.redis.expire.return_value = True
-    result = resource_manager.renew_tag_lease("tag_id")
+    result = resource_manager.renew_tag_lease('tag_id')
     assert result is True
-    resource_manager.redis.get.assert_called_once_with("lease:opc_tags:tag_id")
-    resource_manager.redis.expire.assert_called_once_with(
-        "lease:opc_tags:tag_id", 10)
+    resource_manager.redis.get.assert_called_once_with('lease:opc_tags:tag_id')
+    resource_manager.redis.expire.assert_called_once_with('lease:opc_tags:tag_id', 10)
 
 
 def test_renew_tag_lease_failure(resource_manager):
-    resource_manager.redis.get.return_value = "other_pod_id"
+    resource_manager.redis.get.return_value = 'other_pod_id'
     resource_manager.redis.expire.return_value = False
-    result = resource_manager.renew_tag_lease("tag_id")
+    result = resource_manager.renew_tag_lease('tag_id')
     assert result is False
-    resource_manager.redis.get.assert_called_once_with("lease:opc_tags:tag_id")
+    resource_manager.redis.get.assert_called_once_with('lease:opc_tags:tag_id')
     resource_manager.redis.expire.assert_not_called()
 
 
 def test_drop_tag_lease(resource_manager):
     resource_manager.redis.delete.return_value = True
-    resource_manager.drop_tag_lease("tag_id")
-    resource_manager.redis.delete.assert_called_once_with(
-        "lease:opc_tags:tag_id")
+    resource_manager.drop_tag_lease('tag_id')
+    resource_manager.redis.delete.assert_called_once_with('lease:opc_tags:tag_id')
 
 
 def test_get_all_ingestors(resource_manager):
-    resource_manager.redis.keys.return_value = ["ingestor1", "ingestor2"]
+    resource_manager.redis.keys.return_value = ['ingestor1', 'ingestor2']
     result = resource_manager.get_all_ingestors()
-    assert result == ["ingestor1", "ingestor2"]
-    resource_manager.redis.keys.assert_called_once_with("heartbeat:ingestor:*")
+    assert result == ['ingestor1', 'ingestor2']
+    resource_manager.redis.keys.assert_called_once_with('heartbeat:ingestor:*')
 
 
 def test_get_all_slots(resource_manager):
-    resource_manager.redis.keys.return_value = ["slot1", "slot2"]
+    resource_manager.redis.keys.return_value = ['slot1', 'slot2']
     result = resource_manager.get_all_slots()
-    assert result == ["slot1", "slot2"]
-    resource_manager.redis.keys.assert_called_once_with("slot:opc_tags:*")
+    assert result == ['slot1', 'slot2']
+    resource_manager.redis.keys.assert_called_once_with('slot:opc_tags:*')
 
 
 def test_get_all_leases(resource_manager):
-    resource_manager.redis.keys.return_value = ["lease1", "lease2"]
+    resource_manager.redis.keys.return_value = ['lease1', 'lease2']
     result = resource_manager.get_all_leases()
-    assert result == ["lease1", "lease2"]
-    resource_manager.redis.keys.assert_called_once_with("lease:opc_tags:*")
+    assert result == ['lease1', 'lease2']
+    resource_manager.redis.keys.assert_called_once_with('lease:opc_tags:*')
 
 
 def test_init_connection_failure(monkeypatch):
     # Mock Redis to raise an exception during initialization
     mock_redis = MagicMock()
-    mock_redis.side_effect = Exception("Connection failed")
+    mock_redis.side_effect = Exception('Connection failed')
 
-    monkeypatch.setattr("ingestor.managers.resource_manager.Redis", mock_redis)
+    monkeypatch.setattr('ingestor.managers.resource_manager.Redis', mock_redis)
 
     # Test that the exception is raised and metrics are set properly
-    with patch("ingestor.metrics.REDIS_CONNECTION_STATUS") as mock_metrics:
+    with patch('ingestor.metrics.REDIS_CONNECTION_STATUS') as mock_metrics:
         mock_status = MagicMock()
         mock_metrics.labels.return_value = mock_status
 
-        with raises(Exception, match="Connection failed"):
+        with raises(Exception, match='Connection failed'):
             ResourceManager(
-                host="localhost",
+                host='localhost',
                 port=6379,
                 lease_ttl=10,
                 heartbeat_ttl=10,
-                metadata=metadata["metadata"],
+                metadata=metadata['metadata'],
                 logger=MagicMock(),
                 notification_handler=MagicMock(),
             )
 
-        mock_metrics.labels.assert_called_once_with(pod_id="localhost")
+        mock_metrics.labels.assert_called_once_with(pod_id='localhost')
         mock_status.set.assert_called_once_with(0)
 
 
 def test_init_ping_failure(monkeypatch):
     # Mock Redis ping to raise an exception
     mock_redis_instance = MagicMock()
-    mock_redis_instance.ping.side_effect = Exception("Ping failed")
+    mock_redis_instance.ping.side_effect = Exception('Ping failed')
 
     mock_redis_class = MagicMock(return_value=mock_redis_instance)
-    monkeypatch.setattr(
-        "ingestor.managers.resource_manager.Redis", mock_redis_class)
+    monkeypatch.setattr('ingestor.managers.resource_manager.Redis', mock_redis_class)
 
     # Test that the exception is raised and metrics are set properly
-    with patch("ingestor.metrics.REDIS_CONNECTION_STATUS") as mock_metrics:
+    with patch('ingestor.metrics.REDIS_CONNECTION_STATUS') as mock_metrics:
         mock_status = MagicMock()
         mock_metrics.labels.return_value = mock_status
 
-        with raises(Exception, match="Ping failed"):
+        with raises(Exception, match='Ping failed'):
             ResourceManager(
-                host="localhost",
+                host='localhost',
                 port=6379,
                 lease_ttl=10,
                 heartbeat_ttl=10,
-                metadata=metadata["metadata"],
+                metadata=metadata['metadata'],
                 logger=MagicMock(),
                 notification_handler=MagicMock(),
             )
 
-        mock_metrics.labels.assert_called_once_with(pod_id="localhost")
+        mock_metrics.labels.assert_called_once_with(pod_id='localhost')
         mock_status.set.assert_called_once_with(0)
 
 
 def test_execute_redis_op_success(resource_manager):
     # Mock the Redis operation and time function
-    mock_func = MagicMock(return_value="test_result")
+    mock_func = MagicMock(return_value='test_result')
 
-    with patch("ingestor.managers.resource_manager.time", side_effect=[100, 100.5]):
-        with patch("ingestor.metrics.REDIS_OPERATIONS_TOTAL") as mock_total:
-            with patch("ingestor.metrics.REDIS_OPERATIONS_DURATION") as mock_duration:
+    with patch('ingestor.managers.resource_manager.time', side_effect=[100, 100.5]):
+        with patch('ingestor.metrics.REDIS_OPERATIONS_TOTAL') as mock_total:
+            with patch('ingestor.metrics.REDIS_OPERATIONS_DURATION') as mock_duration:
                 mock_total_labels = MagicMock()
                 mock_duration_labels = MagicMock()
                 mock_total.labels.return_value = mock_total_labels
@@ -187,49 +183,42 @@ def test_execute_redis_op_success(resource_manager):
 
                 # Execute the operation
                 result = resource_manager._execute_redis_op(
-                    "test_op", mock_func, "arg1", kwarg1="value1"
+                    'test_op', mock_func, 'arg1', kwarg1='value1'
                 )
 
                 # Verify the result and metrics
-                assert result == "test_result"
-                mock_func.assert_called_once_with("arg1", kwarg1="value1")
+                assert result == 'test_result'
+                mock_func.assert_called_once_with('arg1', kwarg1='value1')
 
-                mock_total.labels.assert_called_once_with(
-                    pod_id="localhost", operation="test_op"
-                )
+                mock_total.labels.assert_called_once_with(pod_id='localhost', operation='test_op')
                 mock_total_labels.inc.assert_called_once()
 
                 mock_duration.labels.assert_called_once_with(
-                    pod_id="localhost", operation="test_op"
-                )
-                mock_duration_labels.observe.assert_called_once_with(
-                    0.5
+                    pod_id='localhost', operation='test_op'
                 )
+                mock_duration_labels.observe.assert_called_once_with(0.5)
 
 
 def test_execute_redis_op_exception(resource_manager):
     # Mock the Redis operation to raise an exception
-    mock_func = MagicMock(side_effect=Exception("Operation failed"))
+    mock_func = MagicMock(side_effect=Exception('Operation failed'))
 
-    with patch("ingestor.managers.resource_manager.time", return_value=100):
-        with patch("ingestor.metrics.REDIS_OPERATIONS_ERRORS") as mock_errors:
+    with patch('ingestor.managers.resource_manager.time', return_value=100):
+        with patch('ingestor.metrics.REDIS_OPERATIONS_ERRORS') as mock_errors:
             mock_errors_labels = MagicMock()
             mock_errors.labels.return_value = mock_errors_labels
 
             # Execute the operation and expect an exception
-            with raises(Exception, match="Operation failed"):
-                resource_manager._execute_redis_op(
-                    "test_op", mock_func, "arg1")
+            with raises(Exception, match='Operation failed'):
+                resource_manager._execute_redis_op('test_op', mock_func, 'arg1')
 
             # Verify metrics and error handling
-            mock_errors.labels.assert_called_once_with(
-                pod_id="localhost", operation="test_op"
-            )
+            mock_errors.labels.assert_called_once_with(pod_id='localhost', operation='test_op')
             mock_errors_labels.inc.assert_called_once()
             resource_manager.send_notification.assert_called_once_with(
-                metadata=metadata["metadata"],
-                notification_id="REDIS_OPERATION_ERROR_test_op",
+                metadata=metadata['metadata'],
+                notification_id='REDIS_OPERATION_ERROR_test_op',
                 message="Error in Redis operation 'test_op': Operation failed",
-                block="redis_manager",
+                block='redis_manager',
                 level=NotificationLevel.ERROR,
             )

tests/unit/test_app.py

@@ -1,10 +1,9 @@
-import pytest
-import pytest_asyncio
-from unittest.mock import patch, MagicMock, call, AsyncMock
-from threading import Event
-import signal as signal_module  # To avoid conflict with mock names
-import os
 import asyncio
+import signal as signal_module  # To avoid conflict with mock names
+from threading import Event
+from unittest.mock import AsyncMock, MagicMock, call
+
+import pytest
 
 # Import the 'app' module to be tested
 from ingestor import app
@@ -13,7 +12,7 @@ from ingestor import app
 # Custom exception to catch os._exit calls
 class OsExitCalled(Exception):
     def __init__(self, code):
-        super().__init__(f"os._exit({code}) called")
+        super().__init__(f'os._exit({code}) called')
         self.code = code
 
 
@@ -26,66 +25,56 @@ def raise_os_exit_with_code(exit_code):
 def mock_app_env(monkeypatch):
     """Fixture to mock dependencies of app.main and app.signal_handler."""
     mocks = {
-        "start_http_server": MagicMock(),
-        "Ingestor": MagicMock(),
-        "metrics_APP_UP_labels_set": MagicMock(),
-        "metrics_APP_LOOP_COUNT_labels_inc": MagicMock(),
-        "metrics_APP_LOOP_DURATION_labels_observe": MagicMock(),
-        "metrics_APP_ERRORS_TOTAL_labels_inc": MagicMock(),
-        "os_exit": MagicMock(side_effect=raise_os_exit_with_code),
-        "time_time": MagicMock(),
-        "asyncio_sleep": AsyncMock(),
-        "signal_signal": MagicMock(),
-        "traceback_print_exc": MagicMock(),
-        "mock_exit_signal": MagicMock(spec=Event),
+        'start_http_server': MagicMock(),
+        'Ingestor': MagicMock(),
+        'metrics_APP_UP_labels_set': MagicMock(),
+        'metrics_APP_LOOP_COUNT_labels_inc': MagicMock(),
+        'metrics_APP_LOOP_DURATION_labels_observe': MagicMock(),
+        'metrics_APP_ERRORS_TOTAL_labels_inc': MagicMock(),
+        'os_exit': MagicMock(side_effect=raise_os_exit_with_code),
+        'time_time': MagicMock(),
+        'asyncio_sleep': AsyncMock(),
+        'signal_signal': MagicMock(),
+        'traceback_print_exc': MagicMock(),
+        'mock_exit_signal': MagicMock(spec=Event),
     }
 
-    monkeypatch.setattr(app, "start_http_server", mocks["start_http_server"])
-    monkeypatch.setattr(app, "Ingestor", mocks["Ingestor"])
-    monkeypatch.setattr(asyncio, "sleep", mocks["asyncio_sleep"])
+    monkeypatch.setattr(app, 'start_http_server', mocks['start_http_server'])
+    monkeypatch.setattr(app, 'Ingestor', mocks['Ingestor'])
+    monkeypatch.setattr(asyncio, 'sleep', mocks['asyncio_sleep'])
 
     monkeypatch.setattr(
         app.metrics.APP_UP,
-        "labels",
-        MagicMock(return_value=MagicMock(
-            set=mocks["metrics_APP_UP_labels_set"])),
+        'labels',
+        MagicMock(return_value=MagicMock(set=mocks['metrics_APP_UP_labels_set'])),
     )
     monkeypatch.setattr(
         app.metrics.APP_LOOP_COUNT,
-        "labels",
-        MagicMock(
-            return_value=MagicMock(
-                inc=mocks["metrics_APP_LOOP_COUNT_labels_inc"])
-        ),
+        'labels',
+        MagicMock(return_value=MagicMock(inc=mocks['metrics_APP_LOOP_COUNT_labels_inc'])),
     )
     monkeypatch.setattr(
         app.metrics.APP_LOOP_DURATION,
-        "labels",
+        'labels',
         MagicMock(
-            return_value=MagicMock(
-                observe=mocks["metrics_APP_LOOP_DURATION_labels_observe"]
-            )
+            return_value=MagicMock(observe=mocks['metrics_APP_LOOP_DURATION_labels_observe'])
         ),
     )
     monkeypatch.setattr(
         app.metrics.APP_ERRORS_TOTAL,
-        "labels",
-        MagicMock(
-            return_value=MagicMock(
-                inc=mocks["metrics_APP_ERRORS_TOTAL_labels_inc"])
-        ),
+        'labels',
+        MagicMock(return_value=MagicMock(inc=mocks['metrics_APP_ERRORS_TOTAL_labels_inc'])),
     )
 
-    monkeypatch.setattr(app.os, "_exit", mocks["os_exit"])
-    monkeypatch.setattr(app, "time", mocks["time_time"])
-    monkeypatch.setattr(app.signal, "signal", mocks["signal_signal"])
-    monkeypatch.setattr(app.traceback, "print_exc",
-                        mocks["traceback_print_exc"])
+    monkeypatch.setattr(app.os, '_exit', mocks['os_exit'])
+    monkeypatch.setattr(app, 'time', mocks['time_time'])
+    monkeypatch.setattr(app.signal, 'signal', mocks['signal_signal'])
+    monkeypatch.setattr(app.traceback, 'print_exc', mocks['traceback_print_exc'])
 
-    monkeypatch.setattr(app, "exit_signal", mocks["mock_exit_signal"])
-    monkeypatch.setattr(app, "POD_ID", "test_pod")
+    monkeypatch.setattr(app, 'exit_signal', mocks['mock_exit_signal'])
+    monkeypatch.setattr(app, 'POD_ID', 'test_pod')
 
-    mock_ingestor_instance = mocks["Ingestor"].return_value
+    mock_ingestor_instance = mocks['Ingestor'].return_value
     mock_ingestor_instance.poll_interval = 0.01
     mock_ingestor_instance.logger = MagicMock()
 
@@ -100,54 +89,49 @@ def mock_app_env(monkeypatch):
 @pytest.mark.asyncio
 async def test_main_successful_run_one_loop(mock_app_env, capsys):
     """Test a successful run where the loop executes once and then exits gracefully."""
-    mock_ingestor_instance = mock_app_env["Ingestor"].return_value
-    mock_exit_signal = mock_app_env["mock_exit_signal"]
+    mock_ingestor_instance = mock_app_env['Ingestor'].return_value
+    mock_exit_signal = mock_app_env['mock_exit_signal']
 
     mock_exit_signal.is_set.side_effect = [False, True]
-    mock_app_env["time_time"].side_effect = [10.0, 11.5]
+    mock_app_env['time_time'].side_effect = [10.0, 11.5]
 
     with pytest.raises(OsExitCalled) as excinfo:
         await app.main()
     assert excinfo.value.code == 0
 
-    mock_app_env["start_http_server"].assert_called_once_with(9090)
-    app.metrics.APP_UP.labels.assert_any_call(pod_id="test_pod")
-    set_calls = mock_app_env["metrics_APP_UP_labels_set"].call_args_list
+    mock_app_env['start_http_server'].assert_called_once_with(9090)
+    app.metrics.APP_UP.labels.assert_any_call(pod_id='test_pod')
+    set_calls = mock_app_env['metrics_APP_UP_labels_set'].call_args_list
     assert call(1) in set_calls
     assert call(0) in set_calls
     assert set_calls.index(call(1)) < set_calls.index(call(0))
 
-    mock_app_env["Ingestor"].assert_called_once_with()
+    mock_app_env['Ingestor'].assert_called_once_with()
     mock_ingestor_instance.prepare_ingestor.assert_called_once()
-    mock_ingestor_instance.logger.info.assert_any_call(
-        "Ingestor prepared. Starting main loop."
-    )
+    mock_ingestor_instance.logger.info.assert_any_call('Ingestor prepared. Starting main loop.')
 
     mock_ingestor_instance.loop.assert_called_once()
-    app.metrics.APP_LOOP_COUNT.labels.assert_called_with(pod_id="test_pod")
-    mock_app_env["metrics_APP_LOOP_COUNT_labels_inc"].assert_called_once()
+    app.metrics.APP_LOOP_COUNT.labels.assert_called_with(pod_id='test_pod')
+    mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].assert_called_once()
 
-    mock_app_env["asyncio_sleep"].assert_has_calls(
-        [call(mock_ingestor_instance.poll_interval), call(5)])
-
-    app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id="test_pod")
-    mock_app_env["metrics_APP_LOOP_DURATION_labels_observe"].assert_called_once_with(
-        1.5
+    mock_app_env['asyncio_sleep'].assert_has_calls(
+        [call(mock_ingestor_instance.poll_interval), call(5)]
     )
 
+    app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id='test_pod')
+    mock_app_env['metrics_APP_LOOP_DURATION_labels_observe'].assert_called_once_with(1.5)
+
     mock_ingestor_instance.shutdown.assert_called_once()
-    mock_ingestor_instance.logger.info.assert_any_call(
-        "Main loop exit_signaled.")
+    mock_ingestor_instance.logger.info.assert_any_call('Main loop exit_signaled.')
 
     captured = capsys.readouterr()
-    assert "Prometheus server started on port 9090." in captured.out
+    assert 'Prometheus server started on port 9090.' in captured.out
 
 
 @pytest.mark.asyncio
 async def test_main_prometheus_server_fails_to_start(mock_app_env, capsys):
     """Test the scenario where starting the Prometheus server fails."""
-    mock_app_env["start_http_server"].side_effect = OSError(
-        "Port already in use")
+    mock_app_env['start_http_server'].side_effect = OSError('Port already in use')
 
     with pytest.raises(OsExitCalled) as excinfo:
         await app.main()
@@ -155,62 +139,60 @@ async def test_main_prometheus_server_fails_to_start(mock_app_env, capsys):
 
     # Check that .set(1) was not called. .set(0) definitely not called.
     called_with_1 = False
-    for call_args in mock_app_env["metrics_APP_UP_labels_set"].call_args_list:
+    for call_args in mock_app_env['metrics_APP_UP_labels_set'].call_args_list:
         if call_args == call(1):
             called_with_1 = True
             break
-    assert (
-        not called_with_1
-    ), "APP_UP.set(1) should not have been called if server start failed"
+    assert not called_with_1, 'APP_UP.set(1) should not have been called if server start failed'
 
-    mock_app_env["Ingestor"].assert_not_called()
+    mock_app_env['Ingestor'].assert_not_called()
     captured = capsys.readouterr()
-    assert "Failed to start Prometheus server: Port already in use" in captured.out
+    assert 'Failed to start Prometheus server: Port already in use' in captured.out
 
 
 @pytest.mark.asyncio
 async def test_main_loop_exception_handling(mock_app_env, capsys):
     """Test that an exception in ingestor.loop() is handled gracefully."""
-    mock_ingestor_instance = mock_app_env["Ingestor"].return_value
-    mock_exit_signal = mock_app_env["mock_exit_signal"]
+    mock_ingestor_instance = mock_app_env['Ingestor'].return_value
+    mock_exit_signal = mock_app_env['mock_exit_signal']
 
     mock_exit_signal.is_set.side_effect = [False, True]
-    mock_ingestor_instance.loop.side_effect = Exception("Test loop exception")
-    mock_app_env["time_time"].side_effect = [10.0, 10.1]
+    mock_ingestor_instance.loop.side_effect = Exception('Test loop exception')
+    mock_app_env['time_time'].side_effect = [10.0, 10.1]
 
     with pytest.raises(OsExitCalled) as excinfo:
         await app.main()
     assert excinfo.value.code == 0
 
     mock_ingestor_instance.loop.assert_called_once()
-    mock_app_env["traceback_print_exc"].assert_called_once()
+    mock_app_env['traceback_print_exc'].assert_called_once()
 
-    app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id="test_pod")
-    mock_app_env["metrics_APP_ERRORS_TOTAL_labels_inc"].assert_called_once()
+    app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id='test_pod')
+    mock_app_env['metrics_APP_ERRORS_TOTAL_labels_inc'].assert_called_once()
 
     mock_exit_signal.set.assert_called_once()
 
-    mock_app_env["metrics_APP_LOOP_COUNT_labels_inc"].assert_not_called()
+    mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].assert_not_called()
 
-    app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id="test_pod")
-    mock_app_env["metrics_APP_LOOP_DURATION_labels_observe"].assert_called_once_with(
+    app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id='test_pod')
+    mock_app_env['metrics_APP_LOOP_DURATION_labels_observe'].assert_called_once_with(
         pytest.approx(0.1)
     )
 
     mock_ingestor_instance.shutdown.assert_called_once()
     captured = capsys.readouterr()
-    assert "Exception in main loop. Setting exit_signal flag." in captured.out
+    assert 'Exception in main loop. Setting exit_signal flag.' in captured.out
 
 
 @pytest.mark.asyncio
 async def test_main_keyboard_interrupt_handling(mock_app_env, capsys):
     """Test that KeyboardInterrupt in ingestor.loop() is handled."""
-    mock_ingestor_instance = mock_app_env["Ingestor"].return_value
-    mock_exit_signal = mock_app_env["mock_exit_signal"]
+    mock_ingestor_instance = mock_app_env['Ingestor'].return_value
+    mock_exit_signal = mock_app_env['mock_exit_signal']
 
     mock_exit_signal.is_set.side_effect = [False, True]
     mock_ingestor_instance.loop.side_effect = KeyboardInterrupt()
-    mock_app_env["time_time"].side_effect = [10.0, 10.1]
+    mock_app_env['time_time'].side_effect = [10.0, 10.1]
 
     with pytest.raises(OsExitCalled) as excinfo:
         await app.main()
@@ -220,13 +202,13 @@ async def test_main_keyboard_interrupt_handling(mock_app_env, capsys):
     mock_exit_signal.set.assert_called_once()
     mock_ingestor_instance.shutdown.assert_called_once()
     captured = capsys.readouterr()
-    assert "KeyboardInterrupt received. Setting exit_signal flag." in captured.out
-    mock_app_env["metrics_APP_ERRORS_TOTAL_labels_inc"].assert_not_called()
+    assert 'KeyboardInterrupt received. Setting exit_signal flag.' in captured.out
+    mock_app_env['metrics_APP_ERRORS_TOTAL_labels_inc'].assert_not_called()
 
 
 def test_signal_handler_sets_exit_signal(mock_app_env):
     """Test that the signal_handler function calls exit_signal.set()."""
-    mock_exit_signal_set = mock_app_env["mock_exit_signal"].set
+    mock_exit_signal_set = mock_app_env['mock_exit_signal'].set
 
     app.signal_handler(signal_module.SIGINT, None)
     mock_exit_signal_set.assert_called_once()
@@ -235,12 +217,11 @@ def test_signal_handler_sets_exit_signal(mock_app_env):
 @pytest.mark.asyncio
 async def test_main_multiple_loop_iterations(mock_app_env):
     """Test the main loop runs for a few iterations."""
-    mock_ingestor_instance = mock_app_env["Ingestor"].return_value
-    mock_exit_signal = mock_app_env["mock_exit_signal"]
+    mock_ingestor_instance = mock_app_env['Ingestor'].return_value
+    mock_exit_signal = mock_app_env['mock_exit_signal']
 
     mock_exit_signal.is_set.side_effect = [False, False, False, True]
-    mock_app_env["time_time"].side_effect = [
-        10.0, 10.1, 10.2, 10.3, 10.4, 10.5]
+    mock_app_env['time_time'].side_effect = [10.0, 10.1, 10.2, 10.3, 10.4, 10.5]
 
     with pytest.raises(OsExitCalled) as excinfo:
         await app.main()
@@ -250,17 +231,15 @@ async def test_main_multiple_loop_iterations(mock_app_env):
 
     assert app.metrics.APP_LOOP_COUNT.labels.call_count == 3
     app.metrics.APP_LOOP_COUNT.labels.assert_called_with(
-        pod_id="test_pod"
+        pod_id='test_pod'
     )  # Checks last call or any call
-    assert mock_app_env["metrics_APP_LOOP_COUNT_labels_inc"].call_count == 3
+    assert mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].call_count == 3
 
-    assert mock_app_env["asyncio_sleep"].call_count == 4
+    assert mock_app_env['asyncio_sleep'].call_count == 4
 
     assert app.metrics.APP_LOOP_DURATION.labels.call_count == 3
-    app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id="test_pod")
-    duration_calls = mock_app_env[
-        "metrics_APP_LOOP_DURATION_labels_observe"
-    ].call_args_list
+    app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id='test_pod')
+    duration_calls = mock_app_env['metrics_APP_LOOP_DURATION_labels_observe'].call_args_list
     assert duration_calls[0] == call(pytest.approx(0.1, abs=1e-9))
     assert duration_calls[1] == call(pytest.approx(0.1, abs=1e-9))
     assert duration_calls[2] == call(pytest.approx(0.1, abs=1e-9))
@@ -271,32 +250,32 @@ async def test_main_multiple_loop_iterations(mock_app_env):
 @pytest.mark.asyncio
 async def test_main_pod_id_used_in_metrics(mock_app_env):
     """Test that the POD_ID from app module is used in metric labels."""
-    mock_exit_signal = mock_app_env["mock_exit_signal"]
+    mock_exit_signal = mock_app_env['mock_exit_signal']
     mock_exit_signal.is_set.side_effect = [False, True]
-    mock_app_env["time_time"].side_effect = [10.0, 11.0]
+    mock_app_env['time_time'].side_effect = [10.0, 11.0]
 
     with pytest.raises(OsExitCalled):
         await app.main()
 
-    app.metrics.APP_UP.labels.assert_any_call(pod_id="test_pod")
-    app.metrics.APP_LOOP_COUNT.labels.assert_any_call(pod_id="test_pod")
-    app.metrics.APP_LOOP_DURATION.labels.assert_any_call(pod_id="test_pod")
+    app.metrics.APP_UP.labels.assert_any_call(pod_id='test_pod')
+    app.metrics.APP_LOOP_COUNT.labels.assert_any_call(pod_id='test_pod')
+    app.metrics.APP_LOOP_DURATION.labels.assert_any_call(pod_id='test_pod')
     # APP_ERRORS_TOTAL would be checked similarly if it were called in this flow.
 
     # Check the .set() / .inc() calls on the mocks returned by .labels()
-    mock_app_env["metrics_APP_UP_labels_set"].assert_any_call(1)
-    mock_app_env["metrics_APP_UP_labels_set"].assert_any_call(0)
-    mock_app_env["metrics_APP_LOOP_COUNT_labels_inc"].assert_called_once()
+    mock_app_env['metrics_APP_UP_labels_set'].assert_any_call(1)
+    mock_app_env['metrics_APP_UP_labels_set'].assert_any_call(0)
+    mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].assert_called_once()
 
 
 @pytest.mark.asyncio
 async def test_run_async_main(mock_app_env, capsys):
     """Test the run_async_main function that sets up the event loop."""
-    mock_ingestor_instance = mock_app_env["Ingestor"].return_value
-    mock_exit_signal = mock_app_env["mock_exit_signal"]
+    mock_ingestor_instance = mock_app_env['Ingestor'].return_value
+    mock_exit_signal = mock_app_env['mock_exit_signal']
 
     mock_exit_signal.is_set.side_effect = [False, True]
-    mock_app_env["time_time"].side_effect = [10.0, 11.0]
+    mock_app_env['time_time'].side_effect = [10.0, 11.0]
 
     # Instead of calling run_async_main() which creates a new event loop,
     # we test the main() function directly since that's what run_async_main() would call
@@ -305,7 +284,7 @@ async def test_run_async_main(mock_app_env, capsys):
     assert excinfo.value.code == 0
 
     # Verify the main function was called through the event loop
-    mock_app_env["start_http_server"].assert_called_once_with(9090)
+    mock_app_env['start_http_server'].assert_called_once_with(9090)
     mock_ingestor_instance.prepare_ingestor.assert_called_once()
     mock_ingestor_instance.loop.assert_called_once()
     mock_ingestor_instance.shutdown.assert_called_once()
@@ -314,11 +293,10 @@ async def test_run_async_main(mock_app_env, capsys):
 @pytest.mark.asyncio
 async def test_main_prepare_ingestor_failure(mock_app_env, capsys):
     """Test that prepare_ingestor failure is handled correctly."""
-    mock_ingestor_instance = mock_app_env["Ingestor"].return_value
-    mock_exit_signal = mock_app_env["mock_exit_signal"]
+    mock_ingestor_instance = mock_app_env['Ingestor'].return_value
+    mock_exit_signal = mock_app_env['mock_exit_signal']
 
-    mock_ingestor_instance.prepare_ingestor.side_effect = Exception(
-        "Preparation failed")
+    mock_ingestor_instance.prepare_ingestor.side_effect = Exception('Preparation failed')
     mock_exit_signal.is_set.side_effect = [False, True]
 
     with pytest.raises(OsExitCalled) as excinfo:
@@ -326,8 +304,8 @@ async def test_main_prepare_ingestor_failure(mock_app_env, capsys):
     assert excinfo.value.code == 0
 
     # Verify error metrics were incremented
-    app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id="test_pod")
-    mock_app_env["metrics_APP_ERRORS_TOTAL_labels_inc"].assert_called_once()
+    app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id='test_pod')
+    mock_app_env['metrics_APP_ERRORS_TOTAL_labels_inc'].assert_called_once()
 
     # Verify exit signal was set
     mock_exit_signal.set.assert_called_once()
@@ -337,5 +315,5 @@ async def test_main_prepare_ingestor_failure(mock_app_env, capsys):
 
     # Verify error was logged
     mock_ingestor_instance.logger.error.assert_called_once_with(
-        "Failed to prepare ingestor: Preparation failed"
+        'Failed to prepare ingestor: Preparation failed'
     )

tests/unit/test_ingestor.py

@@ -1,68 +1,70 @@
-from unittest.mock import ANY, AsyncMock, MagicMock, patch, call
+from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
+
 from pytest import fixture, mark
+
 from ingestor.ingestor import Ingestor
 
 
-@patch("ingestor.ingestor.getenv")
-@patch("ingestor.ingestor.NotificationHandler")
+@patch('ingestor.ingestor.getenv')
+@patch('ingestor.ingestor.NotificationHandler')
 def test___init__(notification_handler, getenv):
     getenv.side_effect = [
-        "localhost:9092,localhost:35",  # KAFKA_SERVERS
-        "true",  # EXPORT_TO_KAFKA
-        "localhost1",       # REDIS_HOST
-        '63790',             # REDIS_PORT
-        "user",             # REDIS_USERNAME
-        "password",         # REDIS_PASSWORD
-        '100',               # LEASE_TTL
-        '200',               # HEARTBEAT_TTL
-        "localhost1",      # HOSTNAME
-        '50',                # POLL_INTERVAL
-        "localhost:27017",  # MONGODB_URL
-        "sientia",            # MONGODB_USERNAME
-        "sientia",            # MONGODB_PASSWORD
-        "sientia"             # MONGODB_DATABASE
+        'localhost:9092,localhost:35',  # KAFKA_SERVERS
+        'true',  # EXPORT_TO_KAFKA
+        'localhost1',  # REDIS_HOST
+        '63790',  # REDIS_PORT
+        'user',  # REDIS_USERNAME
+        'password',  # REDIS_PASSWORD
+        '100',  # LEASE_TTL
+        '200',  # HEARTBEAT_TTL
+        'localhost1',  # HOSTNAME
+        '50',  # POLL_INTERVAL
+        'localhost:27017',  # MONGODB_URL
+        'sientia',  # MONGODB_USERNAME
+        'sientia',  # MONGODB_PASSWORD
+        'sientia',  # MONGODB_DATABASE
     ]
 
     ingestor = Ingestor()
 
-    getenv.assert_any_call("KAFKA_SERVERS", "localhost:9092")
-    getenv.assert_any_call("REDIS_HOST", "localhost")
-    getenv.assert_any_call("REDIS_PORT", '6379')
-    getenv.assert_any_call("REDIS_USERNAME", None)
-    getenv.assert_any_call("REDIS_PASSWORD", None)
-    getenv.assert_any_call("LEASE_TTL", '10')
-    getenv.assert_any_call("HEARTBEAT_TTL", '20')
-    getenv.assert_any_call("HOSTNAME", "localhost")
-    getenv.assert_any_call("POLL_INTERVAL", '5')
+    getenv.assert_any_call('KAFKA_SERVERS', 'localhost:9092')
+    getenv.assert_any_call('REDIS_HOST', 'localhost')
+    getenv.assert_any_call('REDIS_PORT', '6379')
+    getenv.assert_any_call('REDIS_USERNAME', None)
+    getenv.assert_any_call('REDIS_PASSWORD', None)
+    getenv.assert_any_call('LEASE_TTL', '10')
+    getenv.assert_any_call('HEARTBEAT_TTL', '20')
+    getenv.assert_any_call('HOSTNAME', 'localhost')
+    getenv.assert_any_call('POLL_INTERVAL', '5')
 
-    assert ingestor.kafka_servers == ["localhost:9092", "localhost:35"]
-    assert ingestor.redis_host == "localhost1"
+    assert ingestor.kafka_servers == ['localhost:9092', 'localhost:35']
+    assert ingestor.redis_host == 'localhost1'
     assert ingestor.redis_port == 63790
-    assert ingestor.redis_username == "user"
-    assert ingestor.redis_password == "password"
+    assert ingestor.redis_username == 'user'
+    assert ingestor.redis_password == 'password'
     assert ingestor.lease_ttl == 100
     assert ingestor.heartbeat_ttl == 200
-    assert ingestor.pod_id == "localhost1"
+    assert ingestor.pod_id == 'localhost1'
     assert ingestor.poll_interval == 50
     assert ingestor.metadata == {
-        "model_id": "-",
-        "model_name": "-",
-        "workflow_name": "opc_ingestor",
-        "schema_name": "opc_ingestor",
-        "pod_id": "localhost1"
+        'model_id': '-',
+        'model_name': '-',
+        'workflow_name': 'opc_ingestor',
+        'schema_name': 'opc_ingestor',
+        'pod_id': 'localhost1',
     }
 
     notification_handler.assert_called_once_with(
-        connection_string="mongodb://sientia:sientia@localhost:27017",
-        database="sientia",
+        connection_string='mongodb://sientia:sientia@localhost:27017',
+        database='sientia',
         logger=ingestor.logger,
-        project_name="opc_ingestor"
+        project_name='opc_ingestor',
     )
 
 
 @fixture
-@patch("ingestor.ingestor.getenv")
-@patch("ingestor.ingestor.NotificationHandler")
+@patch('ingestor.ingestor.getenv')
+@patch('ingestor.ingestor.NotificationHandler')
 def ingestor(_notification_handler, _getenv):
     ing = Ingestor()
     ing.logger = MagicMock()
@@ -77,7 +79,7 @@ def ingestor_manager_started(ingestor):
         shutdown=AsyncMock(),
         update_opc_servers=AsyncMock(),
         subscribe_to_tags=AsyncMock(),
-        unsubscribe_slot=AsyncMock()
+        unsubscribe_slot=AsyncMock(),
     )
     return ingestor
 
@@ -93,23 +95,23 @@ async def test_shutdown(ingestor_manager_started):
 async def test_handle_acquired_tags_not_acquired(ingestor_manager_started):
     await ingestor_manager_started.handle_acquired_tags([])
 
-    ingestor_manager_started.logger.warning.assert_called_once_with(
-        "No slots available")
+    ingestor_manager_started.logger.warning.assert_called_once_with('No slots available')
     ingestor_manager_started.ingestor_manager.update_opc_servers.assert_not_called()
     ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_not_called()
 
 
 @mark.asyncio
 async def test_handle_acquired_tags_success(ingestor_manager_started):
-    await ingestor_manager_started.handle_acquired_tags(["tag1", "tag2"])
+    await ingestor_manager_started.handle_acquired_tags(['tag1', 'tag2'])
 
     ingestor_manager_started.logger.warning.assert_not_called()
     ingestor_manager_started.ingestor_manager.update_opc_servers.assert_called_once()
     ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_called_once_with(
-        ["tag1", "tag2"])
+        ['tag1', 'tag2']
+    )
 
 
-@patch("ingestor.ingestor.IngestorManager")
+@patch('ingestor.ingestor.IngestorManager')
 @mark.asyncio
 async def test_prepare_ingestor(ingestor_manager_mock, ingestor):
     ingestor_manager = ingestor_manager_mock.return_value
@@ -122,10 +124,10 @@ async def test_prepare_ingestor(ingestor_manager_mock, ingestor):
     ingestor_manager_mock.assert_called_once_with(
         kafka_servers=ingestor.kafka_servers,
         redis_data={
-            "host": ingestor.redis_host,
-            "port": ingestor.redis_port,
-            "username": ingestor.redis_username,
-            "password": ingestor.redis_password
+            'host': ingestor.redis_host,
+            'port': ingestor.redis_port,
+            'username': ingestor.redis_username,
+            'password': ingestor.redis_password,
         },
         lease_ttl=ingestor.lease_ttl,
         heartbeat_ttl=ingestor.heartbeat_ttl,
@@ -141,7 +143,8 @@ async def test_prepare_ingestor(ingestor_manager_mock, ingestor):
     ingestor_manager.get_slot_leases.assert_called_once()
 
     ingestor.handle_acquired_tags.assert_called_once_with(
-        ingestor_manager.get_slot_leases.return_value)
+        ingestor_manager.get_slot_leases.return_value
+    )
 
 
 def test_manage_slots_has_slots(ingestor_manager_started):
@@ -180,8 +183,7 @@ def test_manage_slots_none_available_none_available(ingestor_manager_started):
 
     ingestor_manager_started.manage_no_slots(2)
 
-    ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with(
-        1)
+    ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with(1)
 
 
 @mark.asyncio
@@ -201,8 +203,7 @@ async def test_manage_leases_available_slots_innactive_ingestors(ingestor_manage
 
     await ingestor_manager_started.manage_leases(2, 2, 5)
 
-    ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with(
-        2)
+    ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with(2)
 
     ingestor_manager_started.ingestor_manager.drop_slot_leases.assert_not_called()
 
@@ -211,9 +212,9 @@ async def test_manage_leases_available_slots_innactive_ingestors(ingestor_manage
 async def test_manage_leases_no_available_slots_extra_sltos(ingestor_manager_started):
     ingestor_manager_started.handle_acquired_tags = MagicMock()
     ingestor_manager_started.ingestor_manager.managed_tags = {
-        "tag1": "server1",
-        "tag2": "server2",
-        "tag3": "server3"
+        'tag1': 'server1',
+        'tag2': 'server2',
+        'tag3': 'server3',
     }
 
     await ingestor_manager_started.manage_leases(0, 0, 2)
@@ -221,7 +222,8 @@ async def test_manage_leases_no_available_slots_extra_sltos(ingestor_manager_sta
     ingestor_manager_started.ingestor_manager.get_slot_leases.assert_not_called()
     ingestor_manager_started.handle_acquired_tags.assert_not_called()
     ingestor_manager_started.ingestor_manager.drop_slot_leases.assert_called_once_with(
-        ["tag2", "tag3"])
+        ['tag2', 'tag3']
+    )
 
 
 @mark.asyncio
@@ -230,16 +232,15 @@ async def test_loop(ingestor_manager_started):
     ingestor_manager_started.manage_leases = AsyncMock()
     ingestor_manager_started.update_ingestor_manager = AsyncMock()
     ingestor_manager_started.ingestor_manager.managed_tags = {
-        "slot1": "server1",
-        "slot2": "server2",
-        "slot3": "server3"
+        'slot1': 'server1',
+        'slot2': 'server2',
+        'slot3': 'server3',
     }
     ingestor_manager_started.ingestor_manager.get_active_ingestors = MagicMock(
-        return_value=["ingestor1", "ingestor2"])
-    ingestor_manager_started.ingestor_manager.get_number_of_slots = MagicMock(
-        return_value=5)
-    ingestor_manager_started.ingestor_manager.get_number_of_leases = MagicMock(
-        return_value=1)
+        return_value=['ingestor1', 'ingestor2']
+    )
+    ingestor_manager_started.ingestor_manager.get_number_of_slots = MagicMock(return_value=5)
+    ingestor_manager_started.ingestor_manager.get_number_of_leases = MagicMock(return_value=1)
 
     await ingestor_manager_started.loop()
 
@@ -248,10 +249,10 @@ async def test_loop(ingestor_manager_started):
     ingestor_manager_started.ingestor_manager.get_number_of_slots.assert_called_once()
 
     ingestor_manager_started.manage_no_slots.assert_called_once_with(
-        ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value)
+        ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value
+    )
     # Explanation: 5 - 2 = 3, 3 - 1 = 2
-    ingestor_manager_started.manage_leases.assert_called_once_with(
-        4, 3, 2)
+    ingestor_manager_started.manage_leases.assert_called_once_with(4, 3, 2)
     ingestor_manager_started.ingestor_manager.update_slot_config.assert_called_once()
 
 
@@ -262,9 +263,9 @@ async def test_loop_no_managed(ingestor_manager_started):
     ingestor_manager_started.update_ingestor_manager = AsyncMock()
     ingestor_manager_started.ingestor_manager.managed_tags = {}
     ingestor_manager_started.ingestor_manager.get_active_ingestors = MagicMock(
-        return_value=["ingestor1", "ingestor2"])
-    ingestor_manager_started.ingestor_manager.get_number_of_slots = MagicMock(
-        return_value=5)
+        return_value=['ingestor1', 'ingestor2']
+    )
+    ingestor_manager_started.ingestor_manager.get_number_of_slots = MagicMock(return_value=5)
 
     await ingestor_manager_started.loop()
 
@@ -273,43 +274,34 @@ async def test_loop_no_managed(ingestor_manager_started):
     ingestor_manager_started.ingestor_manager.get_number_of_slots.assert_called_once()
 
     ingestor_manager_started.manage_no_slots.assert_called_once_with(
-        ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value)
+        ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value
+    )
     # Explanation: 5 - 2 = 3, 3 - 1 = 2
-    ingestor_manager_started.manage_leases.assert_called_once_with(
-        ANY, 3, -1)
+    ingestor_manager_started.manage_leases.assert_called_once_with(ANY, 3, -1)
     ingestor_manager_started.ingestor_manager.update_slot_config.assert_called_once()
-    ingestor_manager_started.logger.info.assert_any_call(
-        "No slots acquired in this loop")
+    ingestor_manager_started.logger.info.assert_any_call('No slots acquired in this loop')
 
 
 @mark.asyncio
 async def test_update_ingestor_manager(ingestor_manager_started):
     ingestor_manager_started.ingestor_manager.managed_tags = {
-        "slot_to_create": "new_config",
-        "slot_to_update": "new_config",
-        "slot_to_do_nothing": "old_config"
+        'slot_to_create': 'new_config',
+        'slot_to_update': 'new_config',
+        'slot_to_do_nothing': 'old_config',
     }
 
     old_managed_tags = {
-        "slot_to_update": "old_config",
-        "slot_to_delete": "old_config",
-        "slot_to_do_nothing": "old_config"
+        'slot_to_update': 'old_config',
+        'slot_to_delete': 'old_config',
+        'slot_to_do_nothing': 'old_config',
     }
 
     await ingestor_manager_started.update_ingestor_manager(old_managed_tags)
 
     ingestor_manager_started.ingestor_manager.update_opc_servers.assert_called_once()
     ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_has_calls(
-        [
-            call(
-                {"slot_to_create": "new_config"}),
-            call(
-                {"slot_to_update": "new_config"})
-        ]
+        [call({'slot_to_create': 'new_config'}), call({'slot_to_update': 'new_config'})]
     )
     ingestor_manager_started.ingestor_manager.unsubscribe_slot.assert_has_calls(
-        [
-            call("slot_to_update"),
-            call("slot_to_delete")
-        ]
+        [call('slot_to_update'), call('slot_to_delete')]
     )

tests/unit/test_metrics.py

@@ -1,7 +1,7 @@
 # tests/unit/test_metrics.py
 
-import pytest
 from prometheus_client import Counter, Gauge, Histogram
+
 import ingestor.metrics as metrics
 
 # --- Test Functions for Each Metric (Corrected for v0.22.0 _name behavior) ---
@@ -11,17 +11,16 @@ def test_ingestor_tag_written_count():
     """Verify the definition of INGESTOR_TAG_WRITTEN_COUNT."""
     assert metrics.TAG_WRITTEN_COUNT is not None
     assert isinstance(metrics.TAG_WRITTEN_COUNT, Counter)
-    assert metrics.TAG_WRITTEN_COUNT._name == "ingestor_tag_written_count"
-    assert set(metrics.TAG_WRITTEN_COUNT._labelnames) == {
-        "pod_id", "tag_name", "collection_name"}
+    assert metrics.TAG_WRITTEN_COUNT._name == 'ingestor_tag_written_count'
+    assert set(metrics.TAG_WRITTEN_COUNT._labelnames) == {'pod_id', 'tag_name', 'collection_name'}
 
 
 def test_app_loop_count():
     """Verify the definition of APP_LOOP_COUNT."""
     assert metrics.APP_LOOP_COUNT is not None
     assert isinstance(metrics.APP_LOOP_COUNT, Counter)
-    assert metrics.APP_LOOP_COUNT._name == "app_main_loop"  # REMOVED _total
-    assert set(metrics.APP_LOOP_COUNT._labelnames) == {"pod_id"}
+    assert metrics.APP_LOOP_COUNT._name == 'app_main_loop'  # REMOVED _total
+    assert set(metrics.APP_LOOP_COUNT._labelnames) == {'pod_id'}
 
 
 def test_app_loop_duration():
@@ -29,32 +28,32 @@ def test_app_loop_duration():
     assert metrics.APP_LOOP_DURATION is not None
     assert isinstance(metrics.APP_LOOP_DURATION, Histogram)
     assert (
-        metrics.APP_LOOP_DURATION._name == "app_main_loop_duration_seconds"
+        metrics.APP_LOOP_DURATION._name == 'app_main_loop_duration_seconds'
     )  # Histograms don't have _total
-    assert set(metrics.APP_LOOP_DURATION._labelnames) == {"pod_id"}
+    assert set(metrics.APP_LOOP_DURATION._labelnames) == {'pod_id'}
 
 
 def test_app_errors_total():
     """Verify the definition of APP_ERRORS_TOTAL."""
     assert metrics.APP_ERRORS_TOTAL is not None
     assert isinstance(metrics.APP_ERRORS_TOTAL, Counter)
-    assert metrics.APP_ERRORS_TOTAL._name == "app_errors"  # REMOVED _total
-    assert set(metrics.APP_ERRORS_TOTAL._labelnames) == {"pod_id"}
+    assert metrics.APP_ERRORS_TOTAL._name == 'app_errors'  # REMOVED _total
+    assert set(metrics.APP_ERRORS_TOTAL._labelnames) == {'pod_id'}
 
 
 def test_app_up():
     """Verify the definition of APP_UP."""
     assert metrics.APP_UP is not None
     assert isinstance(metrics.APP_UP, Gauge)
-    assert metrics.APP_UP._name == "app_up"  # Gauges don't have _total
-    assert set(metrics.APP_UP._labelnames) == {"pod_id"}
+    assert metrics.APP_UP._name == 'app_up'  # Gauges don't have _total
+    assert set(metrics.APP_UP._labelnames) == {'pod_id'}
 
 
 def test_active_ingestors():
     """Verify the definition of ACTIVE_INGESTORS."""
     assert metrics.ACTIVE_INGESTORS is not None
     assert isinstance(metrics.ACTIVE_INGESTORS, Gauge)
-    assert metrics.ACTIVE_INGESTORS._name == "ingestor_active_total"
+    assert metrics.ACTIVE_INGESTORS._name == 'ingestor_active_total'
     assert set(metrics.ACTIVE_INGESTORS._labelnames) == set()
 
 
@@ -62,7 +61,7 @@ def test_slots_total():
     """Verify the definition of SLOTS_TOTAL."""
     assert metrics.SLOTS_TOTAL is not None
     assert isinstance(metrics.SLOTS_TOTAL, Gauge)
-    assert metrics.SLOTS_TOTAL._name == "ingestor_slots_total"
+    assert metrics.SLOTS_TOTAL._name == 'ingestor_slots_total'
     assert set(metrics.SLOTS_TOTAL._labelnames) == set()
 
 
@@ -70,7 +69,7 @@ def test_leases_total():
     """Verify the definition of LEASES_TOTAL."""
     assert metrics.LEASES_TOTAL is not None
     assert isinstance(metrics.LEASES_TOTAL, Gauge)
-    assert metrics.LEASES_TOTAL._name == "ingestor_leases_total"
+    assert metrics.LEASES_TOTAL._name == 'ingestor_leases_total'
     assert set(metrics.LEASES_TOTAL._labelnames) == set()
 
 
@@ -78,32 +77,32 @@ def test_slots_managed():
     """Verify the definition of SLOTS_MANAGED."""
     assert metrics.SLOTS_MANAGED is not None
     assert isinstance(metrics.SLOTS_MANAGED, Gauge)
-    assert metrics.SLOTS_MANAGED._name == "ingestor_slots_managed_current"
-    assert set(metrics.SLOTS_MANAGED._labelnames) == {"pod_id"}
+    assert metrics.SLOTS_MANAGED._name == 'ingestor_slots_managed_current'
+    assert set(metrics.SLOTS_MANAGED._labelnames) == {'pod_id'}
 
 
 def test_slots_acquired():
     """Verify the definition of SLOTS_ACQUIRED."""
     assert metrics.SLOTS_ACQUIRED is not None
     assert isinstance(metrics.SLOTS_ACQUIRED, Counter)
-    assert metrics.SLOTS_ACQUIRED._name == "ingestor_slots_acquired"  # REMOVED _total
-    assert set(metrics.SLOTS_ACQUIRED._labelnames) == {"pod_id"}
+    assert metrics.SLOTS_ACQUIRED._name == 'ingestor_slots_acquired'  # REMOVED _total
+    assert set(metrics.SLOTS_ACQUIRED._labelnames) == {'pod_id'}
 
 
 def test_slots_released():
     """Verify the definition of SLOTS_RELEASED."""
     assert metrics.SLOTS_RELEASED is not None
     assert isinstance(metrics.SLOTS_RELEASED, Counter)
-    assert metrics.SLOTS_RELEASED._name == "ingestor_slots_released"  # REMOVED _total
-    assert set(metrics.SLOTS_RELEASED._labelnames) == {"pod_id"}
+    assert metrics.SLOTS_RELEASED._name == 'ingestor_slots_released'  # REMOVED _total
+    assert set(metrics.SLOTS_RELEASED._labelnames) == {'pod_id'}
 
 
 def test_opc_managers_active():
     """Verify the definition of OPC_MANAGERS_ACTIVE."""
     assert metrics.OPC_MANAGERS_ACTIVE is not None
     assert isinstance(metrics.OPC_MANAGERS_ACTIVE, Gauge)
-    assert metrics.OPC_MANAGERS_ACTIVE._name == "ingestor_opc_managers_active"
-    assert set(metrics.OPC_MANAGERS_ACTIVE._labelnames) == {"pod_id"}
+    assert metrics.OPC_MANAGERS_ACTIVE._name == 'ingestor_opc_managers_active'
+    assert set(metrics.OPC_MANAGERS_ACTIVE._labelnames) == {'pod_id'}
 
 
 def test_opc_subscription_errors():
@@ -111,12 +110,12 @@ def test_opc_subscription_errors():
     assert metrics.OPC_SUBSCRIPTION_ERRORS is not None
     assert isinstance(metrics.OPC_SUBSCRIPTION_ERRORS, Counter)
     assert (
-        metrics.OPC_SUBSCRIPTION_ERRORS._name == "ingestor_opc_subscription_errors"
+        metrics.OPC_SUBSCRIPTION_ERRORS._name == 'ingestor_opc_subscription_errors'
     )  # REMOVED _total
     assert set(metrics.OPC_SUBSCRIPTION_ERRORS._labelnames) == {
-        "pod_id",
-        "server",
-        "slot",
+        'pod_id',
+        'server',
+        'slot',
     }
 
 
@@ -124,33 +123,27 @@ def test_opc_connections_total():
     """Verify the definition of OPC_CONNECTIONS_TOTAL."""
     assert metrics.OPC_CONNECTIONS_TOTAL is not None
     assert isinstance(metrics.OPC_CONNECTIONS_TOTAL, Counter)
-    assert (
-        metrics.OPC_CONNECTIONS_TOTAL._name == "opc_connections_initiated"
-    )  # REMOVED _total
-    assert set(metrics.OPC_CONNECTIONS_TOTAL._labelnames) == {
-        "pod_id", "server_name"}
+    assert metrics.OPC_CONNECTIONS_TOTAL._name == 'opc_connections_initiated'  # REMOVED _total
+    assert set(metrics.OPC_CONNECTIONS_TOTAL._labelnames) == {'pod_id', 'server_name'}
 
 
 def test_opc_connections_failed():
     """Verify the definition of OPC_CONNECTIONS_FAILED."""
     assert metrics.OPC_CONNECTIONS_FAILED is not None
     assert isinstance(metrics.OPC_CONNECTIONS_FAILED, Counter)
-    assert (
-        metrics.OPC_CONNECTIONS_FAILED._name == "opc_connections_failed"
-    )  # REMOVED _total
-    assert set(metrics.OPC_CONNECTIONS_FAILED._labelnames) == {
-        "pod_id", "server_name"}
+    assert metrics.OPC_CONNECTIONS_FAILED._name == 'opc_connections_failed'  # REMOVED _total
+    assert set(metrics.OPC_CONNECTIONS_FAILED._labelnames) == {'pod_id', 'server_name'}
 
 
 def test_opc_connection_status():
     """Verify the definition of OPC_CONNECTION_STATUS."""
     assert metrics.OPC_CONNECTION_STATUS is not None
     assert isinstance(metrics.OPC_CONNECTION_STATUS, Gauge)
-    assert metrics.OPC_CONNECTION_STATUS._name == "opc_connection_status"
+    assert metrics.OPC_CONNECTION_STATUS._name == 'opc_connection_status'
     assert set(metrics.OPC_CONNECTION_STATUS._labelnames) == {
-        "pod_id",
-        "server_name",
-        "server_url",
+        'pod_id',
+        'server_name',
+        'server_url',
     }
 
 
@@ -158,13 +151,11 @@ def test_opc_subscriptions_created():
     """Verify the definition of OPC_SUBSCRIPTIONS_CREATED."""
     assert metrics.OPC_SUBSCRIPTIONS_CREATED is not None
     assert isinstance(metrics.OPC_SUBSCRIPTIONS_CREATED, Counter)
-    assert (
-        metrics.OPC_SUBSCRIPTIONS_CREATED._name == "opc_subscriptions_created"
-    )  # REMOVED _total
+    assert metrics.OPC_SUBSCRIPTIONS_CREATED._name == 'opc_subscriptions_created'  # REMOVED _total
     assert set(metrics.OPC_SUBSCRIPTIONS_CREATED._labelnames) == {
-        "pod_id",
-        "server_name",
-        "slot_name",
+        'pod_id',
+        'server_name',
+        'slot_name',
     }
 
 
@@ -172,101 +163,85 @@ def test_opc_tags_subscribed():
     """Verify the definition of OPC_TAGS_SUBSCRIBED."""
     assert metrics.OPC_TAGS_SUBSCRIBED is not None
     assert isinstance(metrics.OPC_TAGS_SUBSCRIBED, Gauge)
-    assert metrics.OPC_TAGS_SUBSCRIBED._name == "opc_tags_subscribed_current"
-    assert set(metrics.OPC_TAGS_SUBSCRIBED._labelnames) == {
-        "pod_id", "server_name"}
+    assert metrics.OPC_TAGS_SUBSCRIBED._name == 'opc_tags_subscribed_current'
+    assert set(metrics.OPC_TAGS_SUBSCRIBED._labelnames) == {'pod_id', 'server_name'}
 
 
 def test_opc_cycles_without_data():
     """Verify the definition of OPC_CYCLES_WITHOUT_DATA."""
     assert metrics.OPC_CYCLES_WITHOUT_DATA is not None
     assert isinstance(metrics.OPC_CYCLES_WITHOUT_DATA, Gauge)
-    assert metrics.OPC_CYCLES_WITHOUT_DATA._name == "opc_cycles_without_data"
-    assert set(metrics.OPC_CYCLES_WITHOUT_DATA._labelnames) == {
-        "pod_id", "server_name"}
+    assert metrics.OPC_CYCLES_WITHOUT_DATA._name == 'opc_cycles_without_data'
+    assert set(metrics.OPC_CYCLES_WITHOUT_DATA._labelnames) == {'pod_id', 'server_name'}
 
 
 def test_opc_reconnections_total():
     """Verify the definition of OPC_RECONNECTIONS_TOTAL."""
     assert metrics.OPC_RECONNECTIONS_TOTAL is not None
     assert isinstance(metrics.OPC_RECONNECTIONS_TOTAL, Counter)
-    assert (
-        metrics.OPC_RECONNECTIONS_TOTAL._name == "opc_reconnections_tried"
-    )  # REMOVED _total
-    assert set(metrics.OPC_RECONNECTIONS_TOTAL._labelnames) == {
-        "pod_id", "server_name"}
+    assert metrics.OPC_RECONNECTIONS_TOTAL._name == 'opc_reconnections_tried'  # REMOVED _total
+    assert set(metrics.OPC_RECONNECTIONS_TOTAL._labelnames) == {'pod_id', 'server_name'}
 
 
 def test_kafka_messages_sent():
     """Verify the definition of KAFKA_MESSAGES_SENT."""
     assert metrics.KAFKA_MESSAGES_SENT is not None
     assert isinstance(metrics.KAFKA_MESSAGES_SENT, Counter)
-    assert metrics.KAFKA_MESSAGES_SENT._name == "kafka_messages_sent"  # REMOVED _total
-    assert set(metrics.KAFKA_MESSAGES_SENT._labelnames) == {"pod_id", "topic"}
+    assert metrics.KAFKA_MESSAGES_SENT._name == 'kafka_messages_sent'  # REMOVED _total
+    assert set(metrics.KAFKA_MESSAGES_SENT._labelnames) == {'pod_id', 'topic'}
 
 
 def test_kafka_messages_errors():
     """Verify the definition of KAFKA_MESSAGES_ERRORS."""
     assert metrics.KAFKA_MESSAGES_ERRORS is not None
     assert isinstance(metrics.KAFKA_MESSAGES_ERRORS, Counter)
-    assert (
-        metrics.KAFKA_MESSAGES_ERRORS._name == "kafka_messages_errors"
-    )  # REMOVED _total
-    assert set(metrics.KAFKA_MESSAGES_ERRORS._labelnames) == {
-        "pod_id", "topic"}
+    assert metrics.KAFKA_MESSAGES_ERRORS._name == 'kafka_messages_errors'  # REMOVED _total
+    assert set(metrics.KAFKA_MESSAGES_ERRORS._labelnames) == {'pod_id', 'topic'}
 
 
 def test_kafka_connection_status():
     """Verify the definition of KAFKA_CONNECTION_STATUS."""
     assert metrics.KAFKA_CONNECTION_STATUS is not None
     assert isinstance(metrics.KAFKA_CONNECTION_STATUS, Gauge)
-    assert metrics.KAFKA_CONNECTION_STATUS._name == "kafka_connection_status"
-    assert set(metrics.KAFKA_CONNECTION_STATUS._labelnames) == {"pod_id"}
+    assert metrics.KAFKA_CONNECTION_STATUS._name == 'kafka_connection_status'
+    assert set(metrics.KAFKA_CONNECTION_STATUS._labelnames) == {'pod_id'}
 
 
 def test_redis_operations_total():
     """Verify the definition of REDIS_OPERATIONS_TOTAL."""
     assert metrics.REDIS_OPERATIONS_TOTAL is not None
     assert isinstance(metrics.REDIS_OPERATIONS_TOTAL, Counter)
-    assert metrics.REDIS_OPERATIONS_TOTAL._name == "redis_operations"  # REMOVED _total
-    assert set(metrics.REDIS_OPERATIONS_TOTAL._labelnames) == {
-        "pod_id", "operation"}
+    assert metrics.REDIS_OPERATIONS_TOTAL._name == 'redis_operations'  # REMOVED _total
+    assert set(metrics.REDIS_OPERATIONS_TOTAL._labelnames) == {'pod_id', 'operation'}
 
 
 def test_redis_operations_errors():
     """Verify the definition of REDIS_OPERATIONS_ERRORS."""
     assert metrics.REDIS_OPERATIONS_ERRORS is not None
     assert isinstance(metrics.REDIS_OPERATIONS_ERRORS, Counter)
-    assert (
-        metrics.REDIS_OPERATIONS_ERRORS._name == "redis_operations_errors"
-    )  # REMOVED _total
-    assert set(metrics.REDIS_OPERATIONS_ERRORS._labelnames) == {
-        "pod_id", "operation"}
+    assert metrics.REDIS_OPERATIONS_ERRORS._name == 'redis_operations_errors'  # REMOVED _total
+    assert set(metrics.REDIS_OPERATIONS_ERRORS._labelnames) == {'pod_id', 'operation'}
 
 
 def test_redis_operations_duration():
     """Verify the definition of REDIS_OPERATIONS_DURATION."""
     assert metrics.REDIS_OPERATIONS_DURATION is not None
     assert isinstance(metrics.REDIS_OPERATIONS_DURATION, Histogram)
-    assert (
-        metrics.REDIS_OPERATIONS_DURATION._name == "redis_operations_duration_seconds"
-    )
-    assert set(metrics.REDIS_OPERATIONS_DURATION._labelnames) == {
-        "pod_id", "operation"}
+    assert metrics.REDIS_OPERATIONS_DURATION._name == 'redis_operations_duration_seconds'
+    assert set(metrics.REDIS_OPERATIONS_DURATION._labelnames) == {'pod_id', 'operation'}
 
 
 def test_redis_connection_status():
     """Verify the definition of REDIS_CONNECTION_STATUS."""
     assert metrics.REDIS_CONNECTION_STATUS is not None
     assert isinstance(metrics.REDIS_CONNECTION_STATUS, Gauge)
-    assert metrics.REDIS_CONNECTION_STATUS._name == "redis_connection_status"
-    assert set(metrics.REDIS_CONNECTION_STATUS._labelnames) == {"pod_id"}
+    assert metrics.REDIS_CONNECTION_STATUS._name == 'redis_connection_status'
+    assert set(metrics.REDIS_CONNECTION_STATUS._labelnames) == {'pod_id'}
 
 
 def test_notifications_sent():
     """Verify the definition of NOTIFICATIONS_SENT."""
     assert metrics.NOTIFICATIONS_SENT is not None
     assert isinstance(metrics.NOTIFICATIONS_SENT, Counter)
-    assert metrics.NOTIFICATIONS_SENT._name == "notifications_sent"  # REMOVED _total
-    assert set(metrics.NOTIFICATIONS_SENT._labelnames) == {
-        "pod_id", "level", "block"}
+    assert metrics.NOTIFICATIONS_SENT._name == 'notifications_sent'  # REMOVED _total
+    assert set(metrics.NOTIFICATIONS_SENT._labelnames) == {'pod_id', 'level', 'block'}

validate.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+# Model Manager Code Validation Script
+# This script runs all code quality checks before committing or deploying
+
+set -e  # Exit on any error
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Args
+FIX_MODE=false
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --fix)
+            FIX_MODE=true
+            shift
+            ;;
+        -h|--help)
+            echo "Usage: $0 [--fix]"
+            echo "  --fix   Apply Ruff auto-fixes (format and lint fixes)."
+            exit 0
+            ;;
+        *)
+            echo -e "${RED}Unknown option: $1${NC}"
+            echo "Usage: $0 [--fix]"
+            exit 2
+            ;;
+    esac
+done
+
+echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
+echo -e "${BLUE}║  Model Manager - Code Validation Suite                ║${NC}"
+echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
+echo ""
+
+# Check if virtual environment is activated
+if [[ -z "${VIRTUAL_ENV}" ]] && [[ -z "${CONDA_DEFAULT_ENV}" ]]; then
+    echo -e "${YELLOW}⚠️  Warning: No virtual environment detected${NC}"
+    echo -e "${YELLOW}   Consider activating your venv/conda environment${NC}"
+    echo ""
+fi
+
+# Function to run a validation step
+run_step() {
+    local step_name=$1
+    local step_command=$2
+    
+    echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+    echo -e "${BLUE}▶ ${step_name}${NC}"
+    echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+    
+    if eval "$step_command"; then
+        echo -e "${GREEN}✅ ${step_name} - PASSED${NC}"
+        echo ""
+        return 0
+    else
+        echo -e "${RED}❌ ${step_name} - FAILED${NC}"
+        echo ""
+        return 1
+    fi
+}
+
+# Track failures
+FAILED_STEPS=()
+
+# Step 1: Code Formatting Check (Ruff)
+# - default: check only
+# - --fix: write changes
+if ! run_step "1. Code Formatting (Ruff)" "if \$FIX_MODE; then ruff format ingestor/ tests/; else ruff format --check ingestor/ tests/; fi"; then
+    FAILED_STEPS+=("Code Formatting")
+fi
+
+# Step 2: Linting (Ruff)
+# - default: check only
+# - --fix: apply autofixes
+if ! run_step "2. Code Linting (Ruff)" "if \$FIX_MODE; then ruff check --fix ingestor/ tests/; else ruff check ingestor/ tests/; fi"; then
+    FAILED_STEPS+=("Linting")
+fi
+
+# Step 3: Type Checking (mypy)
+if ! run_step "3. Type Checking (mypy)" "mypy ingestor/"; then
+    FAILED_STEPS+=("Type Checking")
+fi
+
+# Step 4: Security Analysis (Bandit)
+if ! run_step "4. Security Analysis (Bandit)" "bandit -r ingestor/ -ll -q"; then
+    FAILED_STEPS+=("Security Analysis")
+fi
+
+# Step 5: Unit Tests (pytest)
+if ! run_step "5. Unit Tests (pytest)" "pytest tests/ --cov=ingestor --cov-report=term-missing --cov-report=xml --cov-report=html --cov-fail-under=80 -q"; then
+    FAILED_STEPS+=("Unit Tests")
+fi
+
+# Summary
+echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
+echo -e "${BLUE}║  Validation Summary                                    ║${NC}"
+echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
+echo ""
+
+if [ ${#FAILED_STEPS[@]} -eq 0 ]; then
+    echo -e "${GREEN}✅ All validation checks passed!${NC}"
+    echo -e "${GREEN}   Your code is ready for commit/deployment.${NC}"
+    echo ""
+    exit 0
+else
+    echo -e "${RED}❌ Validation failed for the following steps:${NC}"
+    for step in "${FAILED_STEPS[@]}"; do
+        echo -e "${RED}   • ${step}${NC}"
+    done
+    echo ""
+    echo -e "${YELLOW}💡 Tips:${NC}"
+    echo -e "${YELLOW}   • Run 'ruff format ingestor/ tests/' to auto-fix formatting${NC}"
+    echo -e "${YELLOW}   • Run 'ruff check --fix ingestor/ tests/' to auto-fix linting issues${NC}"
+    echo -e "${YELLOW}   • Review mypy errors and add type hints where needed${NC}"
+    echo -e "${YELLOW}   • Check bandit warnings for security issues${NC}"
+    echo -e "${YELLOW}   • Fix failing tests or improve test coverage${NC}"
+    echo ""
+    exit 1
+fi