diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 3d04200..8265ff4 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -1,5 +1,6 @@ name: Quality gate + on: push: branches: @@ -19,6 +20,30 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: false + + - name: Generate App Token + id: generate-app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + owner: 'Aignosi' + repositories: 'sientia-dataops-library,sientia-mlops-library' + + - name: Prepare requirements.txt + id: prepare-requirements + run: | + sed -e "s|git+ssh://git@github.com/|git+https://github.com/|g" \ + -e "s|git@github.com:|git+https://github.com/|g" \ + requirements.txt > requirements_prepared.txt + echo "PROCESSED_REQUIREMENTS_FILE=requirements_prepared.txt" >> $GITHUB_OUTPUT + + - name: Configure Git to use App Token + env: + GH_APP_TOKEN: ${{ steps.generate-app-token.outputs.token }} + run: | + git config --global url."https://oauth2:${GH_APP_TOKEN}@github.com/".insteadOf "https://github.com/" - name: ๐Ÿ”ง Setup Python uses: actions/setup-python@v4 @@ -29,25 +54,42 @@ jobs: uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles(steps.prepare-requirements.outputs.PROCESSED_REQUIREMENTS_FILE) }} restore-keys: | ${{ runner.os }}-pip- - - name: Configure SSH - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - - name: Add github.com to known_hosts - run: | - mkdir -p ~/.ssh - ssh-keyscan github.com >> ~/.ssh/known_hosts - - - name: ๐Ÿ“ฆ Install Dependencies + - name: ๐Ÿ“ฆ Install Development Dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pytest pytest-cov pytest-asyncio + pip install -r requirements-dev.txt + + - name: ๐Ÿ“ฆ Install Runtime Dependencies + run: | + pip install -r ${{ steps.prepare-requirements.outputs.PROCESSED_REQUIREMENTS_FILE }} + + - name: ๐Ÿ“ Code Formatting Check (Ruff) + run: | + echo "Checking code formatting..." + ruff format --check ingestor/ tests/ + continue-on-error: false + + - name: ๐Ÿ”Ž Code Linting (Ruff) + run: | + echo "Running linting checks..." + ruff check ingestor/ tests/ + continue-on-error: false + + - name: ๐Ÿท๏ธ Type Checking (mypy) + run: | + echo "Running type checks..." + mypy ingestor/ + continue-on-error: true + + - name: ๐Ÿ”’ Security Analysis (Bandit) + run: | + echo "Running security analysis..." + bandit -r ingestor/ -ll -q + continue-on-error: true - name: ๐Ÿงช Run Tests with Pytest run: | diff --git a/.gitignore b/.gitignore index 0d8a542..198783f 100644 --- a/.gitignore +++ b/.gitignore @@ -177,4 +177,8 @@ cython_debug/ # VSCode .vscode/ -git_log \ No newline at end of file +git_log + +.git/ +.ruff_cache/ +.mypy_cache/ \ No newline at end of file diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/encrypt.py b/encrypt.py new file mode 100644 index 0000000..6aff252 --- /dev/null +++ b/encrypt.py @@ -0,0 +1,112 @@ +import os +import argparse +from pathspec import PathSpec +import yaml + +''' +Usage: + python .\encrypt.py path_to_dir output_file --ignore ignore_file --chunk-size 100000 +''' + + +def load_ignore_patterns(ignore_file, include_library): + # Ensure the .gitignore file exists + if not os.path.exists(ignore_file): + raise FileNotFoundError(f"Ignore file not found at {ignore_file}") + + # Load and parse the .gitignore patterns + with open(ignore_file, 'r') as file: + patterns = file.readlines() + if not include_library: + patterns.append('**/deploy/library/') + + spec = PathSpec.from_lines('gitwildmatch', patterns) + return spec + + +def is_ignored(file_path, spec): + """Check if a file should be ignored based on the ignore patterns.""" + return spec.match_file(file_path) if spec else False + + +def encode_file_tree_to_yaml(directory, ignore_file, include_library): + """Encode the file tree into a single YAML file.""" + ignore_patterns = load_ignore_patterns( + ignore_file, include_library) if ignore_file else None + file_tree = {} + + for root, dirs, files in os.walk(directory): + # Skip ignored directories + dirs[:] = [d for d in dirs if not is_ignored( + os.path.join(root, d), ignore_patterns)] + + for file in files: + file_path = os.path.join(root, file) + + # Skip ignored files + if is_ignored(file_path, ignore_patterns): + continue + + # Read file content + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + except Exception as e: + print(f"Error reading file {file_path}: {e}") + raise + + # Create nested dictionary structure + path_parts = os.path.relpath(file_path, directory).split(os.sep) + current_level = file_tree + + # all except the last part (the file name) + for part in path_parts[:-1]: + current_level = current_level.setdefault(part, {}) + + # Add the file and its content + current_level[path_parts[-1]] = content + return yaml.dump(file_tree, default_flow_style=False) + + +def chunk_and_write_file_tree_to_yaml(yaml_content, output_file, chunk_size=None): + """Chunk the YAML content and write it to the output file.""" + + chunks = [yaml_content] if chunk_size is None else [ + yaml_content[i:i + chunk_size] for i in range(0, len(yaml_content), chunk_size)] + + for i, chunk in enumerate(chunks): + chunk_file = f"{output_file}_{i}.yaml" + # Write the file tree to the output YAML file + with open(chunk_file, 'w', encoding='utf-8') as yaml_file: + yaml_file.write(chunk) + + +def main(): + parser = argparse.ArgumentParser( + description="Encrypts file tree to yaml file") + parser.add_argument("input_directory", help="Directory to encode") + parser.add_argument("output_yaml_file", help="Output YAML file") + parser.add_argument("--ignore", default=None, + help="Path to the ignore file") + parser.add_argument("--chunk-size", type=int, default=None, + help="Chunk size for the output YAML file") + parser.add_argument("--library", type=bool, default=False, + help="Incude the library in the output YAML file") + + # Parse arguments + args = parser.parse_args() + + # Example usage + directory_to_encode = args.input_directory + ignore_file_path = args.ignore + output_yaml_file = args.output_yaml_file + include_library = args.library + + content = encode_file_tree_to_yaml( + directory_to_encode, ignore_file_path, include_library) + chunk_and_write_file_tree_to_yaml( + content, output_yaml_file, args.chunk_size) + + +if __name__ == "__main__": + main() diff --git a/ingestor/app.py b/ingestor/app.py index 4505657..36902f6 100644 --- a/ingestor/app.py +++ b/ingestor/app.py @@ -1,9 +1,9 @@ -import os import asyncio +import os import signal import traceback from threading import Event -from time import sleep, time +from time import time from prometheus_client import start_http_server @@ -11,10 +11,10 @@ import ingestor.metrics as metrics from ingestor.ingestor import Ingestor exit_signal = Event() -POD_ID = os.getenv("HOSTNAME", "localhost") +POD_ID = os.getenv('HOSTNAME', 'localhost') -async def main(): +async def main(): # NOSONAR """ Main asynchronous function that orchestrates the OPC Ingestor application. @@ -43,30 +43,27 @@ async def main(): try: await ingestor.prepare_ingestor() except Exception as e: - metrics.APP_ERRORS_TOTAL.labels( - pod_id=POD_ID).inc() # Increment errors - ingestor.logger.error(f"Failed to prepare ingestor: {e}") + metrics.APP_ERRORS_TOTAL.labels(pod_id=POD_ID).inc() # Increment errors + ingestor.logger.error(f'Failed to prepare ingestor: {e}') exit_signal.set() - ingestor.logger.info("Ingestor prepared. Starting main loop.") + ingestor.logger.info('Ingestor prepared. Starting main loop.') - while not exit_signal.is_set(): + while not exit_signal.is_set(): # NOSONAR start_time = time() # Start loop timer try: await ingestor.loop() - metrics.APP_LOOP_COUNT.labels( - pod_id=POD_ID).inc() # Increment loop counter + metrics.APP_LOOP_COUNT.labels(pod_id=POD_ID).inc() # Increment loop counter # Use asyncio.sleep instead of exit_signal.wait for better async compatibility - await asyncio.sleep(ingestor.poll_interval) + await asyncio.sleep(ingestor.poll_interval) # NOSONAR except KeyboardInterrupt: # Handle Ctrl+C gracefully - print("KeyboardInterrupt received. Setting exit_signal flag.") + print('KeyboardInterrupt received. Setting exit_signal flag.') exit_signal.set() except Exception: - print("Exception in main loop. Setting exit_signal flag.") + print('Exception in main loop. Setting exit_signal flag.') traceback.print_exc() - metrics.APP_ERRORS_TOTAL.labels( - pod_id=POD_ID).inc() # Increment errors + metrics.APP_ERRORS_TOTAL.labels(pod_id=POD_ID).inc() # Increment errors exit_signal.set() finally: # Record loop duration @@ -76,7 +73,7 @@ async def main(): await ingestor.shutdown() metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN - ingestor.logger.info("Main loop exit_signaled.") + ingestor.logger.info('Main loop exit_signaled.') # Give Prometheus a chance to scrape one last time before exiting (optional) await asyncio.sleep(5) @@ -96,7 +93,7 @@ def signal_handler(_signum, _frame): _signum: The signal number received _frame: The current stack frame (unused) """ - print(f"Received signal {_signum}. Setting exit_signal flag.") + print(f'Received signal {_signum}. Setting exit_signal flag.') exit_signal.set() @@ -115,12 +112,12 @@ def start_prometheus_server(): Exception: If the server fails to start, the application will exit """ try: - port = int(os.getenv("HTTP_METRICS_PORT", 9090)) + port = int(os.getenv('HTTP_METRICS_PORT', 9090)) start_http_server(port) - print(f"Prometheus server started on port {port}.") + print(f'Prometheus server started on port {port}.') metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP except Exception as e: - print(f"Failed to start Prometheus server: {e}") + print(f'Failed to start Prometheus server: {e}') os._exit(1) @@ -139,13 +136,13 @@ def run_async_main(): asyncio.set_event_loop(loop) loop.run_until_complete(main()) except KeyboardInterrupt: - print("KeyboardInterrupt received in main thread.") + print('KeyboardInterrupt received in main thread.') exit_signal.set() finally: loop.close() -if __name__ == "__main__": +if __name__ == '__main__': signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGHUP, signal_handler) diff --git a/ingestor/ingestor.py b/ingestor/ingestor.py index 92a7c72..662f539 100644 --- a/ingestor/ingestor.py +++ b/ingestor/ingestor.py @@ -1,14 +1,12 @@ -import asyncio -from os import getenv from copy import deepcopy -from typing import Dict, Any +from os import getenv +from typing import Any from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import get_logger -from ingestor.managers.ingestor_manager import IngestorManager - import ingestor.metrics as metrics +from ingestor.managers.ingestor_manager import IngestorManager class Ingestor: @@ -73,36 +71,31 @@ class Ingestor: - Metrics collection and monitoring """ - kafka_servers = getenv("KAFKA_SERVERS", "localhost:9092") - export_to_kafka = getenv("EXPORT_TO_KAFKA", "false") - - if export_to_kafka and export_to_kafka == "true": - export_to_kafka = True - else: - export_to_kafka = False + kafka_servers = getenv('KAFKA_SERVERS', 'localhost:9092') + export_to_kafka: bool = getenv('EXPORT_TO_KAFKA', 'false') == 'true' self.export_to_kafka = export_to_kafka - self.redis_host = getenv("REDIS_HOST", "localhost") - self.redis_port = int(getenv("REDIS_PORT", "6379")) - self.redis_username = getenv("REDIS_USERNAME", None) - self.redis_password = getenv("REDIS_PASSWORD", None) - self.lease_ttl = int(getenv("LEASE_TTL", "10")) - self.heartbeat_ttl = int(getenv("HEARTBEAT_TTL", "20")) - self.pod_id = getenv("HOSTNAME", "localhost") - self.poll_interval = int(getenv("POLL_INTERVAL", "5")) - mongo_url = getenv("MONGODB_URL", "localhost:27017") - mongo_username = getenv("MONGODB_USERNAME", "sientia") - mongo_password = getenv("MONGODB_PASSWORD", "sientia") - self.mongo_database = getenv("MONGODB_DATABASE", "sientia") - self.mongo_connection_string = f"mongodb://{mongo_username}:{mongo_password}@{mongo_url}" + self.redis_host = getenv('REDIS_HOST', 'localhost') + self.redis_port = int(getenv('REDIS_PORT', '6379')) + self.redis_username = getenv('REDIS_USERNAME', None) + self.redis_password = getenv('REDIS_PASSWORD', None) + self.lease_ttl = int(getenv('LEASE_TTL', '10')) + self.heartbeat_ttl = int(getenv('HEARTBEAT_TTL', '20')) + self.pod_id = getenv('HOSTNAME', 'localhost') + self.poll_interval = int(getenv('POLL_INTERVAL', '5')) + mongo_url = getenv('MONGODB_URL', 'localhost:27017') + mongo_username = getenv('MONGODB_USERNAME', 'sientia') + mongo_password = getenv('MONGODB_PASSWORD', 'sientia') + self.mongo_database = getenv('MONGODB_DATABASE', 'sientia') + self.mongo_connection_string = f'mongodb://{mongo_username}:{mongo_password}@{mongo_url}' - self.kafka_servers = kafka_servers.split(",") + self.kafka_servers = kafka_servers.split(',') self.logger = get_logger(__name__) self.notification_handler = NotificationHandler( connection_string=self.mongo_connection_string, database=self.mongo_database, logger=self.logger, - project_name="opc_ingestor" + project_name='opc_ingestor', ) self.metadata = { @@ -112,7 +105,7 @@ class Ingestor: 'schema_name': 'opc_ingestor', 'pod_id': self.pod_id, } - self.ingestor_manager = None + self.ingestor_manager: IngestorManager | None = None async def shutdown(self): """ @@ -148,12 +141,13 @@ class Ingestor: """ if not acquired: - self.logger.warning("No slots available") + self.logger.warning('No slots available') else: # Subscribe to acquired slots - self.ingestor_manager.update_opc_servers() - await self.ingestor_manager.subscribe_to_tags(acquired) + if self.ingestor_manager: + self.ingestor_manager.update_opc_servers() + await self.ingestor_manager.subscribe_to_tags(acquired) async def prepare_ingestor(self): """ @@ -175,7 +169,7 @@ class Ingestor: """ self.ingestor_manager = IngestorManager( - kafka_servers=self.kafka_servers, + kafka_servers=','.join(self.kafka_servers), redis_data={ 'host': self.redis_host, 'port': self.redis_port, @@ -192,13 +186,14 @@ class Ingestor: notification_handler=self.notification_handler, export_to_kafka=self.export_to_kafka, ) + assert self.ingestor_manager is not None # Declare ingestor active self.ingestor_manager.declare_active() # Get slot lease acquired = self.ingestor_manager.get_slot_leases() - self.logger.info(f"Acquired slots: {acquired}") + self.logger.info(f'Acquired slots: {acquired}') await self.handle_acquired_tags(acquired) @@ -223,15 +218,13 @@ class Ingestor: - Requests a single slot lease to begin processing """ - if not self.ingestor_manager.managed_tags and number_of_slots > 0: + if self.ingestor_manager and not self.ingestor_manager.managed_tags and number_of_slots > 0: # This ingestor is active and has no slots, so we need to try to # Get slot lease self.ingestor_manager.get_slot_leases(1) - async def manage_leases( - self, available_slots: int, lacking_ingestors: int, slot_diff: int - ): + async def manage_leases(self, available_slots: int, lacking_ingestors: int, slot_diff: int): """ Manages the allocation and deallocation of slot leases for ingestors. @@ -254,17 +247,18 @@ class Ingestor: - Logs the number of available slots when attempting to acquire leases. - Logs the number of extra slots when releasing leases. """ + if not self.ingestor_manager: + return if available_slots > 0 and lacking_ingestors > 0: # Some ingestors are inactive, so there are "available_slots" slots available - self.logger.info(f"Slots available: {available_slots}") + self.logger.info(f'Slots available: {available_slots}') # Get slot lease self.ingestor_manager.get_slot_leases(available_slots) elif lacking_ingestors <= 0 and slot_diff > 0: - - self.logger.info(f"Extra slots available: {slot_diff}") + self.logger.info(f'Extra slots available: {slot_diff}') # There's enough slots for all ingestors, but this ingestor has more than one slot # So we need to drop the extra leases @@ -281,7 +275,7 @@ class Ingestor: len(self.ingestor_manager.managed_tags) ) - async def update_ingestor_manager(self, old_managed_tags: Dict[str, Any]): + async def update_ingestor_manager(self, old_managed_tags: dict[str, Any]): """ Updates the ingestor manager with new managed tags and handles configuration changes. @@ -302,37 +296,42 @@ class Ingestor: - Updates metrics to reflect current state """ - self.logger.debug( - f"Current managed tags: {self.ingestor_manager.managed_tags}") + if not self.ingestor_manager: + return + + self.logger.debug(f'Current managed tags: {self.ingestor_manager.managed_tags}') await self.ingestor_manager.update_opc_servers() new_managed_tags = deepcopy(self.ingestor_manager.managed_tags) self.logger.debug( - f"Comparing new managed tags {new_managed_tags} with old managed tags {old_managed_tags}" + f'Comparing new managed tags {new_managed_tags} with old managed tags {old_managed_tags}' ) keys = set(new_managed_tags) | set(old_managed_tags) - changes = {k: (new_managed_tags.get(k), old_managed_tags.get(k)) - for k in keys if new_managed_tags.get(k) != old_managed_tags.get(k)} + changes = { + k: (new_managed_tags.get(k), old_managed_tags.get(k)) + for k in keys + if new_managed_tags.get(k) != old_managed_tags.get(k) + } - self.logger.debug(f"Changes: {changes}") + self.logger.debug(f'Changes: {changes}') for slot, config in new_managed_tags.items(): if slot not in old_managed_tags: - self.logger.info(f"Subscribing to new slot {slot}") + self.logger.info(f'Subscribing to new slot {slot}') await self.ingestor_manager.subscribe_to_tags({slot: config}) continue if config != old_managed_tags[slot]: - self.logger.info(f"Resubscribing to slot {slot}") + self.logger.info(f'Resubscribing to slot {slot}') await self.ingestor_manager.unsubscribe_slot(slot) await self.ingestor_manager.subscribe_to_tags({slot: config}) for slot in old_managed_tags.keys(): if slot not in new_managed_tags: - self.logger.info(f"Unsubscribing from slot {slot}") + self.logger.info(f'Unsubscribing from slot {slot}') await self.ingestor_manager.unsubscribe_slot(slot) # Ensure the gauge is updated after any potential changes here @@ -364,9 +363,12 @@ class Ingestor: and OPC servers. """ + if not self.ingestor_manager: + return + self.ingestor_manager.declare_active() - self.logger.info("Polling for slot updates...") + self.logger.info('Polling for slot updates...') # Get active ingestors current_managed_tags = deepcopy(self.ingestor_manager.managed_tags) @@ -380,14 +382,14 @@ class Ingestor: metrics.ACTIVE_INGESTORS.set(number_of_ingestors) # Handle no slots - self.logger.info("Managing no slots...") + self.logger.info('Managing no slots...') self.manage_no_slots(number_of_slots) available_slots = number_of_slots - number_of_leases lacking_ingestors = number_of_slots - number_of_ingestors slot_diff = len(self.ingestor_manager.managed_tags) - 1 - self.logger.info("Managing leases...") + self.logger.info('Managing leases...') await self.manage_leases(available_slots, lacking_ingestors, slot_diff) # Update managed slots gauge @@ -396,23 +398,23 @@ class Ingestor: ) self.logger.debug( - f"Active ingestors: {ingestors}, " - f"Number of slots: {number_of_slots}, " - f"Number of leases: {number_of_leases}, " - f"Managed tags: {self.ingestor_manager.managed_tags}, " - f"Managed servers: {self.ingestor_manager.opc_managers}" + f'Active ingestors: {ingestors}, ' + f'Number of slots: {number_of_slots}, ' + f'Number of leases: {number_of_leases}, ' + f'Managed tags: {self.ingestor_manager.managed_tags}, ' + f'Managed servers: {self.ingestor_manager.opc_managers}' ) if not self.ingestor_manager.managed_tags: # No slots acquired - self.logger.info("No slots acquired in this loop") + self.logger.info('No slots acquired in this loop') # Update opc servers - self.logger.info("Updating slot config...") + self.logger.info('Updating slot config...') self.ingestor_manager.update_slot_config() # Check OPC cycles - self.logger.info("Checking OPC servers integrity...") + self.logger.info('Checking OPC servers integrity...') self.ingestor_manager.check_opc_servers_integrity() - self.logger.info("Updating managed tags...") + self.logger.info('Updating managed tags...') await self.update_ingestor_manager(current_managed_tags) diff --git a/ingestor/managers/data_manager.py b/ingestor/managers/data_manager.py index e186321..8a9d970 100644 --- a/ingestor/managers/data_manager.py +++ b/ingestor/managers/data_manager.py @@ -1,16 +1,18 @@ import json +import os +import traceback from time import sleep -from pymongo import MongoClient + from kafka import KafkaProducer from kafka.errors import NoBrokersAvailable +from pymongo import MongoClient from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel -from sientia_do.temporal.constants import now -from sientia_do.temporal.activities.base import BaseActivity from sientia_do.observability.logger import Logger -import traceback +from sientia_do.temporal.activities.base import BaseActivity +from sientia_do.temporal.constants import now + import ingestor.metrics as metrics -import os class DataManager(BaseActivity): @@ -85,68 +87,60 @@ class DataManager(BaseActivity): - KAFKA_CONNECTION_STATUS: Set to 1 on successful connection, 0 on failure """ - self.pod_id = os.getenv("HOSTNAME", "localhost") + self.pod_id = os.getenv('HOSTNAME', 'localhost') self.kafka_producer = None self.export_to_kafka = export_to_kafka if self.export_to_kafka: for i in range(0, 3): logger.info( - f"Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}" + f'Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}' ) try: self.kafka_producer = KafkaProducer( bootstrap_servers=kafka_servers, value_serializer=lambda v: json.dumps(v).encode( - "utf-8" + 'utf-8' ), # Serialize JSON messages - key_serializer=lambda k: str( - k).encode("utf-8") if k else None, + key_serializer=lambda k: str(k).encode('utf-8') if k else None, ) # Kafka connected - metrics.KAFKA_CONNECTION_STATUS.labels( - pod_id=self.pod_id).set(1) + metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1) break except NoBrokersAvailable: - logger.error( - f"Kafka servers {kafka_servers} are not available. Retrying..." - ) + logger.error(f'Kafka servers {kafka_servers} are not available. Retrying...') sleep(5) else: # Kafka not connected - metrics.KAFKA_CONNECTION_STATUS.labels( - pod_id=self.pod_id).set(0) + metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0) logger.error( - f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts." + f'Failed to connect to Kafka servers {kafka_servers} after 3 attempts.' ) raise NoBrokersAvailable( - f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts." + f'Failed to connect to Kafka servers {kafka_servers} after 3 attempts.' ) - logger.info( - f"DataManager initialized with Kafka servers: {kafka_servers}") + logger.info(f'DataManager initialized with Kafka servers: {kafka_servers}') logger.info( - f"Trying to initializing DataManager with MongoDB servers: {mongo_connection_string}" + f'Trying to initializing DataManager with MongoDB servers: {mongo_connection_string}' ) self.connection_string = mongo_connection_string self.database = mongo_database - self.mongo_client = MongoClient(self.connection_string) + self.mongo_client: MongoClient = MongoClient(self.connection_string) self.mongo_client.server_info() self.metadata = metadata self.mongo_db = self.mongo_client[self.database] - logger.info( - f"DataManager initialized with MongoDB servers: {self.connection_string}" - ) + logger.info(f'DataManager initialized with MongoDB servers: {self.connection_string}') - BaseActivity.__init__(self, logger=logger, - notification_handler=notification_handler, - set_error_counter=True) + BaseActivity.__init__( + self, logger=logger, notification_handler=notification_handler, set_error_counter=True + ) def shutdown(self): """ @@ -165,27 +159,24 @@ class DataManager(BaseActivity): self.kafka_producer.flush(timeout=10) self.kafka_producer.close() # Mark as disconnected - metrics.KAFKA_CONNECTION_STATUS.labels( - pod_id=self.pod_id).set(0) + metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0) except Exception as e: - self.logger.error(f"Error closing Kafka producer: {e}") + self.logger.error(f'Error closing Kafka producer: {e}') else: - self.logger.warning( - "Kafka producer is already closed or not initialized.") + self.logger.warning('Kafka producer is already closed or not initialized.') if self.mongo_client: try: self.mongo_client.close() except Exception as e: - self.logger.error(f"Error closing MongoDB client: {e}") + self.logger.error(f'Error closing MongoDB client: {e}') else: - self.logger.warning( - "MongoDB client is already closed or not initialized.") + self.logger.warning('MongoDB client is already closed or not initialized.') def __del__(self): self.shutdown() - def delivery_report(self, msg: str): + def delivery_report(self, msg): """ Callback for successful Kafka message delivery reports. @@ -197,10 +188,10 @@ class DataManager(BaseActivity): msg: Kafka message object containing delivery details """ self.logger.debug( - f"Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}" + f'Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}' ) - def delivery_error(self, err: str): + def delivery_error(self, err): """ Callback for Kafka message delivery error reports. @@ -210,7 +201,7 @@ class DataManager(BaseActivity): Args: err: Error information from the failed delivery attempt """ - self.logger.error(f"Delivery failed for record : {err}") + self.logger.error(f'Delivery failed for record : {err}') def publish(self, topic: str, data: dict) -> None: """ @@ -227,28 +218,24 @@ class DataManager(BaseActivity): Exception: If there is an error during message delivery, it will be handled by the `delivery_error` callback. """ - if self.export_to_kafka: + if self.export_to_kafka and self.kafka_producer: try: - - self.logger.debug( - f"Publishing message to topic {topic}: {data}") + self.logger.debug(f'Publishing message to topic {topic}: {data}') self.kafka_producer.send(topic=topic, value=data).add_callback( self.delivery_report ).add_errback(self.delivery_error) self.kafka_producer.flush(timeout=10) - metrics.KAFKA_MESSAGES_SENT.labels( - pod_id=self.pod_id, topic=topic).inc() + metrics.KAFKA_MESSAGES_SENT.labels(pod_id=self.pod_id, topic=topic).inc() except Exception as e: - metrics.KAFKA_MESSAGES_ERRORS.labels( - pod_id=self.pod_id, topic=topic).inc() + metrics.KAFKA_MESSAGES_ERRORS.labels(pod_id=self.pod_id, topic=topic).inc() trace = traceback.format_exc() self.send_notification( metadata=self.metadata, - notification_id=f"KAFKA_PRODUCER_ERROR_{topic}", - message=f"Error publishing message to topic {topic}: {e}", - block="kafka_producer", + notification_id=f'KAFKA_PRODUCER_ERROR_{topic}', + message=f'Error publishing message to topic {topic}: {e}', + block='kafka_producer', level=NotificationLevel.ERROR, attachment_content=trace, ) @@ -260,25 +247,22 @@ class DataManager(BaseActivity): collection.insert_one( { **data, - "inserted_at": now(), + 'inserted_at': now(), } ) - self.logger.debug( - f"Message inserted into MongoDB collection {topic}: {data}") + self.logger.debug(f'Message inserted into MongoDB collection {topic}: {data}') metrics.TAG_WRITTEN_COUNT.labels( - pod_id=self.pod_id, - tag_name=data["name"], - collection_name=topic + pod_id=self.pod_id, tag_name=data['name'], collection_name=topic ).inc() except Exception as e: trace = traceback.format_exc() self.send_notification( metadata=self.metadata, - notification_id=f"MONGO_PRODUCER_ERROR_{topic}", - message=f"Error inserting message to MongoDB: {e}", - block="mongo_producer", + notification_id=f'MONGO_PRODUCER_ERROR_{topic}', + message=f'Error inserting message to MongoDB: {e}', + block='mongo_producer', level=NotificationLevel.ERROR, attachment_content=trace, ) diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index 7206b5f..ea0b3a1 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -1,15 +1,16 @@ import asyncio import traceback -from typing import Dict, List from copy import deepcopy + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel -from sientia_do.temporal.activities.base import BaseActivity from sientia_do.observability.logger import Logger +from sientia_do.temporal.activities.base import BaseActivity + +import ingestor.metrics as metrics from ingestor.managers.data_manager import DataManager from ingestor.managers.opc_manager import OpcManager from ingestor.managers.resource_manager import ResourceManager -import ingestor.metrics as metrics class IngestorManager(BaseActivity): @@ -57,18 +58,24 @@ class IngestorManager(BaseActivity): metadata (dict): Application metadata """ - def __init__(self, - kafka_servers: str, redis_data: dict, - lease_ttl: int, heartbeat_ttl: int, - poll_interval: int, mongo_connection_string: str, mongo_database: str, - metadata: dict, - logger: Logger, notification_handler: NotificationHandler, - export_to_kafka: bool = False): - - redis_host = redis_data.get('host') - redis_port = redis_data.get('port') - redis_username = redis_data.get('username', None) - redis_password = redis_data.get('password', None) + def __init__( + self, + kafka_servers: str, + redis_data: dict, + lease_ttl: int, + heartbeat_ttl: int, + poll_interval: int, + mongo_connection_string: str, + mongo_database: str, + metadata: dict, + logger: Logger, + notification_handler: NotificationHandler, + export_to_kafka: bool = False, + ): + redis_host: str = redis_data['host'] + redis_port: int = int(redis_data['port']) + redis_username: str | None = redis_data.get('username', None) + redis_password: str | None = redis_data.get('password', None) self.data_manager = DataManager( kafka_servers=kafka_servers, @@ -77,9 +84,9 @@ class IngestorManager(BaseActivity): export_to_kafka=export_to_kafka, metadata=metadata, logger=logger, - notification_handler=notification_handler + notification_handler=notification_handler, ) - self.opc_managers = {} + self.opc_managers: dict = {} self.resource_manager = ResourceManager( host=redis_host, port=redis_port, @@ -93,14 +100,14 @@ class IngestorManager(BaseActivity): ) self.number_of_slots = 0 self.poll_interval = poll_interval - self.managed_tags = {} - self.opc_servers = {} + self.managed_tags: dict = {} + self.opc_servers: dict = {} self.metadata = metadata - BaseActivity.__init__(self, logger=logger, - notification_handler=notification_handler, - set_error_counter=True) + BaseActivity.__init__( + self, logger=logger, notification_handler=notification_handler, set_error_counter=True + ) async def initialize_opc_from_config(self, server_config: dict) -> OpcManager | None: """ @@ -130,8 +137,7 @@ class IngestorManager(BaseActivity): """ try: - self.logger.info( - f"Initializing OpcManager at {server_config['url']}") + self.logger.info(f'Initializing OpcManager at {server_config["url"]}') manager = OpcManager( name=server_config['name'], url=server_config['url'], @@ -142,7 +148,7 @@ class IngestorManager(BaseActivity): metadata=self.metadata, cert_path=server_config.get('cert_path'), private_key_path=server_config.get('private_key_path'), - server_cert_path=server_config.get('server_cert_path') + server_cert_path=server_config.get('server_cert_path'), ) manager.config = server_config @@ -153,9 +159,9 @@ class IngestorManager(BaseActivity): metadata=self.metadata, notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}', message=f'Error initializing OPC manager: {e}', - block="opc_manager", + block='opc_manager', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) self.logger.error(trace) @@ -216,49 +222,41 @@ class IngestorManager(BaseActivity): server_config.pop('tags', None) server_instance = self.opc_managers.get(server, None) if server_instance is None: - self.logger.info( - f"Initializing OPC manager for server {server}" - ) - server_instance = await self.initialize_opc_from_config( - server_config - ) + self.logger.info(f'Initializing OPC manager for server {server}') + server_instance = await self.initialize_opc_from_config(server_config) elif server_instance.config != server_config: - self.logger.warning( - f"Reinitializing OPC manager for server {server}" - ) + self.logger.warning(f'Reinitializing OPC manager for server {server}') server_instance.disconnect() del self.opc_managers[server] - server_instance = await self.initialize_opc_from_config( - server_config - ) + server_instance = await self.initialize_opc_from_config(server_config) else: self.logger.debug( - f"OPC manager for server {server} is already initialized and up to date" + f'OPC manager for server {server} is already initialized and up to date' ) if server_instance is not None: self.opc_managers[server] = server_instance else: self.logger.warning( - f"Failed to initialize OPC manager for server {server}, " - f"removing server from managed tags." + f'Failed to initialize OPC manager for server {server}, ' + f'removing server from managed tags.' ) - _a = [self.managed_tags[slot].pop(server, None) - for slot, _value in current_managed_tags.items()] + _a = [ + self.managed_tags[slot].pop(server, None) + for slot, _value in current_managed_tags.items() + ] servers = list(self.opc_managers.keys()) for server in servers: if server not in registered_servers: self.logger.warning( - f"Server {server} not found in managed tags. " - f"Desconnecting from server." + f'Server {server} not found in managed tags. Desconnecting from server.' ) await self.opc_managers[server].disconnect() self.opc_managers.pop(server, None) - metrics.OPC_MANAGERS_ACTIVE.labels( - pod_id=self.pod_id).set(len(self.opc_managers)) + metrics.OPC_MANAGERS_ACTIVE.labels(pod_id=self.pod_id).set(len(self.opc_managers)) def check_opc_servers_integrity(self): """ @@ -280,16 +278,12 @@ class IngestorManager(BaseActivity): is_lost = opc_manager.check_opc_listenning() if is_lost: - self.logger.warning( - f"OPC server {server} is lost. " - f"Server will be disconnected." - ) + self.logger.warning(f'OPC server {server} is lost. Server will be disconnected.') for slot, _config in self.managed_tags.items(): self.managed_tags[slot].pop(server, None) - metrics.OPC_MANAGERS_ACTIVE.labels( - pod_id=self.pod_id).set(len(self.opc_managers)) + metrics.OPC_MANAGERS_ACTIVE.labels(pod_id=self.pod_id).set(len(self.opc_managers)) def declare_active(self): """ @@ -306,7 +300,7 @@ class IngestorManager(BaseActivity): self.resource_manager.ingestor_heartbeat() - def get_active_ingestors(self) -> List[str]: + def get_active_ingestors(self) -> list[str]: """ Retrieve a list of active ingestors. @@ -363,7 +357,7 @@ class IngestorManager(BaseActivity): metrics.SLOTS_TOTAL.set(self.number_of_slots) return self.number_of_slots - def get_slot_leases(self, max_slots: int = 1) -> Dict: + def get_slot_leases(self, max_slots: int = 1) -> dict: """ Acquires a specified number of resource slots by leasing them from the resource manager. @@ -377,7 +371,7 @@ class IngestorManager(BaseActivity): max_slots (int): The maximum number of slots to lease. Defaults to 1. Returns: - Dict: A dictionary where the keys are the slot identifiers (as strings) + Dict: A dictionary where the keys are the slot identifiers (as strings) and the values are the leased slot details. Behavior: @@ -394,7 +388,7 @@ class IngestorManager(BaseActivity): acquired = {} for i in range(1, self.number_of_slots + 1): if self.resource_manager.lease_tag(str(i)): - self.logger.info(f"Leased slot {i}") + self.logger.info(f'Leased slot {i}') slots = self.resource_manager.get_tag_slot(str(i)) if slots is None: continue @@ -403,17 +397,14 @@ class IngestorManager(BaseActivity): if len(acquired) >= max_slots: self.managed_tags.update(acquired) - metrics.SLOTS_MANAGED.labels( - pod_id=self.pod_id).set(len(self.managed_tags)) + metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(len(self.managed_tags)) return acquired self.logger.warning( - f"Unable to acquire {max_slots} slots. " - f"Only {acquired} slots were leased." + f'Unable to acquire {max_slots} slots. Only {acquired} slots were leased.' ) self.managed_tags.update(acquired) - metrics.SLOTS_MANAGED.labels( - pod_id=self.pod_id).set(len(self.managed_tags)) + metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(len(self.managed_tags)) return acquired async def unsubscribe_slot(self, slot: str): @@ -441,7 +432,7 @@ class IngestorManager(BaseActivity): def update_slot_config(self): """ - Updates the configuration of managed slots by renewing their leases, + Updates the configuration of managed slots by renewing their leases, fetching the latest configurations, and handling any changes or removals. This method performs the following steps: @@ -458,7 +449,7 @@ class IngestorManager(BaseActivity): - Updates OPC server subscriptions based on the current state of managed slots. Raises: - None explicitly, but relies on the behavior of `resource_manager` and + None explicitly, but relies on the behavior of `resource_manager` and other dependencies for error handling. Logging: @@ -466,8 +457,7 @@ class IngestorManager(BaseActivity): - Logs informational messages for updated slot configurations. """ - removed_slots = [] - update = {} + removed_slots: list[str] = [] for slot, _slot_config in self.managed_tags.items(): self.resource_manager.renew_tag_lease(slot) update = self.resource_manager.get_tag_slot(slot) @@ -480,17 +470,17 @@ class IngestorManager(BaseActivity): for slot in removed_slots: self.managed_tags.pop(slot, None) - def drop_slot_leases(self, ids: List[str]) -> None: + def drop_slot_leases(self, ids: list[str]) -> None: """ Releases the leases associated with the specified slot IDs. - This method iterates through a list of slot IDs and calls the - `drop_tag_lease` method of the `resource_manager` to release + This method iterates through a list of slot IDs and calls the + `drop_tag_lease` method of the `resource_manager` to release the lease for each ID. It's used during load balancing and graceful shutdown scenarios. Args: - ids (List[str]): A list of slot IDs for which the leases + ids (List[str]): A list of slot IDs for which the leases should be released. Returns: @@ -509,7 +499,7 @@ class IngestorManager(BaseActivity): Manages the subscription of tags to a specified OPC server and slot. This method ensures that the specified server and slot have an active subscription - for the provided tags. If the server or slot is not properly configured, or if + for the provided tags. If the server or slot is not properly configured, or if subscription fails, appropriate error handling is performed. Args: @@ -527,12 +517,12 @@ class IngestorManager(BaseActivity): Logs: - Logs informational messages about the subscription process. - - Logs errors if the server is not found, subscription creation fails, or + - Logs errors if the server is not found, subscription creation fails, or tag subscription fails. - Logs a warning if a subscription is removed due to failure. Raises: - Exception: Any unexpected exceptions during subscription creation or tag + Exception: Any unexpected exceptions during subscription creation or tag subscription are logged but not propagated. Side Effects: @@ -541,77 +531,63 @@ class IngestorManager(BaseActivity): - Updates error metrics and notifications """ - self.logger.info( - f"Subscribing to tags from {slot}:{server}" - ) + self.logger.info(f'Subscribing to tags from {slot}:{server}') tags_to_sub = server_config.get('tags') if server not in self.opc_managers: - self.logger.error( - f"Server {server} not found in opc_managers." - ) + self.logger.error(f'Server {server} not found in opc_managers.') return 1 if slot not in self.opc_managers[server].subscriptions: try: - await self.opc_managers[server].create_subscription( - slot - ) + await self.opc_managers[server].create_subscription(slot) except Exception as e: - self.logger.error( - f"Failed to create subscription for slot {slot}: {e}" - ) + self.logger.error(f'Failed to create subscription for slot {slot}: {e}') return 2 try: - self.logger.info( - tags_to_sub - ) + self.logger.info(tags_to_sub) await self.opc_managers[server].subscribe( slot, deepcopy(tags_to_sub), self.poll_interval ) - self.logger.info( - tags_to_sub - ) + self.logger.info(tags_to_sub) except Exception as e: metrics.OPC_SUBSCRIPTION_ERRORS.labels( - pod_id=self.pod_id, server=server, slot=slot).inc() + pod_id=self.pod_id, server=server, slot=slot + ).inc() trace = traceback.format_exc() self.send_notification( metadata=self.metadata, notification_id=f'OPC_SUBSCRIPTION_ERROR_{slot}:{server}', message=f'Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}', - block="opc_manager", + block='opc_manager', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) self.logger.error(trace) - self.logger.warning( - "Removing subscription from server " - f"{server} for slot {slot}" - ) + self.logger.warning(f'Removing subscription from server {server} for slot {slot}') await self.opc_managers[server].unsubscribe(slot) return 2 return 0 - async def subscribe_to_tags(self, tags: Dict) -> None: + async def subscribe_to_tags(self, tags: dict) -> None: """ Subscribes to a set of tags and manages their configurations. This method processes a dictionary of tags, iterating through each slot and server - configuration. It attempts to manage the server configurations and removes any + configuration. It attempts to manage the server configurations and removes any servers that return a specific response code. Args: - tags (Dict): A dictionary containing tag configurations. The structure is + tags (Dict): A dictionary containing tag configurations. The structure is expected to be {slot: {server: server_config}}. Side Effects: - Logs the provided tags for debugging purposes. - - Updates the `managed_tags` attribute by removing servers that meet the + - Updates the `managed_tags` attribute by removing servers that meet the removal criteria. - Establishes OPC subscriptions for all configured tags. Removal Criteria: - - If the `manage_server` method returns a response code of 2 for a given + - If the `manage_server` method returns a response code of 2 for a given slot and server, that server is removed from the `managed_tags` attribute. The method ensures that only successfully configured servers remain in the @@ -622,9 +598,7 @@ class IngestorManager(BaseActivity): self.logger.info(tags) for slot, slot_config in tags.items(): for server, server_config in slot_config.items(): - response = await self.manage_server( - slot, server, server_config, tags - ) + response = await self.manage_server(slot, server, server_config, tags) if response == 2: to_remove.append([slot, server]) diff --git a/ingestor/managers/opc_manager.py b/ingestor/managers/opc_manager.py index b67fcf6..bf66ef1 100644 --- a/ingestor/managers/opc_manager.py +++ b/ingestor/managers/opc_manager.py @@ -1,15 +1,16 @@ import json -import asyncio from pathlib import Path -from asyncua.crypto.security_policies import SecurityPolicyBasic256 + from asyncua import Client -from sientia_do.notifications.models import NotificationLevel +from asyncua.crypto.security_policies import SecurityPolicyBasic256 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler -from sientia_do.temporal.activities.base import BaseActivity +from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger -from sientia_do.temporal.constants import OPC_TIMEZONE, DATETIME_FORMAT_WITH_TZ -from ingestor.managers.data_manager import DataManager +from sientia_do.temporal.activities.base import BaseActivity +from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, OPC_TIMEZONE + import ingestor.metrics as metrics +from ingestor.managers.data_manager import DataManager class OpcManager(BaseActivity): @@ -54,31 +55,41 @@ class OpcManager(BaseActivity): metadata (dict): Application metadata """ - def __init__(self, name: str, url: str, data_manager: DataManager, logger: Logger, - server_uri: str, notification_handler: NotificationHandler, metadata: dict, - cert_path: str = None, private_key_path: str = None, server_cert_path: str = None): + def __init__( + self, + name: str, + url: str, + data_manager: DataManager, + logger: Logger, + server_uri: str, + notification_handler: NotificationHandler, + metadata: dict, + cert_path: str | None = None, + private_key_path: str | None = None, + server_cert_path: str | None = None, + ): self.url = url self.name = name self.server_uri = server_uri - self.data_queue = {} + self.data_queue: dict = {} self.non_receive_count = 0 - self.client = None + self.client: Client | None = None self.cert_path = cert_path self.private_key_path = private_key_path self.server_cert_path = server_cert_path - self.nodes = {} - self.subscriptions = {} + self.nodes: dict = {} + self.subscriptions: dict = {} self.data_manager = data_manager self.metadata = metadata - BaseActivity.__init__(self, logger=logger, - notification_handler=notification_handler, - set_error_counter=True) + BaseActivity.__init__( + self, logger=logger, notification_handler=notification_handler, set_error_counter=True + ) metrics.OPC_CONNECTION_STATUS.labels( - pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0) - metrics.OPC_TAGS_SUBSCRIBED.labels( - pod_id=self.pod_id, server_name=self.name).set(0) + pod_id=self.pod_id, server_name=self.name, server_url=self.url + ).set(0) + metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(0) def __str__(self): """ @@ -87,8 +98,10 @@ class OpcManager(BaseActivity): Returns: str: Human-readable representation showing server details and current state. """ - return f"OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n" \ - f"nodes={self.nodes}, subscriptions={self.subscriptions}" + return ( + f'OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n' + f'nodes={self.nodes}, subscriptions={self.subscriptions}' + ) async def shutdown(self): """ @@ -103,10 +116,9 @@ class OpcManager(BaseActivity): and ensure clean disconnection from OPC servers. """ try: - await self.disconnect() except Exception as e: - self.logger.error(f"Error during cleanup: {e}") + self.logger.error(f'Error during cleanup: {e}') async def set_security(self): """ @@ -133,22 +145,23 @@ class OpcManager(BaseActivity): if not all([self.cert_path, self.private_key_path]): raise ValueError( - "Certificate and private key paths must be provided for secure connection.") - cert = Path(self.cert_path) - private_key = Path(self.private_key_path) - server_cert = Path( - self.server_cert_path) if self.server_cert_path else None + 'Certificate and private key paths must be provided for secure connection.' + ) + cert = Path(self.cert_path) if self.cert_path else None + private_key = Path(self.private_key_path) if self.private_key_path else None + server_cert = Path(self.server_cert_path) if self.server_cert_path else None - await self.client.set_application_uri(self.server_uri) - self.logger.info('Setting security...') - await self.client.set_security( - SecurityPolicyBasic256, - certificate=str(cert), - private_key=str(private_key), - server_certificate=str(server_cert) - ) - await self.client.set_secure_channel_timeout(10000000) - await self.client.set_session_timeout(10000000) + if self.client: + await self.client.set_application_uri(self.server_uri) + self.logger.info('Setting security...') + await self.client.set_security( + SecurityPolicyBasic256, + certificate=str(cert), + private_key=str(private_key), + server_certificate=str(server_cert), + ) + await self.client.set_secure_channel_timeout(10000000) + await self.client.set_session_timeout(10000000) async def connect(self): """ @@ -172,24 +185,25 @@ class OpcManager(BaseActivity): - OPC_CONNECTION_STATUS: Set to 1 on successful connection """ - metrics.OPC_CONNECTIONS_TOTAL.labels( - pod_id=self.pod_id, server_name=self.name).inc() + metrics.OPC_CONNECTIONS_TOTAL.labels(pod_id=self.pod_id, server_name=self.name).inc() try: self.client = Client(self.url, watchdog_intervall=3600000) + assert self.client is not None # Informa ao mypy que client nรฃo รฉ None self.client.name = self.pod_id if self.cert_path: await self.set_security() self.logger.info(f'Starting connection to {self.name}...') await self.client.connect() metrics.OPC_CONNECTION_STATUS.labels( - pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(1) + pod_id=self.pod_id, server_name=self.name, server_url=self.url + ).set(1) self.logger.info(f'Connection to {self.name} successful.') except Exception as e: metrics.OPC_CONNECTION_STATUS.labels( - pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0) - metrics.OPC_CONNECTIONS_FAILED.labels( - pod_id=self.pod_id, server_name=self.name).inc() - self.logger.error(f"Failed to connect to {self.name}: {e}") + pod_id=self.pod_id, server_name=self.name, server_url=self.url + ).set(0) + metrics.OPC_CONNECTIONS_FAILED.labels(pod_id=self.pod_id, server_name=self.name).inc() + self.logger.error(f'Failed to connect to {self.name}: {e}') raise async def create_subscription(self, name: str, period: int = 500): @@ -214,16 +228,16 @@ class OpcManager(BaseActivity): """ if not self.client: - raise ValueError("Client not connected. Call connect first.") + raise ValueError('Client not connected. Call connect first.') try: p = period if period is not None else 500 self.subscriptions[name] = await self.client.create_subscription(p, self) self.logger.info(f'Subscription {name} created on {self.name}.') metrics.OPC_SUBSCRIPTIONS_CREATED.labels( - pod_id=self.pod_id, server_name=self.name, slot_name=name).inc() + pod_id=self.pod_id, server_name=self.name, slot_name=name + ).inc() except Exception as e: - self.logger.error( - f"Failed to create subscription {name} on {self.name}: {e}") + self.logger.error(f'Failed to create subscription {name} on {self.name}: {e}') raise async def subscribe(self, subscription: str, nodes: dict, collect_period: int): @@ -252,23 +266,24 @@ class OpcManager(BaseActivity): """ if not self.subscriptions.get(subscription): - raise ValueError( - "Subscription not created. Call create_subscription first.") + raise ValueError('Subscription not created. Call create_subscription first.') - self.logger.info(f"Subscribing to {subscription} on {self.name}...") - self.logger.info(f"Subscribing to nodes: {nodes}") + self.logger.info(f'Subscribing to {subscription} on {self.name}...') + self.logger.info(f'Subscribing to nodes: {nodes}') + assert self.client is not None # Informa ao mypy que client nรฃo รฉ None addr_nodes = [self.client.get_node(n) for n in nodes] - self.logger.debug(f"Addr nodes: {addr_nodes}") + self.logger.debug(f'Addr nodes: {addr_nodes}') self.nodes.update(nodes) - self.logger.debug(f"Nodes: {self.nodes}") - metrics.OPC_TAGS_SUBSCRIBED.labels( - pod_id=self.pod_id, server_name=self.name).set(len(self.nodes)) + self.logger.debug(f'Nodes: {self.nodes}') + metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set( + len(self.nodes) + ) self.collect_period = collect_period for node, config in self.nodes.items(): self.nodes[node]['cycle_rule'] = { - 'cycle_increment': collect_period*1000/float(config['frequency']), - 'cycle_count': 0 + 'cycle_increment': collect_period * 1000 / float(config['frequency']), + 'cycle_count': 0, } await self.subscriptions[subscription].subscribe_data_change(addr_nodes) @@ -288,18 +303,17 @@ class OpcManager(BaseActivity): - An info message upon successful unsubscription. Behavior: - - If the subscription exists, it is deleted and removed from the + - If the subscription exists, it is deleted and removed from the subscriptions dictionary. - If the subscription does not exist, no action is taken. """ if not self.subscriptions.get(subscription): - self.logger.warning( - f"Subscription '{subscription}' not found. Cannot unsubscribe.") + self.logger.warning(f"Subscription '{subscription}' not found. Cannot unsubscribe.") return await self.subscriptions[subscription].delete() del self.subscriptions[subscription] - self.logger.info(f"Unsubscribed from {subscription}.") + self.logger.info(f'Unsubscribed from {subscription}.') async def disconnect(self): """ @@ -322,28 +336,27 @@ class OpcManager(BaseActivity): self.logger.warning('Disconnecting from OPC server') if self.client is None: - self.logger.warning("Client already disconnected.") + self.logger.warning('Client already disconnected.') return try: for sub in self.subscriptions: await self.subscriptions[sub].delete() - self.logger.warning("Deleted all subscriptions.") + self.logger.warning('Deleted all subscriptions.') except Exception as sub_error: - self.logger.error(f"Failed to clean up subscription: {sub_error}") + self.logger.error(f'Failed to clean up subscription: {sub_error}') try: await self.client.disconnect() except Exception as conn_error: - self.logger.error( - f"Failed to disconnect from OPC UA server: {conn_error}") + self.logger.error(f'Failed to disconnect from OPC UA server: {conn_error}') finally: del self.client self.client = None metrics.OPC_CONNECTION_STATUS.labels( - pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0) - metrics.OPC_TAGS_SUBSCRIBED.labels( - pod_id=self.pod_id, server_name=self.name).set(0) - self.logger.warning("Disconnected from OPC UA server.") + pod_id=self.pod_id, server_name=self.name, server_url=self.url + ).set(0) + metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(0) + self.logger.warning('Disconnected from OPC UA server.') async def datachange_notification(self, node, _val, data): """ @@ -371,35 +384,34 @@ class OpcManager(BaseActivity): monitored_item = data.monitored_item value = monitored_item.Value.Value.Value # source_timestamp - source_timestamp = monitored_item.Value.SourceTimestamp.replace( - tzinfo=OPC_TIMEZONE) + source_timestamp = monitored_item.Value.SourceTimestamp.replace(tzinfo=OPC_TIMEZONE) tag = str(node) self.logger.debug( - f"Data change notification received for tag:" - f"{tag} after {self.nodes[tag]['cycle_rule']['cycle_count']} cycles") + f'Data change notification received for tag:' + f'{tag} after {self.nodes[tag]["cycle_rule"]["cycle_count"]} cycles' + ) self.nodes[tag]['cycle_rule']['cycle_count'] = 0 self.non_receive_count = 0 - metrics.OPC_CYCLES_WITHOUT_DATA.labels( - pod_id=self.pod_id, server_name=self.name).set(0) + metrics.OPC_CYCLES_WITHOUT_DATA.labels(pod_id=self.pod_id, server_name=self.name).set(0) data = { 'tag': tag, 'name': self.nodes[str(node)]['tag_name'], 'timestamp': source_timestamp.strftime(DATETIME_FORMAT_WITH_TZ), - 'value': value + 'value': value, } - _a = [self.data_manager.publish(e, data) - for e in self.nodes[tag]['topics']] + for topic in self.nodes[tag]['topics']: + self.data_manager.publish(topic, data) def check_cycles(self): """ Checks the cycle counts for all monitored nodes and sends notifications if thresholds are exceeded. - This method iterates through all monitored nodes and updates their cycle counts based on + This method iterates through all monitored nodes and updates their cycle counts based on configured increments. If a node's cycle count exceeds a threshold (5 cycles), it triggers a warning notification. @@ -408,8 +420,7 @@ class OpcManager(BaseActivity): - Sends warning notifications for nodes exceeding cycle thresholds """ for node, config in self.nodes.items(): - self.nodes[node]['cycle_rule']['cycle_count'] += config[ - 'cycle_rule']['cycle_increment'] + self.nodes[node]['cycle_rule']['cycle_count'] += config['cycle_rule']['cycle_increment'] if self.nodes[node]['cycle_rule']['cycle_count'] >= 5: name = config['tag_name'] cycles = self.nodes[node]['cycle_rule']['cycle_count'] @@ -417,8 +428,8 @@ class OpcManager(BaseActivity): metadata=self.metadata, notification_id=f'TAG_{node}:{name}_LISTENNING_STOPPED', message=f'{cycles} cycles without receive from {node}:{name}', - block="opc_manager", - level=NotificationLevel.WARNING + block='opc_manager', + level=NotificationLevel.WARNING, ) def check_opc_listenning(self) -> bool: @@ -441,26 +452,26 @@ class OpcManager(BaseActivity): """ self.non_receive_count += 1 - metrics.OPC_CYCLES_WITHOUT_DATA.labels( - pod_id=self.pod_id, server_name=self.name).set(self.non_receive_count) + metrics.OPC_CYCLES_WITHOUT_DATA.labels(pod_id=self.pod_id, server_name=self.name).set( + self.non_receive_count + ) if self.non_receive_count >= 5: self.send_notification( metadata=self.metadata, notification_id=f'OPC_LISTENNING_STOPPED__{self.name}', message=f'{self.non_receive_count} cycles without ' f'receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}', - block="opc_manager", - level=NotificationLevel.ERROR + block='opc_manager', + level=NotificationLevel.ERROR, ) if self.non_receive_count >= 15: - metrics.OPC_RECONNECTIONS_TOTAL.labels( - pod_id=self.pod_id, server_name=self.name).inc() + metrics.OPC_RECONNECTIONS_TOTAL.labels(pod_id=self.pod_id, server_name=self.name).inc() self.send_notification( metadata=self.metadata, notification_id=f'OPC_CONNECTION_RETRY__{self.name}', message=f'Retrying to connect to server {self.name}', - block="opc_manager", - level=NotificationLevel.ERROR + block='opc_manager', + level=NotificationLevel.ERROR, ) return True return False diff --git a/ingestor/managers/resource_manager.py b/ingestor/managers/resource_manager.py index b77e3c3..30502c7 100644 --- a/ingestor/managers/resource_manager.py +++ b/ingestor/managers/resource_manager.py @@ -1,13 +1,14 @@ import json -from typing import List -from redis import Redis from time import time -import ingestor.metrics as metrics + +from redis import Redis from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger from sientia_do.temporal.activities.base import BaseActivity +import ingestor.metrics as metrics + class ResourceManager(BaseActivity): """ @@ -80,9 +81,9 @@ class ResourceManager(BaseActivity): Metrics: - REDIS_CONNECTION_STATUS: Set to 1 on successful connection, 0 on failure """ - BaseActivity.__init__(self, logger=logger, - notification_handler=notification_handler, - set_error_counter=True) + BaseActivity.__init__( + self, logger=logger, notification_handler=notification_handler, set_error_counter=True + ) try: self.redis = Redis( host=host, @@ -94,7 +95,7 @@ class ResourceManager(BaseActivity): self.redis.ping() metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1) except Exception as e: - self.logger.error(f"Failed to connect to Redis: {e}") + self.logger.error(f'Failed to connect to Redis: {e}') metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0) raise @@ -146,14 +147,14 @@ class ResourceManager(BaseActivity): ).inc() self.send_notification( metadata=self.metadata, - notification_id=f"REDIS_OPERATION_ERROR_{operation_name}", + notification_id=f'REDIS_OPERATION_ERROR_{operation_name}', message=f"Error in Redis operation '{operation_name}': {e}", - block="redis_manager", + block='redis_manager', level=NotificationLevel.ERROR, ) raise - def get(self, key: str) -> dict: + def get(self, key: str) -> dict | None: """ Retrieve a value from Redis by its key and return it as a dictionary. @@ -172,10 +173,10 @@ class ResourceManager(BaseActivity): - REDIS_OPERATIONS_DURATION: Records timing for get operations """ - history = self._execute_redis_op("get", self.redis.get, key) + history = self._execute_redis_op('get', self.redis.get, key) return json.loads(history) if history else None - def get_tag_slot(self, id: str) -> dict: + def get_tag_slot(self, tag_id: str) -> dict | None: """ Retrieve the tag slot information for a given ID. @@ -193,7 +194,7 @@ class ResourceManager(BaseActivity): and delegates to the get() method for the actual Redis operation. """ - return self.get(f"slot:opc_tags:{id}") + return self.get(f'slot:opc_tags:{tag_id}') def ingestor_heartbeat(self) -> None: """ @@ -215,9 +216,9 @@ class ResourceManager(BaseActivity): """ self._execute_redis_op( - "set", + 'set', self.redis.set, - f"heartbeat:ingestor:{self.pod_id}", + f'heartbeat:ingestor:{self.pod_id}', 1, ex=self.heartbeat_ttl, ) @@ -249,9 +250,9 @@ class ResourceManager(BaseActivity): """ return self._execute_redis_op( - "set_nx", + 'set_nx', self.redis.set, - f"lease:opc_tags:{tag_id}", + f'lease:opc_tags:{tag_id}', self.pod_id, nx=True, ex=self.lease_ttl, @@ -282,12 +283,10 @@ class ResourceManager(BaseActivity): - REDIS_OPERATIONS_DURATION: Records timing for renewal operations """ - current = self._execute_redis_op( - "get", self.redis.get, f"lease:opc_tags:{tag_id}" - ) + current = self._execute_redis_op('get', self.redis.get, f'lease:opc_tags:{tag_id}') if current == self.pod_id: self._execute_redis_op( - "expire", self.redis.expire, f"lease:opc_tags:{tag_id}", self.lease_ttl + 'expire', self.redis.expire, f'lease:opc_tags:{tag_id}', self.lease_ttl ) return True return False @@ -313,10 +312,9 @@ class ResourceManager(BaseActivity): - REDIS_OPERATIONS_DURATION: Records timing for lease dropping operations """ - self._execute_redis_op("delete", self.redis.delete, - f"lease:opc_tags:{tag_id}") + self._execute_redis_op('delete', self.redis.delete, f'lease:opc_tags:{tag_id}') - def get_all_ingestors(self) -> List[str]: + def get_all_ingestors(self) -> list[str]: """ Retrieves all active ingestors from Redis. @@ -338,9 +336,9 @@ class ResourceManager(BaseActivity): - REDIS_OPERATIONS_DURATION: Records timing for ingestor discovery """ - return self._execute_redis_op("keys", self.redis.keys, "heartbeat:ingestor:*") + return self._execute_redis_op('keys', self.redis.keys, 'heartbeat:ingestor:*') - def get_all_slots(self) -> List[str]: + def get_all_slots(self) -> list[str]: """ Retrieves all available slots from Redis. @@ -362,9 +360,9 @@ class ResourceManager(BaseActivity): - REDIS_OPERATIONS_DURATION: Records timing for slot discovery """ - return self._execute_redis_op("keys", self.redis.keys, "slot:opc_tags:*") + return self._execute_redis_op('keys', self.redis.keys, 'slot:opc_tags:*') - def get_all_leases(self) -> List[str]: + def get_all_leases(self) -> list[str]: """ Retrieves all active leases from Redis. @@ -386,4 +384,4 @@ class ResourceManager(BaseActivity): - REDIS_OPERATIONS_DURATION: Records timing for lease discovery """ - return self._execute_redis_op("keys", self.redis.keys, "lease:opc_tags:*") + return self._execute_redis_op('keys', self.redis.keys, 'lease:opc_tags:*') diff --git a/ingestor/metrics.py b/ingestor/metrics.py index e1ddb4e..13c121e 100644 --- a/ingestor/metrics.py +++ b/ingestor/metrics.py @@ -17,157 +17,157 @@ labels for multi-dimensional analysis and alerting. from prometheus_client import Counter, Gauge, Histogram # Metric label definitions for consistent labeling across all metrics -POD_ID_LABEL = ["pod_id"] -SERVER_LABELS = ["pod_id", "server_name", "server_url"] -KAFKA_LABELS = ["pod_id", "topic"] -REDIS_LABELS = ["pod_id", "operation"] -NOTIFICATION_LABELS = ["pod_id", "level", "block"] +POD_ID_LABEL = ['pod_id'] +SERVER_LABELS = ['pod_id', 'server_name', 'server_url'] +KAFKA_LABELS = ['pod_id', 'topic'] +REDIS_LABELS = ['pod_id', 'operation'] +NOTIFICATION_LABELS = ['pod_id', 'level', 'block'] -MAIN_LABELS = ["pod_id"] +MAIN_LABELS = ['pod_id'] # --- Reliability Metrics --- TAG_WRITTEN_COUNT = Counter( - "ingestor_tag_written_count", - "Number of writing process to the collection", - [*MAIN_LABELS, "tag_name", "collection_name"], + 'ingestor_tag_written_count', + 'Number of writing process to the collection', + [*MAIN_LABELS, 'tag_name', 'collection_name'], ) # --- General Application Metrics --- APP_LOOP_COUNT = Counter( - "app_main_loop_total", - "Total number of times the application main loop has run", + 'app_main_loop_total', + 'Total number of times the application main loop has run', POD_ID_LABEL, ) APP_LOOP_DURATION = Histogram( - "app_main_loop_duration_seconds", - "Duration of the application main loop in seconds", + 'app_main_loop_duration_seconds', + 'Duration of the application main loop in seconds', POD_ID_LABEL, ) APP_ERRORS_TOTAL = Counter( - "app_errors_total", - "Total number of unhandled errors in the main loop", + 'app_errors_total', + 'Total number of unhandled errors in the main loop', POD_ID_LABEL, ) APP_UP = Gauge( - "app_up", - "Indicates if the application is running (1) or shutting down (0)", + 'app_up', + 'Indicates if the application is running (1) or shutting down (0)', POD_ID_LABEL, ) # --- Ingestor Manager Metrics --- ACTIVE_INGESTORS = Gauge( - "ingestor_active_total", - "Number of active ingestors reported by Redis", + 'ingestor_active_total', + 'Number of active ingestors reported by Redis', ) SLOTS_TOTAL = Gauge( - "ingestor_slots_total", - "Total number of slots configured in Redis", + 'ingestor_slots_total', + 'Total number of slots configured in Redis', ) LEASES_TOTAL = Gauge( - "ingestor_leases_total", - "Total number of leases (allocated slots) in Redis", + 'ingestor_leases_total', + 'Total number of leases (allocated slots) in Redis', ) SLOTS_MANAGED = Gauge( - "ingestor_slots_managed_current", - "Number of slots currently managed by this ingestor instance", + 'ingestor_slots_managed_current', + 'Number of slots currently managed by this ingestor instance', POD_ID_LABEL, ) SLOTS_ACQUIRED = Counter( - "ingestor_slots_acquired_total", - "Total number of slots acquired by this instance", + 'ingestor_slots_acquired_total', + 'Total number of slots acquired by this instance', POD_ID_LABEL, ) SLOTS_RELEASED = Counter( - "ingestor_slots_released_total", - "Total number of slots released by this instance", + 'ingestor_slots_released_total', + 'Total number of slots released by this instance', POD_ID_LABEL, ) OPC_MANAGERS_ACTIVE = Gauge( - "ingestor_opc_managers_active", - "Number of active OPC Managers in this instance", + 'ingestor_opc_managers_active', + 'Number of active OPC Managers in this instance', POD_ID_LABEL, ) OPC_SUBSCRIPTION_ERRORS = Counter( - "ingestor_opc_subscription_errors_total", - "Errors when trying to subscribe to OPC tags", - ["pod_id", "server", "slot"], + 'ingestor_opc_subscription_errors_total', + 'Errors when trying to subscribe to OPC tags', + ['pod_id', 'server', 'slot'], ) # --- OPC Manager Metrics --- OPC_CONNECTIONS_TOTAL = Counter( - "opc_connections_initiated_total", - "Total connection attempts to OPC servers", - ["pod_id", "server_name"], + 'opc_connections_initiated_total', + 'Total connection attempts to OPC servers', + ['pod_id', 'server_name'], ) OPC_CONNECTIONS_FAILED = Counter( - "opc_connections_failed_total", - "Total failed connection attempts to OPC servers", - ["pod_id", "server_name"], + 'opc_connections_failed_total', + 'Total failed connection attempts to OPC servers', + ['pod_id', 'server_name'], ) OPC_CONNECTION_STATUS = Gauge( - "opc_connection_status", - "Connection status with the OPC server (1=connected, 0=disconnected)", + 'opc_connection_status', + 'Connection status with the OPC server (1=connected, 0=disconnected)', SERVER_LABELS, ) OPC_SUBSCRIPTIONS_CREATED = Counter( - "opc_subscriptions_created_total", - "Total OPC subscriptions created", - ["pod_id", "server_name", "slot_name"], + 'opc_subscriptions_created_total', + 'Total OPC subscriptions created', + ['pod_id', 'server_name', 'slot_name'], ) OPC_TAGS_SUBSCRIBED = Gauge( - "opc_tags_subscribed_current", - "Current number of OPC tags subscribed on a server", - ["pod_id", "server_name"], + 'opc_tags_subscribed_current', + 'Current number of OPC tags subscribed on a server', + ['pod_id', 'server_name'], ) OPC_CYCLES_WITHOUT_DATA = Gauge( - "opc_cycles_without_data", - "Current number of cycles without receiving data from a server", - ["pod_id", "server_name"], + 'opc_cycles_without_data', + 'Current number of cycles without receiving data from a server', + ['pod_id', 'server_name'], ) OPC_RECONNECTIONS_TOTAL = Counter( - "opc_reconnections_tried_total", - "Reconnection attempts to an OPC server after a loss", - ["pod_id", "server_name"], + 'opc_reconnections_tried_total', + 'Reconnection attempts to an OPC server after a loss', + ['pod_id', 'server_name'], ) # --- Data Manager (Kafka) Metrics --- KAFKA_MESSAGES_SENT = Counter( - "kafka_messages_sent_total", "Total messages sent to Kafka", KAFKA_LABELS + 'kafka_messages_sent_total', 'Total messages sent to Kafka', KAFKA_LABELS ) KAFKA_MESSAGES_ERRORS = Counter( - "kafka_messages_errors_total", - "Total errors sending messages to Kafka", + 'kafka_messages_errors_total', + 'Total errors sending messages to Kafka', KAFKA_LABELS, ) KAFKA_CONNECTION_STATUS = Gauge( - "kafka_connection_status", - "Connection status with Kafka (1=connected, 0=disconnected)", + 'kafka_connection_status', + 'Connection status with Kafka (1=connected, 0=disconnected)', POD_ID_LABEL, ) # --- Resource Manager (Redis) Metrics --- REDIS_OPERATIONS_TOTAL = Counter( - "redis_operations_total", "Total number of Redis operations performed", REDIS_LABELS + 'redis_operations_total', 'Total number of Redis operations performed', REDIS_LABELS ) REDIS_OPERATIONS_ERRORS = Counter( - "redis_operations_errors_total", - "Total number of errors in Redis operations", + 'redis_operations_errors_total', + 'Total number of errors in Redis operations', REDIS_LABELS, ) REDIS_OPERATIONS_DURATION = Histogram( - "redis_operations_duration_seconds", - "Duration of Redis operations in seconds", + 'redis_operations_duration_seconds', + 'Duration of Redis operations in seconds', REDIS_LABELS, ) REDIS_CONNECTION_STATUS = Gauge( - "redis_connection_status", - "Connection status with Redis (1=connected, 0=disconnected)", + 'redis_connection_status', + 'Connection status with Redis (1=connected, 0=disconnected)', POD_ID_LABEL, ) # --- Notification Metrics --- NOTIFICATIONS_SENT = Counter( - "notifications_sent_total", - "Total number of notifications sent", + 'notifications_sent_total', + 'Total number of notifications sent', NOTIFICATION_LABELS, ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ae13d36 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,159 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "ingestor" +version = "0.0.0" +description = "Sientia DataOps Ingestor - OPC Tag Ingestor" +readme = "README.md" +requires-python = ">=3.11" +authors = [ + {name = "Aignosi", email = "dev@aignosi.com"} +] + +[tool.ruff] +line-length = 100 +target-version = "py311" +exclude = [ + ".git", + ".venv", + "venv", + "__pycache__", + "*.pyc", + ".pytest_cache", + "htmlcov", +] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "UP", # pyupgrade + "N", # pep8-naming + "YTT", # flake8-2020 + "S", # flake8-bandit + "BLE", # flake8-blind-except + "A", # flake8-builtins + "C90", # mccabe complexity +] + +ignore = [ + "BLE001", # ignore blind except, we need to send notifications with any error + "E501", # line too long (handled by formatter) + "S101", # use of assert (needed for tests) + "S105", # possible hardcoded password (false positives) + "S106", # possible hardcoded password (false positives) + "N802", # function name should be lowercase (temporal decorators) + "N806", # variable in function should be lowercase +] + +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = [ + "S101", # assert allowed in tests + "S105", # hardcoded passwords ok in tests + "S106", # hardcoded passwords ok in tests +] + +[tool.ruff.lint.mccabe] +max-complexity = 15 + +[tool.ruff.format] +quote-style = "single" +indent-style = "space" +line-ending = "auto" + +[tool.mypy] +python_version = "3.11" +warn_return_any = false +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = false +warn_no_return = true +strict_equality = true +ignore_missing_imports = true + +# Ignore missing imports for external packages +[[tool.mypy.overrides]] +module = "temporalio.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sientia_do.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mlflow.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "prometheus_client.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sientia.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pandas.*" +ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "-v", + "--strict-markers", + "--cov=model_manager", + "--cov-report=term-missing", + "--cov-report=html", + "--cov-report=xml", +] +markers = [ + "asyncio: marks tests as async", + "integration: marks tests as integration tests", + "unit: marks tests as unit tests", +] + +[tool.coverage.run] +source = ["model_manager"] +omit = [ + "*/tests/*", + "*/venv/*", + "*/__pycache__/*", + "*/site-packages/*", +] +branch = true + +[tool.coverage.report] +precision = 2 +show_missing = true +skip_covered = false +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "def __str__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", + "class .*\\bProtocol\\):", + "@(abc\\.)?abstractmethod", +] + +[tool.coverage.html] +directory = "htmlcov" + +[tool.bandit] +exclude_dirs = ["tests", "venv", ".venv"] +skips = ["B101", "B601"] # Skip assert and shell injection in controlled environments \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..56ab376 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,19 @@ +# Development and Testing Dependencies +# These packages are only needed for development, testing, and code quality checks +# Install with: pip install -r requirements-dev.txt + +# Code Quality & Linting +ruff>=0.1.0 # Fast Python linter and formatter (replaces flake8, black, isort) +mypy>=1.7.0 # Static type checker +bandit>=1.7.5 # Security vulnerability scanner +pandas-stubs>=2.0.0 # Type stubs for pandas +types-requests>=2.31.0 # Type stubs for requests + +# Testing +pytest>=7.4.0 # Testing framework +pytest-cov>=4.1.0 # Coverage plugin for pytest +pytest-asyncio>=0.21.0 # Async test support (already in main requirements) + +# Development Tools +ipython>=8.12.0 # Enhanced Python shell +ipdb>=0.13.13 # IPython debugger \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ecc7949..1174a65 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ asyncua==1.1.5 redis -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.3 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.6 prometheus_client pymongo \ No newline at end of file diff --git a/tests/unit/managers/test_data_manager.py b/tests/unit/managers/test_data_manager.py index 344b203..037579e 100644 --- a/tests/unit/managers/test_data_manager.py +++ b/tests/unit/managers/test_data_manager.py @@ -1,33 +1,34 @@ from unittest.mock import ANY, MagicMock, patch -from pytest import fixture + from kafka.errors import NoBrokersAvailable +from pytest import fixture from sientia_do.notifications.models import NotificationLevel + from ingestor.managers.data_manager import DataManager metadata = { - "metadata": { - "model_id": "test_model", - "model_name": "test_model", - "workflow_name": "test_workflow", - "schema_name": "test_schedule", - "pod_id": "localhost", + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', + 'pod_id': 'localhost', }, } @fixture -@patch("ingestor.managers.data_manager.KafkaProducer") -@patch("ingestor.managers.data_manager.MongoClient") +@patch('ingestor.managers.data_manager.KafkaProducer') +@patch('ingestor.managers.data_manager.MongoClient') def data_manager(mongo, kafka): - data_manager = DataManager( - kafka_servers="localhost:9092", - mongo_connection_string="mongodb://localhost:27017", - mongo_database="sientia", + kafka_servers='localhost:9092', + mongo_connection_string='mongodb://localhost:27017', + mongo_database='sientia', export_to_kafka=True, logger=MagicMock(), notification_handler=MagicMock(), - metadata=metadata["metadata"], + metadata=metadata['metadata'], ) data_manager.send_notification = MagicMock() @@ -35,112 +36,106 @@ def data_manager(mongo, kafka): return data_manager -@patch("ingestor.managers.data_manager.KafkaProducer") -@patch("ingestor.managers.data_manager.MongoClient") +@patch('ingestor.managers.data_manager.KafkaProducer') +@patch('ingestor.managers.data_manager.MongoClient') def test___init___success(mongo, kafka): logger_mock = MagicMock() data_manager = DataManager( - metadata=metadata["metadata"], - kafka_servers="localhost:9092", - mongo_connection_string="mongodb://localhost:27017", - mongo_database="sientia", + metadata=metadata['metadata'], + kafka_servers='localhost:9092', + mongo_connection_string='mongodb://localhost:27017', + mongo_database='sientia', export_to_kafka=True, logger=logger_mock, - notification_handler=MagicMock() + notification_handler=MagicMock(), ) kafka.assert_called_once_with( - bootstrap_servers="localhost:9092", - value_serializer=ANY, - key_serializer=ANY + bootstrap_servers='localhost:9092', value_serializer=ANY, key_serializer=ANY ) assert data_manager.kafka_producer is not None logger_mock.info.assert_any_call( - "Trying (0) to initializing DataManager with Kafka servers: localhost:9092" - ) - logger_mock.info.assert_any_call( - "DataManager initialized with Kafka servers: localhost:9092" + 'Trying (0) to initializing DataManager with Kafka servers: localhost:9092' ) + logger_mock.info.assert_any_call('DataManager initialized with Kafka servers: localhost:9092') logger_mock.error.assert_not_called() assert logger_mock.info.call_count == 4 -@patch("ingestor.managers.data_manager.KafkaProducer") -@patch("ingestor.managers.data_manager.MongoClient") +@patch('ingestor.managers.data_manager.KafkaProducer') +@patch('ingestor.managers.data_manager.MongoClient') def test___init___second_attempt(mongo, kafka): kafka.side_effect = [NoBrokersAvailable, MagicMock()] logger_mock = MagicMock() data_manager = DataManager( - kafka_servers="localhost:9092", - mongo_connection_string="mongodb://localhost:27017", - mongo_database="sientia", + kafka_servers='localhost:9092', + mongo_connection_string='mongodb://localhost:27017', + mongo_database='sientia', export_to_kafka=True, logger=logger_mock, notification_handler=MagicMock(), - metadata=metadata["metadata"], + metadata=metadata['metadata'], ) kafka.assert_any_call( - bootstrap_servers="localhost:9092", - value_serializer=ANY, - key_serializer=ANY + bootstrap_servers='localhost:9092', value_serializer=ANY, key_serializer=ANY ) assert kafka.call_count == 2 assert data_manager.kafka_producer is not None logger_mock.info.assert_any_call( - "Trying (0) to initializing DataManager with Kafka servers: localhost:9092" + 'Trying (0) to initializing DataManager with Kafka servers: localhost:9092' ) logger_mock.info.assert_any_call( - "Trying (1) to initializing DataManager with Kafka servers: localhost:9092" - ) - logger_mock.info.assert_any_call( - "DataManager initialized with Kafka servers: localhost:9092" + 'Trying (1) to initializing DataManager with Kafka servers: localhost:9092' ) + logger_mock.info.assert_any_call('DataManager initialized with Kafka servers: localhost:9092') logger_mock.error.assert_called_once_with( - "Kafka servers localhost:9092 are not available. Retrying..." + 'Kafka servers localhost:9092 are not available. Retrying...' ) assert logger_mock.info.call_count == 5 -@patch("ingestor.managers.data_manager.KafkaProducer") -@patch("ingestor.managers.data_manager.MongoClient") +@patch('ingestor.managers.data_manager.KafkaProducer') +@patch('ingestor.managers.data_manager.MongoClient') def test___init___failure_max_attempts(mongo, kafka): kafka.side_effect = NoBrokersAvailable logger_mock = MagicMock() try: DataManager( - kafka_servers="localhost:9092", - mongo_connection_string="mongodb://localhost:27017", - mongo_database="sientia", + kafka_servers='localhost:9092', + mongo_connection_string='mongodb://localhost:27017', + mongo_database='sientia', export_to_kafka=True, logger=logger_mock, notification_handler=MagicMock(), - metadata=metadata["metadata"], + metadata=metadata['metadata'], ) except NoBrokersAvailable as e: - assert str( - e) == "NoBrokersAvailable: Failed to connect to Kafka servers localhost:9092 after 3 attempts." + assert ( + str(e) + == 'NoBrokersAvailable: Failed to connect to Kafka servers localhost:9092 after 3 attempts.' + ) assert kafka.call_count == 3 logger_mock.info.assert_any_call( - "Trying (0) to initializing DataManager with Kafka servers: localhost:9092" + 'Trying (0) to initializing DataManager with Kafka servers: localhost:9092' ) logger_mock.info.assert_any_call( - "Trying (1) to initializing DataManager with Kafka servers: localhost:9092" + 'Trying (1) to initializing DataManager with Kafka servers: localhost:9092' ) logger_mock.info.assert_any_call( - "Trying (2) to initializing DataManager with Kafka servers: localhost:9092" + 'Trying (2) to initializing DataManager with Kafka servers: localhost:9092' ) logger_mock.error.assert_called_with( - "Failed to connect to Kafka servers localhost:9092 after 3 attempts." + 'Failed to connect to Kafka servers localhost:9092 after 3 attempts.' ) assert logger_mock.info.call_count == 3 else: - assert False, "Expected NoBrokersAvailable exception was not raised." + raise AssertionError('Expected NoBrokersAvailable exception was not raised.') def test_shutdown_has_producer(data_manager): @@ -161,7 +156,7 @@ def test_shutdown_no_producer(data_manager): data_manager.shutdown() data_manager.logger.warning.assert_any_call( - "Kafka producer is already closed or not initialized." + 'Kafka producer is already closed or not initialized.' ) @@ -171,30 +166,24 @@ def test_shutdown_no_mongo_client(data_manager): data_manager.shutdown() data_manager.logger.warning.assert_any_call( - "MongoDB client is already closed or not initialized." + 'MongoDB client is already closed or not initialized.' ) def test_shutdown_exception(data_manager): - data_manager.kafka_producer.flush = MagicMock( - side_effect=Exception("Test error")) + data_manager.kafka_producer.flush = MagicMock(side_effect=Exception('Test error')) data_manager.kafka_producer.close = MagicMock() data_manager.shutdown() - data_manager.logger.error.assert_called_once_with( - "Error closing Kafka producer: Test error" - ) + data_manager.logger.error.assert_called_once_with('Error closing Kafka producer: Test error') def test_shutdown_exception_mongo(data_manager): - data_manager.mongo_client.close = MagicMock( - side_effect=Exception("Test error")) + data_manager.mongo_client.close = MagicMock(side_effect=Exception('Test error')) data_manager.shutdown() - data_manager.logger.error.assert_called_once_with( - "Error closing MongoDB client: Test error" - ) + data_manager.logger.error.assert_called_once_with('Error closing MongoDB client: Test error') def test___del__(data_manager): @@ -205,29 +194,27 @@ def test___del__(data_manager): def test_delivery_report(data_manager): msg = MagicMock() - msg.topic = "test_topic" + msg.topic = 'test_topic' msg.partition = 0 msg.offset = 1 data_manager.delivery_report(msg) data_manager.logger.debug.assert_called_once_with( - f"Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}" + f'Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}' ) def test_delivery_error(data_manager): - err = "Test error" + err = 'Test error' data_manager.delivery_error(err) - data_manager.logger.error.assert_called_once_with( - f"Delivery failed for record : {err}" - ) + data_manager.logger.error.assert_called_once_with(f'Delivery failed for record : {err}') def test_publish(data_manager): - topic = "test_topic" - data = {"key": "value"} + topic = 'test_topic' + data = {'key': 'value'} # Mock the send method of the Kafka producer send_mock = MagicMock() @@ -237,9 +224,7 @@ def test_publish(data_manager): data_manager.publish(topic, data) # Check if the send method was called with the correct arguments - send_mock.assert_called_once_with( - topic=topic, value=data - ) + send_mock.assert_called_once_with(topic=topic, value=data) send_mock.return_value.add_callback.assert_called_once() @@ -248,57 +233,53 @@ def test_publish(data_manager): def test_publish_no_kafka(data_manager): data_manager.export_to_kafka = False - topic = "test_topic" - data = {"key": "value"} + topic = 'test_topic' + data = {'key': 'value'} data_manager.publish(topic, data) data_manager.kafka_producer.send.assert_not_called() -@patch("ingestor.managers.data_manager.traceback") +@patch('ingestor.managers.data_manager.traceback') def test_publish_error(traceback, data_manager): - topic = "test_topic" - data = { - "key": "value", - "name": "test_tag" - } + topic = 'test_topic' + data = {'key': 'value', 'name': 'test_tag'} # Mock the send method of the Kafka producer to raise an exception - send_mock = MagicMock(side_effect=Exception("Test error")) + send_mock = MagicMock(side_effect=Exception('Test error')) data_manager.kafka_producer.send = send_mock # Call the publish method data_manager.publish(topic, data) # Check if the send method was called with the correct arguments - send_mock.assert_called_once_with( - topic=topic, value=data - ) + send_mock.assert_called_once_with(topic=topic, value=data) # Check if the error was logged data_manager.send_notification.assert_called_once_with( - notification_id=f"KAFKA_PRODUCER_ERROR_{topic}", - message=f"Error publishing message to topic {topic}: Test error", - block="kafka_producer", + notification_id=f'KAFKA_PRODUCER_ERROR_{topic}', + message=f'Error publishing message to topic {topic}: Test error', + block='kafka_producer', level=NotificationLevel.ERROR, attachment_content=traceback.format_exc.return_value, - metadata=metadata["metadata"] + metadata=metadata['metadata'], ) def test_publish_error_mongo(data_manager): data_manager.export_to_kafka = False data_manager.mongo_db.__getitem__.return_value.insert_one = MagicMock( - side_effect=Exception("Test error")) + side_effect=Exception('Test error') + ) - data_manager.publish("test_topic", {"key": "value"}) + data_manager.publish('test_topic', {'key': 'value'}) data_manager.send_notification.assert_called_once_with( - notification_id="MONGO_PRODUCER_ERROR_test_topic", - message="Error inserting message to MongoDB: Test error", - block="mongo_producer", + notification_id='MONGO_PRODUCER_ERROR_test_topic', + message='Error inserting message to MongoDB: Test error', + block='mongo_producer', level=NotificationLevel.ERROR, attachment_content=ANY, - metadata=metadata["metadata"] + metadata=metadata['metadata'], ) diff --git a/tests/unit/managers/test_ingestor_manager.py b/tests/unit/managers/test_ingestor_manager.py index 7dc297d..2625fd3 100644 --- a/tests/unit/managers/test_ingestor_manager.py +++ b/tests/unit/managers/test_ingestor_manager.py @@ -1,14 +1,16 @@ from unittest.mock import AsyncMock, MagicMock, patch + from pytest import fixture, mark from sientia_do.notifications.models import NotificationLevel + from ingestor.managers.ingestor_manager import IngestorManager metadata = { - "metadata": { - "model_id": "test_model", - "model_name": "test_model", - "workflow_name": "test_workflow", - "schema_name": "test_schedule", + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', }, } @@ -18,20 +20,17 @@ metadata = { @patch('ingestor.managers.ingestor_manager.ResourceManager') def ingestor_manager(data_manager_mock, resource_manager_mock): ingestor = IngestorManager( - kafka_servers="localhost:9092", - redis_data={ - "host": "localhost", - "port": 6379 - }, + kafka_servers='localhost:9092', + redis_data={'host': 'localhost', 'port': 6379}, lease_ttl=60, heartbeat_ttl=60, poll_interval=5, - mongo_connection_string="mongodb://localhost:27017", - mongo_database="sientia", + mongo_connection_string='mongodb://localhost:27017', + mongo_database='sientia', export_to_kafka=False, logger=MagicMock(), notification_handler=MagicMock(), - metadata=metadata["metadata"], + metadata=metadata['metadata'], ) ingestor.send_notification = MagicMock() @@ -43,41 +42,39 @@ def ingestor_manager(data_manager_mock, resource_manager_mock): @patch('ingestor.managers.ingestor_manager.DataManager') @patch('ingestor.managers.ingestor_manager.ResourceManager') @patch('ingestor.managers.ingestor_manager.NotificationHandler') -def test___init__(notification_handler_mock, resource_manager_mock, data_manager_mock, opc_manager_mock): - +def test___init__( + notification_handler_mock, resource_manager_mock, data_manager_mock, opc_manager_mock +): ingestor = IngestorManager( - kafka_servers="localhost:9092", - redis_data={ - "host": "localhost", - "port": 6379 - }, + kafka_servers='localhost:9092', + redis_data={'host': 'localhost', 'port': 6379}, lease_ttl=60, heartbeat_ttl=60, poll_interval=5, - mongo_connection_string="mongodb://localhost:27017", - mongo_database="sientia", + mongo_connection_string='mongodb://localhost:27017', + mongo_database='sientia', export_to_kafka=False, logger=MagicMock(), notification_handler=MagicMock(), - metadata=metadata["metadata"], + metadata=metadata['metadata'], ) opc_manager_mock.assert_not_called() data_manager_mock.assert_called_once_with( - kafka_servers="localhost:9092", - mongo_connection_string="mongodb://localhost:27017", - mongo_database="sientia", + kafka_servers='localhost:9092', + mongo_connection_string='mongodb://localhost:27017', + mongo_database='sientia', export_to_kafka=False, - metadata=metadata["metadata"], + metadata=metadata['metadata'], logger=ingestor.logger, notification_handler=ingestor.notification_handler, ) resource_manager_mock.assert_called_once_with( - host="localhost", + host='localhost', port=6379, lease_ttl=60, heartbeat_ttl=60, - metadata=metadata["metadata"], + metadata=metadata['metadata'], logger=ingestor.logger, notification_handler=ingestor.notification_handler, username=None, @@ -101,12 +98,11 @@ async def test_initialize_opc_from_config(opc_manager, ingestor_manager): 'cert_path': '/path/to/cert', 'private_key_path': '/path/to/private_key', 'server_cert_path': '/path/to/server_cert', - 'pod_id': 'test_pod' + 'pod_id': 'test_pod', } opc_manager.return_value = MagicMock(connect=AsyncMock()) - result = await ingestor_manager.initialize_opc_from_config( - server_config) + result = await ingestor_manager.initialize_opc_from_config(server_config) opc_manager.assert_called_once_with( name=server_config['name'], @@ -118,7 +114,7 @@ async def test_initialize_opc_from_config(opc_manager, ingestor_manager): cert_path=server_config['cert_path'], private_key_path=server_config['private_key_path'], server_cert_path=server_config['server_cert_path'], - metadata=metadata["metadata"], + metadata=metadata['metadata'], ) assert result == opc_manager.return_value @@ -135,25 +131,24 @@ async def test_initialize_opc_from_config_exception(traceback_mock, opc_manager, 'server_uri': 'http://opcua-server.simulator', 'cert_path': '/path/to/cert', 'private_key_path': '/path/to/private_key', - 'server_cert_path': '/path/to/server_cert' + 'server_cert_path': '/path/to/server_cert', } ingestor_manager.logger.error = MagicMock() - opc_manager.side_effect = Exception("Initialization error") + opc_manager.side_effect = Exception('Initialization error') - result = await ingestor_manager.initialize_opc_from_config( - server_config) + result = await ingestor_manager.initialize_opc_from_config(server_config) assert result is None traceback_mock.format_exc.assert_called_once() ingestor_manager.send_notification.assert_called_once_with( - metadata=metadata["metadata"], + metadata=metadata['metadata'], notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}', message='Error initializing OPC manager: Initialization error', - block="opc_manager", + block='opc_manager', level=NotificationLevel.ERROR, - attachment_content=traceback_mock.format_exc.return_value + attachment_content=traceback_mock.format_exc.return_value, ) @@ -161,46 +156,33 @@ async def test_initialize_opc_from_config_exception(traceback_mock, opc_manager, @patch('ingestor.managers.ingestor_manager.OpcManager') @patch('ingestor.managers.ingestor_manager.metrics') async def test_update_opc_servers(metrics, opc_manager, ingestor_manager): - manager1 = MagicMock( - config={"config": "config1"} - ) - manager2 = MagicMock( - config={"config": "config2"} - ) - manager3 = MagicMock( - config={"config": "config3"} - ) + manager1 = MagicMock(config={'config': 'config1'}) + manager2 = MagicMock(config={'config': 'config2'}) + manager3 = MagicMock(config={'config': 'config3'}) async def mock_initialize_from_config(config): - if config == {"config": "config1"}: + if config == {'config': 'config1'}: return manager1 - elif config == {"config": "config2"}: + elif config == {'config': 'config2'}: return manager2 - elif config == {"config": "config3"}: + elif config == {'config': 'config3'}: return manager3 else: return None - ingestor_manager.initialize_opc_from_config = AsyncMock( - side_effect=mock_initialize_from_config - ) + ingestor_manager.initialize_opc_from_config = AsyncMock(side_effect=mock_initialize_from_config) ingestor_manager.managed_tags = { - "slot1": { - "server1": {"config": "config1"}, - "server2": {"config": "config2"}, - 'server5': {"config": "config5"} + 'slot1': { + 'server1': {'config': 'config1'}, + 'server2': {'config': 'config2'}, + 'server5': {'config': 'config5'}, }, - "slot2": { - "server3": {"config": "config3"}, - "server1": {"config": "config1"} - } + 'slot2': {'server3': {'config': 'config3'}, 'server1': {'config': 'config1'}}, } - mock = MagicMock( - config={"config": "old_config2"}) - ingestor_manager.opc_managers['server3'] = AsyncMock( - config={"config": "config3"}) + mock = MagicMock(config={'config': 'old_config2'}) + ingestor_manager.opc_managers['server3'] = AsyncMock(config={'config': 'config3'}) ingestor_manager.opc_managers['server2'] = mock ingestor_manager.opc_managers['server4'] = AsyncMock() @@ -208,29 +190,23 @@ async def test_update_opc_servers(metrics, opc_manager, ingestor_manager): assert len(ingestor_manager.opc_managers) == 3 - ingestor_manager.initialize_opc_from_config.assert_any_call( - {"config": "config1"}) - ingestor_manager.initialize_opc_from_config.assert_any_call( - {"config": "config2"}) - ingestor_manager.initialize_opc_from_config.assert_any_call( - {"config": "config5"}) + ingestor_manager.initialize_opc_from_config.assert_any_call({'config': 'config1'}) + ingestor_manager.initialize_opc_from_config.assert_any_call({'config': 'config2'}) + ingestor_manager.initialize_opc_from_config.assert_any_call({'config': 'config5'}) assert ingestor_manager.initialize_opc_from_config.call_count == 3 - assert ingestor_manager.opc_managers['server1'].config == { - "config": "config1"} - assert ingestor_manager.opc_managers['server2'].config == { - "config": "config2"} - assert ingestor_manager.opc_managers['server3'].config == { - "config": "config3"} + assert ingestor_manager.opc_managers['server1'].config == {'config': 'config1'} + assert ingestor_manager.opc_managers['server2'].config == {'config': 'config2'} + assert ingestor_manager.opc_managers['server3'].config == {'config': 'config3'} assert 'server4' not in ingestor_manager.opc_managers assert 'server5' not in ingestor_manager.opc_managers assert ingestor_manager.opc_managers['server2'] != mock - metrics.OPC_MANAGERS_ACTIVE.labels.assert_called_once_with( - pod_id=ingestor_manager.pod_id) + metrics.OPC_MANAGERS_ACTIVE.labels.assert_called_once_with(pod_id=ingestor_manager.pod_id) metrics.OPC_MANAGERS_ACTIVE.labels.return_value.set.assert_called_once_with( - len(ingestor_manager.opc_managers)) + len(ingestor_manager.opc_managers) + ) def test_declare_active(ingestor_manager): @@ -246,8 +222,7 @@ def test_get_active_ingestors(ingestor_manager): def test_get_active_ingestors_empty(ingestor_manager): - ingestor_manager.resource_manager.get_all_ingestors = MagicMock( - return_value=None) + ingestor_manager.resource_manager.get_all_ingestors = MagicMock(return_value=None) result = ingestor_manager.get_active_ingestors() assert result == [] ingestor_manager.resource_manager.get_all_ingestors.assert_called_once() @@ -255,8 +230,7 @@ def test_get_active_ingestors_empty(ingestor_manager): @patch('ingestor.managers.ingestor_manager.metrics') def test_get_number_of_leases_success(metrics, ingestor_manager): - ingestor_manager.resource_manager.get_all_leases = MagicMock( - return_value=["lease1", "lease2"]) + ingestor_manager.resource_manager.get_all_leases = MagicMock(return_value=['lease1', 'lease2']) result = ingestor_manager.get_number_of_leases() assert result == 2 ingestor_manager.resource_manager.get_all_leases.assert_called_once() @@ -265,8 +239,7 @@ def test_get_number_of_leases_success(metrics, ingestor_manager): @patch('ingestor.managers.ingestor_manager.metrics') def test_get_number_of_leases_empty(metrics, ingestor_manager): - ingestor_manager.resource_manager.get_all_leases = MagicMock( - return_value=None) + ingestor_manager.resource_manager.get_all_leases = MagicMock(return_value=None) result = ingestor_manager.get_number_of_leases() assert result == 0 ingestor_manager.resource_manager.get_all_leases.assert_called_once() @@ -275,8 +248,7 @@ def test_get_number_of_leases_empty(metrics, ingestor_manager): @patch('ingestor.managers.ingestor_manager.metrics') def test_get_number_of_slots_success(metrics, ingestor_manager): - ingestor_manager.resource_manager.get_all_slots = MagicMock( - return_value=["slot1", "slot2"]) + ingestor_manager.resource_manager.get_all_slots = MagicMock(return_value=['slot1', 'slot2']) result = ingestor_manager.get_number_of_slots() assert result == 2 ingestor_manager.resource_manager.get_all_slots.assert_called_once() @@ -285,8 +257,7 @@ def test_get_number_of_slots_success(metrics, ingestor_manager): @patch('ingestor.managers.ingestor_manager.metrics') def test_get_number_of_slots_empty(metrics, ingestor_manager): - ingestor_manager.resource_manager.get_all_slots = MagicMock( - return_value=None) + ingestor_manager.resource_manager.get_all_slots = MagicMock(return_value=None) result = ingestor_manager.get_number_of_slots() assert result == 0 ingestor_manager.resource_manager.get_all_slots.assert_called_once() @@ -294,39 +265,32 @@ def test_get_number_of_slots_empty(metrics, ingestor_manager): def test_get_slot_leases_1_success(ingestor_manager): - ingestor_manager.resource_manager.lease_tag = MagicMock( - return_value=True) - ingestor_manager.resource_manager.get_tag_slot = MagicMock( - return_value={"tags": ["tag1"]}) + ingestor_manager.resource_manager.lease_tag = MagicMock(return_value=True) + ingestor_manager.resource_manager.get_tag_slot = MagicMock(return_value={'tags': ['tag1']}) ingestor_manager.number_of_slots = 1 result = ingestor_manager.get_slot_leases() - assert result == { - "1": {"tags": ["tag1"]} - } + assert result == {'1': {'tags': ['tag1']}} def test_get_slot_leases_2_success(ingestor_manager): - ingestor_manager.resource_manager.lease_tag = MagicMock( - side_effect=[True, True]) + ingestor_manager.resource_manager.lease_tag = MagicMock(side_effect=[True, True]) ingestor_manager.resource_manager.get_tag_slot = MagicMock( - side_effect=[{"tags": ["tag1"]}, {"tags": ["tag2"]}]) + side_effect=[{'tags': ['tag1']}, {'tags': ['tag2']}] + ) ingestor_manager.number_of_slots = 2 result = ingestor_manager.get_slot_leases(max_slots=2) - assert result == { - "1": {"tags": ["tag1"]}, - "2": {"tags": ["tag2"]} - } + assert result == {'1': {'tags': ['tag1']}, '2': {'tags': ['tag2']}} def test_get_slot_leases_2_1_none(ingestor_manager): - ingestor_manager.resource_manager.lease_tag = MagicMock( - side_effect=[True, True]) + ingestor_manager.resource_manager.lease_tag = MagicMock(side_effect=[True, True]) ingestor_manager.resource_manager.get_tag_slot = MagicMock( - side_effect=[None, {"tags": ["tag1"]}]) + side_effect=[None, {'tags': ['tag1']}] + ) ingestor_manager.number_of_slots = 1 result = ingestor_manager.get_slot_leases(max_slots=1) @@ -335,10 +299,8 @@ def test_get_slot_leases_2_1_none(ingestor_manager): def test_get_slot_leases_1_failure(ingestor_manager): - ingestor_manager.resource_manager.lease_tag = MagicMock( - return_value=False) - ingestor_manager.resource_manager.get_tag_slot = MagicMock( - return_value={"tags": ["tag1"]}) + ingestor_manager.resource_manager.lease_tag = MagicMock(return_value=False) + ingestor_manager.resource_manager.get_tag_slot = MagicMock(return_value={'tags': ['tag1']}) result = ingestor_manager.get_slot_leases() ingestor_manager.resource_manager.get_tag_slot.assert_not_called() @@ -349,42 +311,30 @@ def test_get_slot_leases_1_failure(ingestor_manager): @mark.asyncio async def test_unsubscribe_slot(ingestor_manager): ingestor_manager.managed_tags = { - "slot1": { - "server1": {"tags": "config1"}, - "server2": {"tags": "config2"} - }, - "slot2": { - "server3": {"tags": "config3"}, - "server1": {"tags": "config1"} - } + 'slot1': {'server1': {'tags': 'config1'}, 'server2': {'tags': 'config2'}}, + 'slot2': {'server3': {'tags': 'config3'}, 'server1': {'tags': 'config1'}}, } ingestor_manager.opc_managers = { - "server1": AsyncMock(), - "server2": AsyncMock(), - "server3": AsyncMock() + 'server1': AsyncMock(), + 'server2': AsyncMock(), + 'server3': AsyncMock(), } - await ingestor_manager.unsubscribe_slot("slot1") + await ingestor_manager.unsubscribe_slot('slot1') - ingestor_manager.opc_managers["server1"].unsubscribe.assert_called_once_with( - "slot1") - ingestor_manager.opc_managers["server2"].unsubscribe.assert_called_once_with( - "slot1") - ingestor_manager.opc_managers["server3"].unsubscribe.assert_not_called() + ingestor_manager.opc_managers['server1'].unsubscribe.assert_called_once_with('slot1') + ingestor_manager.opc_managers['server2'].unsubscribe.assert_called_once_with('slot1') + ingestor_manager.opc_managers['server3'].unsubscribe.assert_not_called() def test_update_slot_config(ingestor_manager): ingestor_manager.managed_tags = { - "slot1": {"config": "old_config"}, - "slot2": {"config": "new_config"}, - "slot3": {"config": "old_config"} + 'slot1': {'config': 'old_config'}, + 'slot2': {'config': 'new_config'}, + 'slot3': {'config': 'old_config'}, } ingestor_manager.resource_manager.get_tag_slot = MagicMock( - side_effect=[ - {"config": "updated_config"}, - {"config": "new_config"}, - None - ] + side_effect=[{'config': 'updated_config'}, {'config': 'new_config'}, None] ) ingestor_manager.update_opc_servers = MagicMock() @@ -393,179 +343,130 @@ def test_update_slot_config(ingestor_manager): ingestor_manager.update_slot_config() - assert ingestor_manager.managed_tags["slot1"] == { - "config": "updated_config"} - assert ingestor_manager.managed_tags["slot2"] == { - "config": "new_config"} - assert "slot3" not in ingestor_manager.managed_tags + assert ingestor_manager.managed_tags['slot1'] == {'config': 'updated_config'} + assert ingestor_manager.managed_tags['slot2'] == {'config': 'new_config'} + assert 'slot3' not in ingestor_manager.managed_tags - ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot1") - ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot3") - ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot2") + ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot1') + ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot3') + ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot2') assert ingestor_manager.resource_manager.renew_tag_lease.call_count == 3 @patch('ingestor.managers.ingestor_manager.metrics') def test_drop_slot_leases(metrics, ingestor_manager): ingestor_manager.resource_manager.drop_tag_lease = MagicMock() - ingestor_manager.drop_slot_leases(["1", "2"]) + ingestor_manager.drop_slot_leases(['1', '2']) - ingestor_manager.resource_manager.drop_tag_lease.assert_any_call("1") - ingestor_manager.resource_manager.drop_tag_lease.assert_any_call("2") + ingestor_manager.resource_manager.drop_tag_lease.assert_any_call('1') + ingestor_manager.resource_manager.drop_tag_lease.assert_any_call('2') - metrics.SLOTS_RELEASED.labels.assert_any_call( - pod_id=ingestor_manager.pod_id) + metrics.SLOTS_RELEASED.labels.assert_any_call(pod_id=ingestor_manager.pod_id) metrics.SLOTS_RELEASED.labels.return_value.inc.assert_any_call() @mark.asyncio async def test_manage_server_no_server(ingestor_manager): - ingestor_manager.opc_managers = { - "server1": MagicMock(), - "server2": MagicMock() - } - server_config = { - 'tags': 'config1' - } + ingestor_manager.opc_managers = {'server1': MagicMock(), 'server2': MagicMock()} + server_config = {'tags': 'config1'} - result = await ingestor_manager.manage_server( - 'slot1', 'server3', server_config, server_config) + result = await ingestor_manager.manage_server('slot1', 'server3', server_config, server_config) assert result == 1 - ingestor_manager.opc_managers["server1"].create_subscription.assert_not_called( - ) - ingestor_manager.opc_managers["server1"].subscribe.assert_not_called() + ingestor_manager.opc_managers['server1'].create_subscription.assert_not_called() + ingestor_manager.opc_managers['server1'].subscribe.assert_not_called() @mark.asyncio async def test_manage_server_create_subscription_failure(ingestor_manager): - ingestor_manager.opc_managers = { - "server1": MagicMock(), - "server2": MagicMock() - } - ingestor_manager.subscriptions = { - "server1": MagicMock() - } - server_config = { - 'tags': 'config1' - } + ingestor_manager.opc_managers = {'server1': MagicMock(), 'server2': MagicMock()} + ingestor_manager.subscriptions = {'server1': MagicMock()} + server_config = {'tags': 'config1'} - ingestor_manager.opc_managers["server1"].create_subscription.side_effect = Exception( - "Subscription error") + ingestor_manager.opc_managers['server1'].create_subscription.side_effect = Exception( + 'Subscription error' + ) - result = await ingestor_manager.manage_server( - 'slot1', 'server1', server_config, server_config) + result = await ingestor_manager.manage_server('slot1', 'server1', server_config, server_config) assert result == 2 - ingestor_manager.opc_managers["server1"].create_subscription.assert_called_once_with( - 'slot1') - ingestor_manager.opc_managers["server1"].subscribe.assert_not_called() + ingestor_manager.opc_managers['server1'].create_subscription.assert_called_once_with('slot1') + ingestor_manager.opc_managers['server1'].subscribe.assert_not_called() @mark.asyncio async def test_manage_server(ingestor_manager): - ingestor_manager.opc_managers = { - "server1": AsyncMock(), - "server2": AsyncMock() - } - ingestor_manager.subscriptions = { - "server1": AsyncMock() - } - server_config = { - 'tags': 'config1' - } + ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()} + ingestor_manager.subscriptions = {'server1': AsyncMock()} + server_config = {'tags': 'config1'} - result = await ingestor_manager.manage_server( - 'slot1', 'server1', server_config, server_config) + result = await ingestor_manager.manage_server('slot1', 'server1', server_config, server_config) assert result == 0 - ingestor_manager.opc_managers["server1"].create_subscription.assert_called_once_with( - 'slot1') - ingestor_manager.opc_managers["server1"].subscribe.assert_called_once_with( - 'slot1', 'config1', ingestor_manager.poll_interval) + ingestor_manager.opc_managers['server1'].create_subscription.assert_called_once_with('slot1') + ingestor_manager.opc_managers['server1'].subscribe.assert_called_once_with( + 'slot1', 'config1', ingestor_manager.poll_interval + ) @patch('ingestor.managers.ingestor_manager.traceback') @mark.asyncio async def test_manage_server_subscribe_failure(traceback_mock, ingestor_manager): - ingestor_manager.opc_managers = { - "server1": AsyncMock(), - "server2": AsyncMock() - } - ingestor_manager.subscriptions = { - "server1": AsyncMock() - } - server_config = { - 'tags': 'config1' - } + ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()} + ingestor_manager.subscriptions = {'server1': AsyncMock()} + server_config = {'tags': 'config1'} - ingestor_manager.opc_managers["server1"].subscribe.side_effect = Exception( - "Subscription error") + ingestor_manager.opc_managers['server1'].subscribe.side_effect = Exception('Subscription error') - result = await ingestor_manager.manage_server( - 'slot1', 'server1', server_config, server_config) + result = await ingestor_manager.manage_server('slot1', 'server1', server_config, server_config) assert result == 2 - ingestor_manager.opc_managers["server1"].create_subscription.assert_called_once_with( - 'slot1') - ingestor_manager.opc_managers["server1"].subscribe.assert_called_once_with( - 'slot1', 'config1', ingestor_manager.poll_interval) - ingestor_manager.opc_managers["server1"].unsubscribe.assert_called_once_with( - 'slot1') + ingestor_manager.opc_managers['server1'].create_subscription.assert_called_once_with('slot1') + ingestor_manager.opc_managers['server1'].subscribe.assert_called_once_with( + 'slot1', 'config1', ingestor_manager.poll_interval + ) + ingestor_manager.opc_managers['server1'].unsubscribe.assert_called_once_with('slot1') traceback_mock.format_exc.assert_called_once() ingestor_manager.send_notification.assert_called_once_with( - metadata=metadata["metadata"], + metadata=metadata['metadata'], notification_id='OPC_SUBSCRIPTION_ERROR_slot1:server1', - message='Failed to subscribe to tags from slot1:server1\n{\'tags\': \'config1\'}: Subscription error', - block="opc_manager", + message="Failed to subscribe to tags from slot1:server1\n{'tags': 'config1'}: Subscription error", + block='opc_manager', level=NotificationLevel.ERROR, - attachment_content=traceback_mock.format_exc.return_value + attachment_content=traceback_mock.format_exc.return_value, ) ingestor_manager.logger.warning.assert_any_call( - "Removing subscription from server server1 for slot slot1" + 'Removing subscription from server server1 for slot slot1' ) @mark.asyncio async def test_subscribe_to_tags(ingestor_manager): - ingestor_manager.manage_server = AsyncMock( - side_effect=[0, 1, 2]) - ingestor_manager.managed_tags = { - "slot1": MagicMock(), - "slot2": MagicMock() - } + ingestor_manager.manage_server = AsyncMock(side_effect=[0, 1, 2]) + ingestor_manager.managed_tags = {'slot1': MagicMock(), 'slot2': MagicMock()} - ingestor_manager.opc_managers = { - "server1": AsyncMock(), - "server2": AsyncMock() - } - ingestor_manager.subscriptions = { - "server1": AsyncMock() - } + ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()} + ingestor_manager.subscriptions = {'server1': AsyncMock()} tags = { 'slot1': { - "server1": {"tags": "config1"}, - "server2": {"tags": "config2"}, - 'server3': {"tags": "config3"}, + 'server1': {'tags': 'config1'}, + 'server2': {'tags': 'config2'}, + 'server3': {'tags': 'config3'}, } } await ingestor_manager.subscribe_to_tags(tags) - ingestor_manager.manage_server.assert_any_call( - 'slot1', 'server1', {"tags": "config1"}, tags) - ingestor_manager.manage_server.assert_any_call( - 'slot1', 'server2', {"tags": "config2"}, tags) - ingestor_manager.manage_server.assert_any_call( - 'slot1', 'server3', {"tags": "config3"}, tags) + ingestor_manager.manage_server.assert_any_call('slot1', 'server1', {'tags': 'config1'}, tags) + ingestor_manager.manage_server.assert_any_call('slot1', 'server2', {'tags': 'config2'}, tags) + ingestor_manager.manage_server.assert_any_call('slot1', 'server3', {'tags': 'config3'}, tags) assert ingestor_manager.manage_server.call_count == 3 - ingestor_manager.managed_tags['slot1'].pop.assert_called_once_with( - 'server3', None) + ingestor_manager.managed_tags['slot1'].pop.assert_called_once_with('server3', None) @patch('ingestor.managers.ingestor_manager.metrics') @@ -574,17 +475,14 @@ def test_check_opc_servers_integrity_all_healthy(metrics, ingestor_manager): opc_manager1 = MagicMock() opc_manager1.check_cycles.return_value = None opc_manager1.check_opc_listenning.return_value = False - opc_manager1.config = {"config": "config1"} + opc_manager1.config = {'config': 'config1'} opc_manager2 = MagicMock() opc_manager2.check_cycles.return_value = None opc_manager2.check_opc_listenning.return_value = False - opc_manager2.config = {"config": "config2"} + opc_manager2.config = {'config': 'config2'} - ingestor_manager.opc_managers = { - "server1": opc_manager1, - "server2": opc_manager2 - } + ingestor_manager.opc_managers = {'server1': opc_manager1, 'server2': opc_manager2} # Mock the initialize_opc_from_config method ingestor_manager.initialize_opc_from_config = MagicMock() @@ -601,10 +499,10 @@ def test_check_opc_servers_integrity_all_healthy(metrics, ingestor_manager): # Verify that no reinitialization was needed ingestor_manager.initialize_opc_from_config.assert_not_called() - metrics.OPC_MANAGERS_ACTIVE.labels.assert_called_once_with( - pod_id=ingestor_manager.pod_id) + metrics.OPC_MANAGERS_ACTIVE.labels.assert_called_once_with(pod_id=ingestor_manager.pod_id) metrics.OPC_MANAGERS_ACTIVE.labels.return_value.set.assert_called_once_with( - len(ingestor_manager.opc_managers)) + len(ingestor_manager.opc_managers) + ) def test_check_opc_servers_integrity_server_lost(ingestor_manager): @@ -612,16 +510,13 @@ def test_check_opc_servers_integrity_server_lost(ingestor_manager): opc_manager = MagicMock() opc_manager.check_cycles.return_value = None opc_manager.check_opc_listenning.return_value = True # Server is lost - opc_manager.config = {"config": "config1"} + opc_manager.config = {'config': 'config1'} - ingestor_manager.opc_managers = { - "server1": opc_manager - } + ingestor_manager.opc_managers = {'server1': opc_manager} # Mock the initialize_opc_from_config method to return a new manager new_manager = MagicMock() - ingestor_manager.initialize_opc_from_config = MagicMock( - return_value=new_manager) + ingestor_manager.initialize_opc_from_config = MagicMock(return_value=new_manager) # Call the method ingestor_manager.check_opc_servers_integrity() @@ -632,26 +527,18 @@ def test_check_opc_servers_integrity_server_lost_with_tags(ingestor_manager): opc_manager = MagicMock() opc_manager.check_cycles.return_value = None opc_manager.check_opc_listenning.return_value = True # Server is lost - opc_manager.config = {"config": "config1"} + opc_manager.config = {'config': 'config1'} - ingestor_manager.opc_managers = { - "server1": opc_manager - } + ingestor_manager.opc_managers = {'server1': opc_manager} # Setup managed tags ingestor_manager.managed_tags = { - "slot1": { - "server1": { - "config": "config1", - "tags": {"tag1": "value1"} - } - } + 'slot1': {'server1': {'config': 'config1', 'tags': {'tag1': 'value1'}}} } # Mock the initialize_opc_from_config method to return a new manager new_manager = MagicMock() - ingestor_manager.initialize_opc_from_config = MagicMock( - return_value=new_manager) + ingestor_manager.initialize_opc_from_config = MagicMock(return_value=new_manager) # Call the method ingestor_manager.check_opc_servers_integrity() diff --git a/tests/unit/managers/test_opc_manager.py b/tests/unit/managers/test_opc_manager.py index cbabcc7..0c4e625 100644 --- a/tests/unit/managers/test_opc_manager.py +++ b/tests/unit/managers/test_opc_manager.py @@ -1,66 +1,68 @@ import json from datetime import datetime from unittest.mock import AsyncMock, MagicMock, call, patch -from pytest import fixture, mark -from asyncua.crypto.security_policies import SecurityPolicyBasic256 + import pytest -from ingestor.managers.opc_manager import OpcManager +from asyncua.crypto.security_policies import SecurityPolicyBasic256 +from pytest import fixture, mark from sientia_do.notifications.models import NotificationLevel +from ingestor.managers.opc_manager import OpcManager + tags = { - "ns=3;i=1001": { - "aggregation_function": "LTS", - "frequency": 1000, - "max_value": 100, - "min_value": 0, - "tag_name": "Counter", + 'ns=3;i=1001': { + 'aggregation_function': 'LTS', + 'frequency': 1000, + 'max_value': 100, + 'min_value': 0, + 'tag_name': 'Counter', }, - "ns=3;i=1003": { - "aggregation_function": "AVG", - "frequency": 1000, - "max_value": 100, - "min_value": 0, - "tag_name": "Random", + 'ns=3;i=1003': { + 'aggregation_function': 'AVG', + 'frequency': 1000, + 'max_value': 100, + 'min_value': 0, + 'tag_name': 'Random', }, - "ns=3;i=1004": { - "aggregation_function": "MDN", - "frequency": 1000, - "max_value": 100, - "min_value": 0, - "tag_name": "Sawtooth", + 'ns=3;i=1004': { + 'aggregation_function': 'MDN', + 'frequency': 1000, + 'max_value': 100, + 'min_value': 0, + 'tag_name': 'Sawtooth', }, } metadata = { - "metadata": { - "model_id": "test_model", - "model_name": "test_model", - "workflow_name": "test_workflow", - "schema_name": "test_schedule", + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', }, } @fixture -@patch("ingestor.managers.opc_manager.metrics") +@patch('ingestor.managers.opc_manager.metrics') def raw_opc_manager(mock_metrics): return OpcManager( - name="TestConnector", - url="opc.tcp://localhost:4840", + name='TestConnector', + url='opc.tcp://localhost:4840', data_manager=MagicMock(), logger=MagicMock(), - server_uri="opc.tcp://localhost:4840", + server_uri='opc.tcp://localhost:4840', notification_handler=MagicMock(), - metadata=metadata["metadata"], + metadata=metadata['metadata'], ) @fixture def opc_manager(raw_opc_manager): raw_opc_manager.client = AsyncMock() - raw_opc_manager.cert_path = "cert.pem" - raw_opc_manager.private_key_path = "private_key.pem" - raw_opc_manager.server_cert_path = "server_cert.pem" + raw_opc_manager.cert_path = 'cert.pem' + raw_opc_manager.private_key_path = 'private_key.pem' + raw_opc_manager.server_cert_path = 'server_cert.pem' raw_opc_manager.send_notification = MagicMock() return raw_opc_manager @@ -68,7 +70,7 @@ def opc_manager(raw_opc_manager): @fixture def opc_manager_subscribed(opc_manager): - opc_manager.subscriptions["sub1"] = AsyncMock() + opc_manager.subscriptions['sub1'] = AsyncMock() return opc_manager @@ -76,7 +78,7 @@ def opc_manager_subscribed(opc_manager): def test___str__(opc_manager): assert ( str(opc_manager) - == "OpcManager(name=TestConnector, url=opc.tcp://localhost:4840, server_uri=opc.tcp://localhost:4840)\nnodes={}, subscriptions={}" + == 'OpcManager(name=TestConnector, url=opc.tcp://localhost:4840, server_uri=opc.tcp://localhost:4840)\nnodes={}, subscriptions={}' ) @@ -91,21 +93,18 @@ async def test_shutdown_success(opc_manager): @mark.asyncio async def test_shutdown_error(opc_manager): - opc_manager.disconnect = AsyncMock(side_effect=Exception("Test error")) + opc_manager.disconnect = AsyncMock(side_effect=Exception('Test error')) await opc_manager.shutdown() - opc_manager.logger.error.assert_called_once_with( - "Error during cleanup: Test error") + opc_manager.logger.error.assert_called_once_with('Error during cleanup: Test error') @mark.asyncio async def test_set_security_success(opc_manager): await opc_manager.set_security() - opc_manager.client.set_application_uri.assert_called_once_with( - opc_manager.server_uri - ) + opc_manager.client.set_application_uri.assert_called_once_with(opc_manager.server_uri) opc_manager.client.set_security.assert_called_once_with( SecurityPolicyBasic256, @@ -114,8 +113,7 @@ async def test_set_security_success(opc_manager): server_certificate=opc_manager.server_cert_path, ) - opc_manager.client.set_secure_channel_timeout.assert_called_once_with( - 10000000) + opc_manager.client.set_secure_channel_timeout.assert_called_once_with(10000000) opc_manager.client.set_session_timeout.assert_called_once_with(10000000) @@ -127,19 +125,16 @@ async def test_set_security_no_cert(opc_manager): try: await opc_manager.set_security() except ValueError as e: - assert ( - str(e) - == "Certificate and private key paths must be provided for secure connection." - ) + assert str(e) == 'Certificate and private key paths must be provided for secure connection.' else: - assert False, "ValueError not raised" + raise AssertionError('ValueError not raised') assert opc_manager.client.set_security.call_count == 0 @mark.asyncio -@patch("ingestor.managers.opc_manager.metrics") -@patch("ingestor.managers.opc_manager.Client") +@patch('ingestor.managers.opc_manager.metrics') +@patch('ingestor.managers.opc_manager.Client') async def test_connect_no_security(client, mock_metrics, raw_opc_manager): raw_opc_manager.set_security = AsyncMock() client.return_value = AsyncMock() @@ -150,26 +145,24 @@ async def test_connect_no_security(client, mock_metrics, raw_opc_manager): raw_opc_manager.client.connect.assert_called_once() raw_opc_manager.set_security.assert_not_called() mock_metrics.OPC_CONNECTIONS_TOTAL.labels.assert_called_once_with( - pod_id=raw_opc_manager.pod_id, - server_name=raw_opc_manager.name + pod_id=raw_opc_manager.pod_id, server_name=raw_opc_manager.name ) mock_metrics.OPC_CONNECTIONS_TOTAL.labels.return_value.inc.assert_called_once() mock_metrics.OPC_CONNECTION_STATUS.labels.assert_called_once_with( pod_id=raw_opc_manager.pod_id, server_name=raw_opc_manager.name, - server_url=raw_opc_manager.url + server_url=raw_opc_manager.url, ) - mock_metrics.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with( - 1) + mock_metrics.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(1) mock_metrics.OPC_CONNECTIONS_FAILED.labels.assert_not_called() @mark.asyncio -@patch("ingestor.managers.opc_manager.Client") +@patch('ingestor.managers.opc_manager.Client') async def test_connect_with_security(client, raw_opc_manager): - raw_opc_manager.cert_path = "cert.pem" - raw_opc_manager.private_key_path = "private_key.pem" - raw_opc_manager.server_cert_path = "server_cert.pem" + raw_opc_manager.cert_path = 'cert.pem' + raw_opc_manager.private_key_path = 'private_key.pem' + raw_opc_manager.server_cert_path = 'server_cert.pem' raw_opc_manager.set_security = AsyncMock() client.return_value = AsyncMock() @@ -181,15 +174,14 @@ async def test_connect_with_security(client, raw_opc_manager): @mark.asyncio -@patch("ingestor.managers.opc_manager.Client") -@patch("ingestor.managers.opc_manager.metrics") +@patch('ingestor.managers.opc_manager.Client') +@patch('ingestor.managers.opc_manager.metrics') async def test_connect_exception_handling_and_metrics( mock_metrics_module, mock_opc_client_class, raw_opc_manager ): mock_client_instance = mock_opc_client_class.return_value - simulated_error_message = "Erro de conexรฃo simulado" - mock_client_instance.connect.side_effect = Exception( - simulated_error_message) + simulated_error_message = 'Erro de conexรฃo simulado' + mock_client_instance.connect.side_effect = Exception(simulated_error_message) opc_manager_instance = raw_opc_manager opc_manager_instance.cert_path = None @@ -207,9 +199,7 @@ async def test_connect_exception_handling_and_metrics( server_name=opc_manager_instance.name, server_url=opc_manager_instance.url, ) - mock_metrics_module.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with( - 0 - ) + mock_metrics_module.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(0) mock_metrics_module.OPC_CONNECTIONS_FAILED.labels.assert_called_once_with( pod_id=opc_manager_instance.pod_id, server_name=opc_manager_instance.name @@ -217,50 +207,48 @@ async def test_connect_exception_handling_and_metrics( mock_metrics_module.OPC_CONNECTIONS_FAILED.labels.return_value.inc.assert_called_once() opc_manager_instance.logger.error.assert_called_once_with( - f"Failed to connect to {opc_manager_instance.name}: {simulated_error_message}" + f'Failed to connect to {opc_manager_instance.name}: {simulated_error_message}' ) @mark.asyncio async def test_create_subscription_no_client(raw_opc_manager): try: - await raw_opc_manager.create_subscription("sub1") + await raw_opc_manager.create_subscription('sub1') except ValueError as e: - assert str(e) == "Client not connected. Call connect first." + assert str(e) == 'Client not connected. Call connect first.' else: - assert False, "ValueError not raised" + raise AssertionError('ValueError not raised') @mark.asyncio async def test_create_subscription_success_has_period(opc_manager): - await opc_manager.create_subscription("sub1", 1000) + await opc_manager.create_subscription('sub1', 1000) - opc_manager.client.create_subscription.assert_called_once_with( - 1000, opc_manager) - assert opc_manager.subscriptions["sub1"] is not None + opc_manager.client.create_subscription.assert_called_once_with(1000, opc_manager) + assert opc_manager.subscriptions['sub1'] is not None @mark.asyncio async def test_create_subscription_success_no_period(opc_manager): - await opc_manager.create_subscription("sub1", None) + await opc_manager.create_subscription('sub1', None) - opc_manager.client.create_subscription.assert_called_once_with( - 500, opc_manager) - assert opc_manager.subscriptions["sub1"] is not None + opc_manager.client.create_subscription.assert_called_once_with(500, opc_manager) + assert opc_manager.subscriptions['sub1'] is not None -@patch("ingestor.managers.opc_manager.metrics") +@patch('ingestor.managers.opc_manager.metrics') @mark.asyncio async def test_create_subscription_with_metrics(metrics, opc_manager): - await opc_manager.create_subscription("sub1", 1000) + await opc_manager.create_subscription('sub1', 1000) metrics.OPC_SUBSCRIPTIONS_CREATED.labels.assert_called_once_with( - pod_id=opc_manager.pod_id, server_name=opc_manager.name, slot_name="sub1" + pod_id=opc_manager.pod_id, server_name=opc_manager.name, slot_name='sub1' ) metrics.OPC_SUBSCRIPTIONS_CREATED.labels.return_value.inc.assert_called_once() -@patch("ingestor.managers.opc_manager.metrics") +@patch('ingestor.managers.opc_manager.metrics') @mark.asyncio async def test_create_subscription_exception_during_client_call( mock_metrics_module, raw_opc_manager @@ -268,71 +256,66 @@ async def test_create_subscription_exception_during_client_call( opc_manager_instance = raw_opc_manager opc_manager_instance.client = AsyncMock() - subscription_name = "test_sub_client_error" + subscription_name = 'test_sub_client_error' simulated_period = 750 - simulated_error_message = "Falha ao criar subscriรงรฃo no cliente OPC" + simulated_error_message = 'Falha ao criar subscriรงรฃo no cliente OPC' - opc_manager_instance.client.create_subscription.side_effect = Exception( - simulated_error_message - ) + opc_manager_instance.client.create_subscription.side_effect = Exception(simulated_error_message) with pytest.raises(Exception, match=simulated_error_message): - await opc_manager_instance.create_subscription( - subscription_name, period=simulated_period - ) + await opc_manager_instance.create_subscription(subscription_name, period=simulated_period) opc_manager_instance.client.create_subscription.assert_called_once_with( simulated_period, opc_manager_instance ) opc_manager_instance.logger.error.assert_called_once_with( - f"Failed to create subscription {subscription_name} on {opc_manager_instance.name}: {simulated_error_message}" + f'Failed to create subscription {subscription_name} on {opc_manager_instance.name}: {simulated_error_message}' ) mock_metrics_module.OPC_SUBSCRIPTIONS_CREATED.labels.assert_not_called() -@patch("ingestor.managers.opc_manager.metrics") +@patch('ingestor.managers.opc_manager.metrics') @mark.asyncio async def test_subscribe_no_subscription(metrics, opc_manager): try: - await opc_manager.subscribe("sub1", tags, 1000) + await opc_manager.subscribe('sub1', tags, 1000) except ValueError as e: - assert str( - e) == "Subscription not created. Call create_subscription first." + assert str(e) == 'Subscription not created. Call create_subscription first.' else: - assert False, "ValueError not raised" + raise AssertionError('ValueError not raised') metrics.OPC_TAGS_SUBSCRIBED.labels.assert_not_called() @mark.asyncio async def test_subscribe_success(opc_manager_subscribed): opc_manager_subscribed.client.get_node = MagicMock() - opc_manager_subscribed.nodes = {"ns=3;i=1001": "data"} + opc_manager_subscribed.nodes = {'ns=3;i=1001': 'data'} - await opc_manager_subscribed.subscribe("sub1", tags, 1000) + await opc_manager_subscribed.subscribe('sub1', tags, 1000) assert opc_manager_subscribed.nodes == tags - opc_manager_subscribed.subscriptions["sub1"].subscribe_data_change.assert_called_once_with( + opc_manager_subscribed.subscriptions['sub1'].subscribe_data_change.assert_called_once_with( [opc_manager_subscribed.client.get_node(n) for n in tags] ) @mark.asyncio async def test_unsubscribe_no_subscription(opc_manager): - await opc_manager.unsubscribe("sub1") + await opc_manager.unsubscribe('sub1') opc_manager.logger.warning.assert_called_once_with( "Subscription 'sub1' not found. Cannot unsubscribe." ) - assert opc_manager.subscriptions.get("sub1") is None + assert opc_manager.subscriptions.get('sub1') is None @mark.asyncio async def test_unsubscribe_success(opc_manager_subscribed): - await opc_manager_subscribed.unsubscribe("sub1") + await opc_manager_subscribed.unsubscribe('sub1') - opc_manager_subscribed.subscriptions.get("sub1") is None + assert opc_manager_subscribed.subscriptions.get('sub1') is None @mark.asyncio @@ -341,7 +324,7 @@ async def test_disconnect_success(opc_manager_subscribed): await opc_manager_subscribed.disconnect() - opc_manager_subscribed.subscriptions["sub1"].delete.assert_called_once() + opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once() assert opc_manager_subscribed.client is None @@ -352,42 +335,38 @@ async def test_disconnect_no_client(opc_manager_subscribed): opc_manager_subscribed.logger.warning.assert_has_calls( [ - call("Client already disconnected."), + call('Client already disconnected.'), ] ) @mark.asyncio async def test_disconnect_error_unsubscribe(opc_manager_subscribed): - opc_manager_subscribed.client = MagicMock( - disconnect=AsyncMock() - ) - opc_manager_subscribed.subscriptions["sub1"] = MagicMock( - delete=AsyncMock(side_effect=Exception("Test error")) + opc_manager_subscribed.client = MagicMock(disconnect=AsyncMock()) + opc_manager_subscribed.subscriptions['sub1'] = MagicMock( + delete=AsyncMock(side_effect=Exception('Test error')) ) await opc_manager_subscribed.disconnect() - opc_manager_subscribed.subscriptions["sub1"].delete.assert_called_once() + opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once() opc_manager_subscribed.client = None opc_manager_subscribed.logger.error.assert_called_once_with( - "Failed to clean up subscription: Test error" + 'Failed to clean up subscription: Test error' ) @mark.asyncio async def test_disconnect_error(opc_manager_subscribed): opc_manager_subscribed.client = MagicMock() - opc_manager_subscribed.client.disconnect = MagicMock( - side_effect=Exception("Test error") - ) + opc_manager_subscribed.client.disconnect = MagicMock(side_effect=Exception('Test error')) await opc_manager_subscribed.disconnect() - opc_manager_subscribed.subscriptions["sub1"].delete.assert_called_once() + opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once() opc_manager_subscribed.client = None opc_manager_subscribed.logger.error.assert_called_once_with( - "Failed to disconnect from OPC UA server: Test error" + 'Failed to disconnect from OPC UA server: Test error' ) @@ -400,24 +379,21 @@ async def test_disconnect_metrics_on_successful_path(mock_metrics_module, raw_op raw_opc_manager.client = MagicMock() mock_sub1 = MagicMock() mock_sub2 = MagicMock() - raw_opc_manager.subscriptions = {"sub1": mock_sub1, "sub2": mock_sub2} + raw_opc_manager.subscriptions = {'sub1': mock_sub1, 'sub2': mock_sub2} await raw_opc_manager.disconnect() mock_metrics_module.OPC_CONNECTION_STATUS.labels.assert_called_once_with( pod_id=raw_opc_manager.pod_id, server_name=raw_opc_manager.name, - server_url=raw_opc_manager.url + server_url=raw_opc_manager.url, ) - mock_metrics_module.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with( - 0) + mock_metrics_module.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(0) mock_metrics_module.OPC_TAGS_SUBSCRIBED.labels.assert_called_once_with( - pod_id=raw_opc_manager.pod_id, - server_name=raw_opc_manager.name + pod_id=raw_opc_manager.pod_id, server_name=raw_opc_manager.name ) - mock_metrics_module.OPC_TAGS_SUBSCRIBED.labels.return_value.set.assert_called_once_with( - 0) + mock_metrics_module.OPC_TAGS_SUBSCRIBED.labels.return_value.set.assert_called_once_with(0) @patch('ingestor.managers.opc_manager.metrics') @@ -427,76 +403,70 @@ async def test_datachange_notification(metrics, opc_manager_subscribed): monitored_item=MagicMock( Value=MagicMock( Value=MagicMock(Value=42), - SourceTimestamp=datetime.strptime( - "2021-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S" - ), + SourceTimestamp=datetime.strptime('2021-01-01T00:00:00', '%Y-%m-%dT%H:%M:%S'), ) ) ) opc_manager_subscribed.nodes = { - "ns=3;i=1001": { - "tag_name": "Counter", - "cycle_rule": {"cycle_increment": 1.0, "cycle_count": 2}, - "topics": ["topic1", "topic2"], + 'ns=3;i=1001': { + 'tag_name': 'Counter', + 'cycle_rule': {'cycle_increment': 1.0, 'cycle_count': 2}, + 'topics': ['topic1', 'topic2'], } } metrics.OPC_CYCLES_WITHOUT_DATA.reset_mock() - await opc_manager_subscribed.datachange_notification("ns=3;i=1001", None, data) + await opc_manager_subscribed.datachange_notification('ns=3;i=1001', None, data) opc_manager_subscribed.data_manager.publish.assert_any_call( - "topic1", + 'topic1', { - "tag": "ns=3;i=1001", - "name": "Counter", - "timestamp": "2021-01-01 00:00:00-0300", - "value": 42, + 'tag': 'ns=3;i=1001', + 'name': 'Counter', + 'timestamp': '2021-01-01 00:00:00-0300', + 'value': 42, }, ) opc_manager_subscribed.data_manager.publish.assert_any_call( - "topic2", + 'topic2', { - "tag": "ns=3;i=1001", - "name": "Counter", - "timestamp": "2021-01-01 00:00:00-0300", - "value": 42, + 'tag': 'ns=3;i=1001', + 'name': 'Counter', + 'timestamp': '2021-01-01 00:00:00-0300', + 'value': 42, }, ) - assert opc_manager_subscribed.nodes["ns=3;i=1001"]["cycle_rule"]["cycle_count"] == 0 + assert opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == 0 metrics.OPC_CYCLES_WITHOUT_DATA.labels.assert_called_once_with( - pod_id=opc_manager_subscribed.pod_id, - server_name=opc_manager_subscribed.name + pod_id=opc_manager_subscribed.pod_id, server_name=opc_manager_subscribed.name ) - metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with( - 0) + metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with(0) def test_check_cycles_no_notification(opc_manager): # Setup: node with cycle_count just below threshold opc_manager.nodes = { - "ns=3;i=1001": { - "tag_name": "Counter", - "cycle_rule": {"cycle_increment": 1.0, "cycle_count": 3.0}, + 'ns=3;i=1001': { + 'tag_name': 'Counter', + 'cycle_rule': {'cycle_increment': 1.0, 'cycle_count': 3.0}, } } opc_manager.check_cycles() # After one increment, cycle_count = 4.0, still below threshold - assert opc_manager.nodes["ns=3;i=1001"]["cycle_rule"][ - "cycle_count" - ] == pytest.approx(4.0) + assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(4.0) opc_manager.send_notification.assert_not_called() def test_check_cycles_triggers_notification(opc_manager): # Setup: node with cycle_count just below threshold, increment will cross threshold opc_manager.nodes = { - "ns=3;i=1001": { - "tag_name": "Counter", - "cycle_rule": {"cycle_increment": 2.5, "cycle_count": 3.0}, + 'ns=3;i=1001': { + 'tag_name': 'Counter', + 'cycle_rule': {'cycle_increment': 2.5, 'cycle_count': 3.0}, } } opc_manager.notification_handler.build_and_send_notification = MagicMock() @@ -504,15 +474,13 @@ def test_check_cycles_triggers_notification(opc_manager): opc_manager.check_cycles() # After increment, cycle_count = 5.5, should trigger notification - assert opc_manager.nodes["ns=3;i=1001"]["cycle_rule"][ - "cycle_count" - ] == pytest.approx(5.5) + assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(5.5) opc_manager.send_notification.assert_called_once_with( - notification_id="TAG_ns=3;i=1001:Counter_LISTENNING_STOPPED", - message="5.5 cycles without receive from ns=3;i=1001:Counter", - block="opc_manager", + notification_id='TAG_ns=3;i=1001:Counter_LISTENNING_STOPPED', + message='5.5 cycles without receive from ns=3;i=1001:Counter', + block='opc_manager', level=NotificationLevel.WARNING, - metadata=metadata["metadata"], + metadata=metadata['metadata'], ) @@ -530,11 +498,11 @@ def test_check_opc_listenning_no_notification(metrics, opc_manager): assert result is False metrics.OPC_CYCLES_WITHOUT_DATA.labels.assert_called_once_with( - pod_id=opc_manager.pod_id, - server_name=opc_manager.name + pod_id=opc_manager.pod_id, server_name=opc_manager.name ) metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with( - opc_manager.non_receive_count) + opc_manager.non_receive_count + ) metrics.OPC_RECONNECTIONS_TOTAL.labels.assert_not_called() @@ -545,11 +513,11 @@ def test_check_opc_listenning_warning_notification(opc_manager): assert opc_manager.non_receive_count == 5 opc_manager.send_notification.assert_called_once_with( - notification_id=f"OPC_LISTENNING_STOPPED__{opc_manager.name}", - message=f"5 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}", - block="opc_manager", + notification_id=f'OPC_LISTENNING_STOPPED__{opc_manager.name}', + message=f'5 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}', + block='opc_manager', level=NotificationLevel.ERROR, - metadata=metadata["metadata"], + metadata=metadata['metadata'], ) assert result is False @@ -569,47 +537,45 @@ def test_check_opc_listenning_error_notification_and_retry(metrics, opc_manager) calls = opc_manager.send_notification.call_args_list # First call: 5 cycles warning assert calls[0].kwargs == { - "notification_id": f"OPC_LISTENNING_STOPPED__{opc_manager.name}", - "message": f"15 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}", - "block": "opc_manager", - "level": NotificationLevel.ERROR, - "metadata": metadata["metadata"], + 'notification_id': f'OPC_LISTENNING_STOPPED__{opc_manager.name}', + 'message': f'15 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}', + 'block': 'opc_manager', + 'level': NotificationLevel.ERROR, + 'metadata': metadata['metadata'], } # Second call: 15 cycles retry assert calls[1].kwargs == { - "notification_id": f"OPC_CONNECTION_RETRY__{opc_manager.name}", - "message": f"Retrying to connect to server {opc_manager.name}", - "block": "opc_manager", - "level": NotificationLevel.ERROR, - "metadata": metadata["metadata"], + 'notification_id': f'OPC_CONNECTION_RETRY__{opc_manager.name}', + 'message': f'Retrying to connect to server {opc_manager.name}', + 'block': 'opc_manager', + 'level': NotificationLevel.ERROR, + 'metadata': metadata['metadata'], } assert result is True metrics.OPC_CYCLES_WITHOUT_DATA.labels.assert_called_once_with( - pod_id=opc_manager.pod_id, - server_name=opc_manager.name + pod_id=opc_manager.pod_id, server_name=opc_manager.name ) metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with( - opc_manager.non_receive_count) + opc_manager.non_receive_count + ) metrics.OPC_RECONNECTIONS_TOTAL.labels.assert_called_once_with( - pod_id=opc_manager.pod_id, - server_name=opc_manager.name + pod_id=opc_manager.pod_id, server_name=opc_manager.name ) metrics.OPC_RECONNECTIONS_TOTAL.labels.return_value.inc.assert_called_once() -@patch("ingestor.managers.opc_manager.metrics") +@patch('ingestor.managers.opc_manager.metrics') def test_init_metrics_calls_correct_metric_methods(metrics): - opc_manager = OpcManager( - name="TestInitConnector", - url="opc.tcp://init.test:4840", + name='TestInitConnector', + url='opc.tcp://init.test:4840', data_manager=MagicMock(), logger=MagicMock(), - server_uri="opc.tcp://init.test:4840/uri", + server_uri='opc.tcp://init.test:4840/uri', notification_handler=MagicMock(), - metadata=metadata["metadata"], + metadata=metadata['metadata'], ) metrics.OPC_CONNECTION_STATUS.labels.assert_called_with( @@ -618,13 +584,10 @@ def test_init_metrics_calls_correct_metric_methods(metrics): server_url=opc_manager.url, ) - metrics.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with( - 0) + metrics.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(0) metrics.OPC_TAGS_SUBSCRIBED.labels.assert_called_once_with( - pod_id=opc_manager.pod_id, - server_name=opc_manager.name + pod_id=opc_manager.pod_id, server_name=opc_manager.name ) - metrics.OPC_TAGS_SUBSCRIBED.labels.return_value.set.assert_called_once_with( - 0) + metrics.OPC_TAGS_SUBSCRIBED.labels.return_value.set.assert_called_once_with(0) diff --git a/tests/unit/managers/test_resource_manager.py b/tests/unit/managers/test_resource_manager.py index 00238ea..9479672 100644 --- a/tests/unit/managers/test_resource_manager.py +++ b/tests/unit/managers/test_resource_manager.py @@ -1,28 +1,29 @@ from unittest.mock import MagicMock, patch + from pytest import fixture, raises from sientia_do.notifications.models import NotificationLevel + from ingestor.managers.resource_manager import ResourceManager metadata = { - "metadata": { - "model_id": "test_model", - "model_name": "test_model", - "workflow_name": "test_workflow", - "schema_name": "test_schedule", + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', }, } @fixture -@patch("ingestor.managers.resource_manager.Redis") +@patch('ingestor.managers.resource_manager.Redis') def resource_manager(redis): - resource_manager = ResourceManager( - host="localhost", + host='localhost', port=6379, lease_ttl=10, heartbeat_ttl=10, - metadata=metadata["metadata"], + metadata=metadata['metadata'], logger=MagicMock(), notification_handler=MagicMock(), ) @@ -34,152 +35,147 @@ def resource_manager(redis): def test_get_success(resource_manager): resource_manager.redis.get.return_value = '{"key": "value"}' - result = resource_manager.get("key") - assert result == {"key": "value"} - resource_manager.redis.get.assert_called_once_with("key") + result = resource_manager.get('key') + assert result == {'key': 'value'} + resource_manager.redis.get.assert_called_once_with('key') def test_get_failure(resource_manager): resource_manager.redis.get.return_value = None - result = resource_manager.get("key") + result = resource_manager.get('key') assert result is None - resource_manager.redis.get.assert_called_once_with("key") + resource_manager.redis.get.assert_called_once_with('key') def test_get_tag_slot(resource_manager): - resource_manager.get = MagicMock(return_value={"tag": "slot"}) - result = resource_manager.get_tag_slot("id") - assert result == {"tag": "slot"} - resource_manager.get.assert_called_once_with("slot:opc_tags:id") + resource_manager.get = MagicMock(return_value={'tag': 'slot'}) + result = resource_manager.get_tag_slot('id') + assert result == {'tag': 'slot'} + resource_manager.get.assert_called_once_with('slot:opc_tags:id') def test_ingestor_heartbeat(resource_manager): resource_manager.redis.set.return_value = True resource_manager.ingestor_heartbeat() - resource_manager.redis.set.assert_called_once_with( - "heartbeat:ingestor:localhost", 1, ex=10 - ) + resource_manager.redis.set.assert_called_once_with('heartbeat:ingestor:localhost', 1, ex=10) def test_lease_tag(resource_manager): resource_manager.redis.set.return_value = True - output = resource_manager.lease_tag("tag_id") + output = resource_manager.lease_tag('tag_id') assert output is True resource_manager.redis.set.assert_called_once_with( - "lease:opc_tags:tag_id", "localhost", nx=True, ex=10 + 'lease:opc_tags:tag_id', 'localhost', nx=True, ex=10 ) def test_renew_tag_lease_success(resource_manager): - resource_manager.redis.get.return_value = "localhost" + resource_manager.redis.get.return_value = 'localhost' resource_manager.redis.expire.return_value = True - result = resource_manager.renew_tag_lease("tag_id") + result = resource_manager.renew_tag_lease('tag_id') assert result is True - resource_manager.redis.get.assert_called_once_with("lease:opc_tags:tag_id") - resource_manager.redis.expire.assert_called_once_with( - "lease:opc_tags:tag_id", 10) + resource_manager.redis.get.assert_called_once_with('lease:opc_tags:tag_id') + resource_manager.redis.expire.assert_called_once_with('lease:opc_tags:tag_id', 10) def test_renew_tag_lease_failure(resource_manager): - resource_manager.redis.get.return_value = "other_pod_id" + resource_manager.redis.get.return_value = 'other_pod_id' resource_manager.redis.expire.return_value = False - result = resource_manager.renew_tag_lease("tag_id") + result = resource_manager.renew_tag_lease('tag_id') assert result is False - resource_manager.redis.get.assert_called_once_with("lease:opc_tags:tag_id") + resource_manager.redis.get.assert_called_once_with('lease:opc_tags:tag_id') resource_manager.redis.expire.assert_not_called() def test_drop_tag_lease(resource_manager): resource_manager.redis.delete.return_value = True - resource_manager.drop_tag_lease("tag_id") - resource_manager.redis.delete.assert_called_once_with( - "lease:opc_tags:tag_id") + resource_manager.drop_tag_lease('tag_id') + resource_manager.redis.delete.assert_called_once_with('lease:opc_tags:tag_id') def test_get_all_ingestors(resource_manager): - resource_manager.redis.keys.return_value = ["ingestor1", "ingestor2"] + resource_manager.redis.keys.return_value = ['ingestor1', 'ingestor2'] result = resource_manager.get_all_ingestors() - assert result == ["ingestor1", "ingestor2"] - resource_manager.redis.keys.assert_called_once_with("heartbeat:ingestor:*") + assert result == ['ingestor1', 'ingestor2'] + resource_manager.redis.keys.assert_called_once_with('heartbeat:ingestor:*') def test_get_all_slots(resource_manager): - resource_manager.redis.keys.return_value = ["slot1", "slot2"] + resource_manager.redis.keys.return_value = ['slot1', 'slot2'] result = resource_manager.get_all_slots() - assert result == ["slot1", "slot2"] - resource_manager.redis.keys.assert_called_once_with("slot:opc_tags:*") + assert result == ['slot1', 'slot2'] + resource_manager.redis.keys.assert_called_once_with('slot:opc_tags:*') def test_get_all_leases(resource_manager): - resource_manager.redis.keys.return_value = ["lease1", "lease2"] + resource_manager.redis.keys.return_value = ['lease1', 'lease2'] result = resource_manager.get_all_leases() - assert result == ["lease1", "lease2"] - resource_manager.redis.keys.assert_called_once_with("lease:opc_tags:*") + assert result == ['lease1', 'lease2'] + resource_manager.redis.keys.assert_called_once_with('lease:opc_tags:*') def test_init_connection_failure(monkeypatch): # Mock Redis to raise an exception during initialization mock_redis = MagicMock() - mock_redis.side_effect = Exception("Connection failed") + mock_redis.side_effect = Exception('Connection failed') - monkeypatch.setattr("ingestor.managers.resource_manager.Redis", mock_redis) + monkeypatch.setattr('ingestor.managers.resource_manager.Redis', mock_redis) # Test that the exception is raised and metrics are set properly - with patch("ingestor.metrics.REDIS_CONNECTION_STATUS") as mock_metrics: + with patch('ingestor.metrics.REDIS_CONNECTION_STATUS') as mock_metrics: mock_status = MagicMock() mock_metrics.labels.return_value = mock_status - with raises(Exception, match="Connection failed"): + with raises(Exception, match='Connection failed'): ResourceManager( - host="localhost", + host='localhost', port=6379, lease_ttl=10, heartbeat_ttl=10, - metadata=metadata["metadata"], + metadata=metadata['metadata'], logger=MagicMock(), notification_handler=MagicMock(), ) - mock_metrics.labels.assert_called_once_with(pod_id="localhost") + mock_metrics.labels.assert_called_once_with(pod_id='localhost') mock_status.set.assert_called_once_with(0) def test_init_ping_failure(monkeypatch): # Mock Redis ping to raise an exception mock_redis_instance = MagicMock() - mock_redis_instance.ping.side_effect = Exception("Ping failed") + mock_redis_instance.ping.side_effect = Exception('Ping failed') mock_redis_class = MagicMock(return_value=mock_redis_instance) - monkeypatch.setattr( - "ingestor.managers.resource_manager.Redis", mock_redis_class) + monkeypatch.setattr('ingestor.managers.resource_manager.Redis', mock_redis_class) # Test that the exception is raised and metrics are set properly - with patch("ingestor.metrics.REDIS_CONNECTION_STATUS") as mock_metrics: + with patch('ingestor.metrics.REDIS_CONNECTION_STATUS') as mock_metrics: mock_status = MagicMock() mock_metrics.labels.return_value = mock_status - with raises(Exception, match="Ping failed"): + with raises(Exception, match='Ping failed'): ResourceManager( - host="localhost", + host='localhost', port=6379, lease_ttl=10, heartbeat_ttl=10, - metadata=metadata["metadata"], + metadata=metadata['metadata'], logger=MagicMock(), notification_handler=MagicMock(), ) - mock_metrics.labels.assert_called_once_with(pod_id="localhost") + mock_metrics.labels.assert_called_once_with(pod_id='localhost') mock_status.set.assert_called_once_with(0) def test_execute_redis_op_success(resource_manager): # Mock the Redis operation and time function - mock_func = MagicMock(return_value="test_result") + mock_func = MagicMock(return_value='test_result') - with patch("ingestor.managers.resource_manager.time", side_effect=[100, 100.5]): - with patch("ingestor.metrics.REDIS_OPERATIONS_TOTAL") as mock_total: - with patch("ingestor.metrics.REDIS_OPERATIONS_DURATION") as mock_duration: + with patch('ingestor.managers.resource_manager.time', side_effect=[100, 100.5]): + with patch('ingestor.metrics.REDIS_OPERATIONS_TOTAL') as mock_total: + with patch('ingestor.metrics.REDIS_OPERATIONS_DURATION') as mock_duration: mock_total_labels = MagicMock() mock_duration_labels = MagicMock() mock_total.labels.return_value = mock_total_labels @@ -187,49 +183,42 @@ def test_execute_redis_op_success(resource_manager): # Execute the operation result = resource_manager._execute_redis_op( - "test_op", mock_func, "arg1", kwarg1="value1" + 'test_op', mock_func, 'arg1', kwarg1='value1' ) # Verify the result and metrics - assert result == "test_result" - mock_func.assert_called_once_with("arg1", kwarg1="value1") + assert result == 'test_result' + mock_func.assert_called_once_with('arg1', kwarg1='value1') - mock_total.labels.assert_called_once_with( - pod_id="localhost", operation="test_op" - ) + mock_total.labels.assert_called_once_with(pod_id='localhost', operation='test_op') mock_total_labels.inc.assert_called_once() mock_duration.labels.assert_called_once_with( - pod_id="localhost", operation="test_op" - ) - mock_duration_labels.observe.assert_called_once_with( - 0.5 + pod_id='localhost', operation='test_op' ) + mock_duration_labels.observe.assert_called_once_with(0.5) def test_execute_redis_op_exception(resource_manager): # Mock the Redis operation to raise an exception - mock_func = MagicMock(side_effect=Exception("Operation failed")) + mock_func = MagicMock(side_effect=Exception('Operation failed')) - with patch("ingestor.managers.resource_manager.time", return_value=100): - with patch("ingestor.metrics.REDIS_OPERATIONS_ERRORS") as mock_errors: + with patch('ingestor.managers.resource_manager.time', return_value=100): + with patch('ingestor.metrics.REDIS_OPERATIONS_ERRORS') as mock_errors: mock_errors_labels = MagicMock() mock_errors.labels.return_value = mock_errors_labels # Execute the operation and expect an exception - with raises(Exception, match="Operation failed"): - resource_manager._execute_redis_op( - "test_op", mock_func, "arg1") + with raises(Exception, match='Operation failed'): + resource_manager._execute_redis_op('test_op', mock_func, 'arg1') # Verify metrics and error handling - mock_errors.labels.assert_called_once_with( - pod_id="localhost", operation="test_op" - ) + mock_errors.labels.assert_called_once_with(pod_id='localhost', operation='test_op') mock_errors_labels.inc.assert_called_once() resource_manager.send_notification.assert_called_once_with( - metadata=metadata["metadata"], - notification_id="REDIS_OPERATION_ERROR_test_op", + metadata=metadata['metadata'], + notification_id='REDIS_OPERATION_ERROR_test_op', message="Error in Redis operation 'test_op': Operation failed", - block="redis_manager", + block='redis_manager', level=NotificationLevel.ERROR, ) diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py index 8d89858..558cf1d 100644 --- a/tests/unit/test_app.py +++ b/tests/unit/test_app.py @@ -1,91 +1,80 @@ -import pytest -import pytest_asyncio -from unittest.mock import patch, MagicMock, call, AsyncMock -from threading import Event -import signal as signal_module # To avoid conflict with mock names -import os import asyncio +import signal as signal_module # To avoid conflict with mock names +from threading import Event +from unittest.mock import AsyncMock, MagicMock, call + +import pytest # Import the 'app' module to be tested from ingestor import app # Custom exception to catch os._exit calls -class OsExitCalled(Exception): +class OsExitCalledError(Exception): def __init__(self, code): - super().__init__(f"os._exit({code}) called") + super().__init__(f'os._exit({code}) called') self.code = code # Helper function for the os_exit mock's side_effect def raise_os_exit_with_code(exit_code): - raise OsExitCalled(exit_code) + raise OsExitCalledError(exit_code) @pytest.fixture def mock_app_env(monkeypatch): """Fixture to mock dependencies of app.main and app.signal_handler.""" mocks = { - "start_http_server": MagicMock(), - "Ingestor": MagicMock(), - "metrics_APP_UP_labels_set": MagicMock(), - "metrics_APP_LOOP_COUNT_labels_inc": MagicMock(), - "metrics_APP_LOOP_DURATION_labels_observe": MagicMock(), - "metrics_APP_ERRORS_TOTAL_labels_inc": MagicMock(), - "os_exit": MagicMock(side_effect=raise_os_exit_with_code), - "time_time": MagicMock(), - "asyncio_sleep": AsyncMock(), - "signal_signal": MagicMock(), - "traceback_print_exc": MagicMock(), - "mock_exit_signal": MagicMock(spec=Event), + 'start_http_server': MagicMock(), + 'Ingestor': MagicMock(), + 'metrics_APP_UP_labels_set': MagicMock(), + 'metrics_APP_LOOP_COUNT_labels_inc': MagicMock(), + 'metrics_APP_LOOP_DURATION_labels_observe': MagicMock(), + 'metrics_APP_ERRORS_TOTAL_labels_inc': MagicMock(), + 'os_exit': MagicMock(side_effect=raise_os_exit_with_code), + 'time_time': MagicMock(), + 'asyncio_sleep': AsyncMock(), + 'signal_signal': MagicMock(), + 'traceback_print_exc': MagicMock(), + 'mock_exit_signal': MagicMock(spec=Event), } - monkeypatch.setattr(app, "start_http_server", mocks["start_http_server"]) - monkeypatch.setattr(app, "Ingestor", mocks["Ingestor"]) - monkeypatch.setattr(asyncio, "sleep", mocks["asyncio_sleep"]) + monkeypatch.setattr(app, 'start_http_server', mocks['start_http_server']) + monkeypatch.setattr(app, 'Ingestor', mocks['Ingestor']) + monkeypatch.setattr(asyncio, 'sleep', mocks['asyncio_sleep']) monkeypatch.setattr( app.metrics.APP_UP, - "labels", - MagicMock(return_value=MagicMock( - set=mocks["metrics_APP_UP_labels_set"])), + 'labels', + MagicMock(return_value=MagicMock(set=mocks['metrics_APP_UP_labels_set'])), ) monkeypatch.setattr( app.metrics.APP_LOOP_COUNT, - "labels", - MagicMock( - return_value=MagicMock( - inc=mocks["metrics_APP_LOOP_COUNT_labels_inc"]) - ), + 'labels', + MagicMock(return_value=MagicMock(inc=mocks['metrics_APP_LOOP_COUNT_labels_inc'])), ) monkeypatch.setattr( app.metrics.APP_LOOP_DURATION, - "labels", + 'labels', MagicMock( - return_value=MagicMock( - observe=mocks["metrics_APP_LOOP_DURATION_labels_observe"] - ) + return_value=MagicMock(observe=mocks['metrics_APP_LOOP_DURATION_labels_observe']) ), ) monkeypatch.setattr( app.metrics.APP_ERRORS_TOTAL, - "labels", - MagicMock( - return_value=MagicMock( - inc=mocks["metrics_APP_ERRORS_TOTAL_labels_inc"]) - ), + 'labels', + MagicMock(return_value=MagicMock(inc=mocks['metrics_APP_ERRORS_TOTAL_labels_inc'])), ) - monkeypatch.setattr(app.os, "_exit", mocks["os_exit"]) - monkeypatch.setattr(app, "time", mocks["time_time"]) - monkeypatch.setattr(app.signal, "signal", mocks["signal_signal"]) - monkeypatch.setattr(app.traceback, "print_exc", - mocks["traceback_print_exc"]) + monkeypatch.setattr(app.os, '_exit', mocks['os_exit']) + monkeypatch.setattr(app, 'time', mocks['time_time']) + monkeypatch.setattr(app.signal, 'signal', mocks['signal_signal']) + monkeypatch.setattr(app.traceback, 'print_exc', mocks['traceback_print_exc']) - monkeypatch.setattr(app, "exit_signal", mocks["mock_exit_signal"]) - monkeypatch.setattr(app, "POD_ID", "test_pod") + monkeypatch.setattr(app, 'exit_signal', mocks['mock_exit_signal']) + monkeypatch.setattr(app, 'POD_ID', 'test_pod') - mock_ingestor_instance = mocks["Ingestor"].return_value + mock_ingestor_instance = mocks['Ingestor'].return_value mock_ingestor_instance.poll_interval = 0.01 mock_ingestor_instance.logger = MagicMock() @@ -100,119 +89,112 @@ def mock_app_env(monkeypatch): @pytest.mark.asyncio async def test_main_successful_run_one_loop(mock_app_env, capsys): """Test a successful run where the loop executes once and then exits gracefully.""" - mock_ingestor_instance = mock_app_env["Ingestor"].return_value - mock_exit_signal = mock_app_env["mock_exit_signal"] + mock_ingestor_instance = mock_app_env['Ingestor'].return_value + mock_exit_signal = mock_app_env['mock_exit_signal'] mock_exit_signal.is_set.side_effect = [False, True] - mock_app_env["time_time"].side_effect = [10.0, 11.5] + mock_app_env['time_time'].side_effect = [10.0, 11.5] - with pytest.raises(OsExitCalled) as excinfo: + with pytest.raises(OsExitCalledError) as excinfo: await app.main() assert excinfo.value.code == 0 - mock_app_env["start_http_server"].assert_called_once_with(9090) - app.metrics.APP_UP.labels.assert_any_call(pod_id="test_pod") - set_calls = mock_app_env["metrics_APP_UP_labels_set"].call_args_list + mock_app_env['start_http_server'].assert_called_once_with(9090) + app.metrics.APP_UP.labels.assert_any_call(pod_id='test_pod') + set_calls = mock_app_env['metrics_APP_UP_labels_set'].call_args_list assert call(1) in set_calls assert call(0) in set_calls assert set_calls.index(call(1)) < set_calls.index(call(0)) - mock_app_env["Ingestor"].assert_called_once_with() + mock_app_env['Ingestor'].assert_called_once_with() mock_ingestor_instance.prepare_ingestor.assert_called_once() - mock_ingestor_instance.logger.info.assert_any_call( - "Ingestor prepared. Starting main loop." - ) + mock_ingestor_instance.logger.info.assert_any_call('Ingestor prepared. Starting main loop.') mock_ingestor_instance.loop.assert_called_once() - app.metrics.APP_LOOP_COUNT.labels.assert_called_with(pod_id="test_pod") - mock_app_env["metrics_APP_LOOP_COUNT_labels_inc"].assert_called_once() + app.metrics.APP_LOOP_COUNT.labels.assert_called_with(pod_id='test_pod') + mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].assert_called_once() - mock_app_env["asyncio_sleep"].assert_has_calls( - [call(mock_ingestor_instance.poll_interval), call(5)]) - - app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id="test_pod") - mock_app_env["metrics_APP_LOOP_DURATION_labels_observe"].assert_called_once_with( - 1.5 + mock_app_env['asyncio_sleep'].assert_has_calls( + [call(mock_ingestor_instance.poll_interval), call(5)] ) + app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id='test_pod') + mock_app_env['metrics_APP_LOOP_DURATION_labels_observe'].assert_called_once_with(1.5) + mock_ingestor_instance.shutdown.assert_called_once() - mock_ingestor_instance.logger.info.assert_any_call( - "Main loop exit_signaled.") + mock_ingestor_instance.logger.info.assert_any_call('Main loop exit_signaled.') captured = capsys.readouterr() - assert "Prometheus server started on port 9090." in captured.out + assert 'Prometheus server started on port 9090.' in captured.out @pytest.mark.asyncio async def test_main_prometheus_server_fails_to_start(mock_app_env, capsys): """Test the scenario where starting the Prometheus server fails.""" - mock_app_env["start_http_server"].side_effect = OSError( - "Port already in use") + mock_app_env['start_http_server'].side_effect = OSError('Port already in use') - with pytest.raises(OsExitCalled) as excinfo: + with pytest.raises(OsExitCalledError) as excinfo: await app.main() assert excinfo.value.code == 1 # Check that .set(1) was not called. .set(0) definitely not called. called_with_1 = False - for call_args in mock_app_env["metrics_APP_UP_labels_set"].call_args_list: + for call_args in mock_app_env['metrics_APP_UP_labels_set'].call_args_list: if call_args == call(1): called_with_1 = True break - assert ( - not called_with_1 - ), "APP_UP.set(1) should not have been called if server start failed" + assert not called_with_1, 'APP_UP.set(1) should not have been called if server start failed' - mock_app_env["Ingestor"].assert_not_called() + mock_app_env['Ingestor'].assert_not_called() captured = capsys.readouterr() - assert "Failed to start Prometheus server: Port already in use" in captured.out + assert 'Failed to start Prometheus server: Port already in use' in captured.out @pytest.mark.asyncio async def test_main_loop_exception_handling(mock_app_env, capsys): """Test that an exception in ingestor.loop() is handled gracefully.""" - mock_ingestor_instance = mock_app_env["Ingestor"].return_value - mock_exit_signal = mock_app_env["mock_exit_signal"] + mock_ingestor_instance = mock_app_env['Ingestor'].return_value + mock_exit_signal = mock_app_env['mock_exit_signal'] mock_exit_signal.is_set.side_effect = [False, True] - mock_ingestor_instance.loop.side_effect = Exception("Test loop exception") - mock_app_env["time_time"].side_effect = [10.0, 10.1] + mock_ingestor_instance.loop.side_effect = Exception('Test loop exception') + mock_app_env['time_time'].side_effect = [10.0, 10.1] - with pytest.raises(OsExitCalled) as excinfo: + with pytest.raises(OsExitCalledError) as excinfo: await app.main() assert excinfo.value.code == 0 mock_ingestor_instance.loop.assert_called_once() - mock_app_env["traceback_print_exc"].assert_called_once() + mock_app_env['traceback_print_exc'].assert_called_once() - app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id="test_pod") - mock_app_env["metrics_APP_ERRORS_TOTAL_labels_inc"].assert_called_once() + app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id='test_pod') + mock_app_env['metrics_APP_ERRORS_TOTAL_labels_inc'].assert_called_once() mock_exit_signal.set.assert_called_once() - mock_app_env["metrics_APP_LOOP_COUNT_labels_inc"].assert_not_called() + mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].assert_not_called() - app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id="test_pod") - mock_app_env["metrics_APP_LOOP_DURATION_labels_observe"].assert_called_once_with( + app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id='test_pod') + mock_app_env['metrics_APP_LOOP_DURATION_labels_observe'].assert_called_once_with( pytest.approx(0.1) ) mock_ingestor_instance.shutdown.assert_called_once() captured = capsys.readouterr() - assert "Exception in main loop. Setting exit_signal flag." in captured.out + assert 'Exception in main loop. Setting exit_signal flag.' in captured.out @pytest.mark.asyncio async def test_main_keyboard_interrupt_handling(mock_app_env, capsys): """Test that KeyboardInterrupt in ingestor.loop() is handled.""" - mock_ingestor_instance = mock_app_env["Ingestor"].return_value - mock_exit_signal = mock_app_env["mock_exit_signal"] + mock_ingestor_instance = mock_app_env['Ingestor'].return_value + mock_exit_signal = mock_app_env['mock_exit_signal'] mock_exit_signal.is_set.side_effect = [False, True] mock_ingestor_instance.loop.side_effect = KeyboardInterrupt() - mock_app_env["time_time"].side_effect = [10.0, 10.1] + mock_app_env['time_time'].side_effect = [10.0, 10.1] - with pytest.raises(OsExitCalled) as excinfo: + with pytest.raises(OsExitCalledError) as excinfo: await app.main() assert excinfo.value.code == 0 @@ -220,13 +202,13 @@ async def test_main_keyboard_interrupt_handling(mock_app_env, capsys): mock_exit_signal.set.assert_called_once() mock_ingestor_instance.shutdown.assert_called_once() captured = capsys.readouterr() - assert "KeyboardInterrupt received. Setting exit_signal flag." in captured.out - mock_app_env["metrics_APP_ERRORS_TOTAL_labels_inc"].assert_not_called() + assert 'KeyboardInterrupt received. Setting exit_signal flag.' in captured.out + mock_app_env['metrics_APP_ERRORS_TOTAL_labels_inc'].assert_not_called() def test_signal_handler_sets_exit_signal(mock_app_env): """Test that the signal_handler function calls exit_signal.set().""" - mock_exit_signal_set = mock_app_env["mock_exit_signal"].set + mock_exit_signal_set = mock_app_env['mock_exit_signal'].set app.signal_handler(signal_module.SIGINT, None) mock_exit_signal_set.assert_called_once() @@ -235,14 +217,13 @@ def test_signal_handler_sets_exit_signal(mock_app_env): @pytest.mark.asyncio async def test_main_multiple_loop_iterations(mock_app_env): """Test the main loop runs for a few iterations.""" - mock_ingestor_instance = mock_app_env["Ingestor"].return_value - mock_exit_signal = mock_app_env["mock_exit_signal"] + mock_ingestor_instance = mock_app_env['Ingestor'].return_value + mock_exit_signal = mock_app_env['mock_exit_signal'] mock_exit_signal.is_set.side_effect = [False, False, False, True] - mock_app_env["time_time"].side_effect = [ - 10.0, 10.1, 10.2, 10.3, 10.4, 10.5] + mock_app_env['time_time'].side_effect = [10.0, 10.1, 10.2, 10.3, 10.4, 10.5] - with pytest.raises(OsExitCalled) as excinfo: + with pytest.raises(OsExitCalledError) as excinfo: await app.main() assert excinfo.value.code == 0 @@ -250,17 +231,15 @@ async def test_main_multiple_loop_iterations(mock_app_env): assert app.metrics.APP_LOOP_COUNT.labels.call_count == 3 app.metrics.APP_LOOP_COUNT.labels.assert_called_with( - pod_id="test_pod" + pod_id='test_pod' ) # Checks last call or any call - assert mock_app_env["metrics_APP_LOOP_COUNT_labels_inc"].call_count == 3 + assert mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].call_count == 3 - assert mock_app_env["asyncio_sleep"].call_count == 4 + assert mock_app_env['asyncio_sleep'].call_count == 4 assert app.metrics.APP_LOOP_DURATION.labels.call_count == 3 - app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id="test_pod") - duration_calls = mock_app_env[ - "metrics_APP_LOOP_DURATION_labels_observe" - ].call_args_list + app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id='test_pod') + duration_calls = mock_app_env['metrics_APP_LOOP_DURATION_labels_observe'].call_args_list assert duration_calls[0] == call(pytest.approx(0.1, abs=1e-9)) assert duration_calls[1] == call(pytest.approx(0.1, abs=1e-9)) assert duration_calls[2] == call(pytest.approx(0.1, abs=1e-9)) @@ -271,41 +250,41 @@ async def test_main_multiple_loop_iterations(mock_app_env): @pytest.mark.asyncio async def test_main_pod_id_used_in_metrics(mock_app_env): """Test that the POD_ID from app module is used in metric labels.""" - mock_exit_signal = mock_app_env["mock_exit_signal"] + mock_exit_signal = mock_app_env['mock_exit_signal'] mock_exit_signal.is_set.side_effect = [False, True] - mock_app_env["time_time"].side_effect = [10.0, 11.0] + mock_app_env['time_time'].side_effect = [10.0, 11.0] - with pytest.raises(OsExitCalled): + with pytest.raises(OsExitCalledError): await app.main() - app.metrics.APP_UP.labels.assert_any_call(pod_id="test_pod") - app.metrics.APP_LOOP_COUNT.labels.assert_any_call(pod_id="test_pod") - app.metrics.APP_LOOP_DURATION.labels.assert_any_call(pod_id="test_pod") + app.metrics.APP_UP.labels.assert_any_call(pod_id='test_pod') + app.metrics.APP_LOOP_COUNT.labels.assert_any_call(pod_id='test_pod') + app.metrics.APP_LOOP_DURATION.labels.assert_any_call(pod_id='test_pod') # APP_ERRORS_TOTAL would be checked similarly if it were called in this flow. # Check the .set() / .inc() calls on the mocks returned by .labels() - mock_app_env["metrics_APP_UP_labels_set"].assert_any_call(1) - mock_app_env["metrics_APP_UP_labels_set"].assert_any_call(0) - mock_app_env["metrics_APP_LOOP_COUNT_labels_inc"].assert_called_once() + mock_app_env['metrics_APP_UP_labels_set'].assert_any_call(1) + mock_app_env['metrics_APP_UP_labels_set'].assert_any_call(0) + mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].assert_called_once() @pytest.mark.asyncio async def test_run_async_main(mock_app_env, capsys): """Test the run_async_main function that sets up the event loop.""" - mock_ingestor_instance = mock_app_env["Ingestor"].return_value - mock_exit_signal = mock_app_env["mock_exit_signal"] + mock_ingestor_instance = mock_app_env['Ingestor'].return_value + mock_exit_signal = mock_app_env['mock_exit_signal'] mock_exit_signal.is_set.side_effect = [False, True] - mock_app_env["time_time"].side_effect = [10.0, 11.0] + mock_app_env['time_time'].side_effect = [10.0, 11.0] # Instead of calling run_async_main() which creates a new event loop, # we test the main() function directly since that's what run_async_main() would call - with pytest.raises(OsExitCalled) as excinfo: + with pytest.raises(OsExitCalledError) as excinfo: await app.main() assert excinfo.value.code == 0 # Verify the main function was called through the event loop - mock_app_env["start_http_server"].assert_called_once_with(9090) + mock_app_env['start_http_server'].assert_called_once_with(9090) mock_ingestor_instance.prepare_ingestor.assert_called_once() mock_ingestor_instance.loop.assert_called_once() mock_ingestor_instance.shutdown.assert_called_once() @@ -314,20 +293,19 @@ async def test_run_async_main(mock_app_env, capsys): @pytest.mark.asyncio async def test_main_prepare_ingestor_failure(mock_app_env, capsys): """Test that prepare_ingestor failure is handled correctly.""" - mock_ingestor_instance = mock_app_env["Ingestor"].return_value - mock_exit_signal = mock_app_env["mock_exit_signal"] + mock_ingestor_instance = mock_app_env['Ingestor'].return_value + mock_exit_signal = mock_app_env['mock_exit_signal'] - mock_ingestor_instance.prepare_ingestor.side_effect = Exception( - "Preparation failed") + mock_ingestor_instance.prepare_ingestor.side_effect = Exception('Preparation failed') mock_exit_signal.is_set.side_effect = [False, True] - with pytest.raises(OsExitCalled) as excinfo: + with pytest.raises(OsExitCalledError) as excinfo: await app.main() assert excinfo.value.code == 0 # Verify error metrics were incremented - app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id="test_pod") - mock_app_env["metrics_APP_ERRORS_TOTAL_labels_inc"].assert_called_once() + app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id='test_pod') + mock_app_env['metrics_APP_ERRORS_TOTAL_labels_inc'].assert_called_once() # Verify exit signal was set mock_exit_signal.set.assert_called_once() @@ -337,5 +315,5 @@ async def test_main_prepare_ingestor_failure(mock_app_env, capsys): # Verify error was logged mock_ingestor_instance.logger.error.assert_called_once_with( - "Failed to prepare ingestor: Preparation failed" + 'Failed to prepare ingestor: Preparation failed' ) diff --git a/tests/unit/test_ingestor.py b/tests/unit/test_ingestor.py index 0f72195..b636f2b 100644 --- a/tests/unit/test_ingestor.py +++ b/tests/unit/test_ingestor.py @@ -1,68 +1,70 @@ -from unittest.mock import ANY, AsyncMock, MagicMock, patch, call +from unittest.mock import ANY, AsyncMock, MagicMock, call, patch + from pytest import fixture, mark + from ingestor.ingestor import Ingestor -@patch("ingestor.ingestor.getenv") -@patch("ingestor.ingestor.NotificationHandler") +@patch('ingestor.ingestor.getenv') +@patch('ingestor.ingestor.NotificationHandler') def test___init__(notification_handler, getenv): getenv.side_effect = [ - "localhost:9092,localhost:35", # KAFKA_SERVERS - "true", # EXPORT_TO_KAFKA - "localhost1", # REDIS_HOST - '63790', # REDIS_PORT - "user", # REDIS_USERNAME - "password", # REDIS_PASSWORD - '100', # LEASE_TTL - '200', # HEARTBEAT_TTL - "localhost1", # HOSTNAME - '50', # POLL_INTERVAL - "localhost:27017", # MONGODB_URL - "sientia", # MONGODB_USERNAME - "sientia", # MONGODB_PASSWORD - "sientia" # MONGODB_DATABASE + 'localhost:9092,localhost:35', # KAFKA_SERVERS + 'true', # EXPORT_TO_KAFKA + 'localhost1', # REDIS_HOST + '63790', # REDIS_PORT + 'user', # REDIS_USERNAME + 'password', # REDIS_PASSWORD + '100', # LEASE_TTL + '200', # HEARTBEAT_TTL + 'localhost1', # HOSTNAME + '50', # POLL_INTERVAL + 'localhost:27017', # MONGODB_URL + 'sientia', # MONGODB_USERNAME + 'sientia', # MONGODB_PASSWORD + 'sientia', # MONGODB_DATABASE ] ingestor = Ingestor() - getenv.assert_any_call("KAFKA_SERVERS", "localhost:9092") - getenv.assert_any_call("REDIS_HOST", "localhost") - getenv.assert_any_call("REDIS_PORT", '6379') - getenv.assert_any_call("REDIS_USERNAME", None) - getenv.assert_any_call("REDIS_PASSWORD", None) - getenv.assert_any_call("LEASE_TTL", '10') - getenv.assert_any_call("HEARTBEAT_TTL", '20') - getenv.assert_any_call("HOSTNAME", "localhost") - getenv.assert_any_call("POLL_INTERVAL", '5') + getenv.assert_any_call('KAFKA_SERVERS', 'localhost:9092') + getenv.assert_any_call('REDIS_HOST', 'localhost') + getenv.assert_any_call('REDIS_PORT', '6379') + getenv.assert_any_call('REDIS_USERNAME', None) + getenv.assert_any_call('REDIS_PASSWORD', None) + getenv.assert_any_call('LEASE_TTL', '10') + getenv.assert_any_call('HEARTBEAT_TTL', '20') + getenv.assert_any_call('HOSTNAME', 'localhost') + getenv.assert_any_call('POLL_INTERVAL', '5') - assert ingestor.kafka_servers == ["localhost:9092", "localhost:35"] - assert ingestor.redis_host == "localhost1" + assert ingestor.kafka_servers == ['localhost:9092', 'localhost:35'] + assert ingestor.redis_host == 'localhost1' assert ingestor.redis_port == 63790 - assert ingestor.redis_username == "user" - assert ingestor.redis_password == "password" + assert ingestor.redis_username == 'user' + assert ingestor.redis_password == 'password' assert ingestor.lease_ttl == 100 assert ingestor.heartbeat_ttl == 200 - assert ingestor.pod_id == "localhost1" + assert ingestor.pod_id == 'localhost1' assert ingestor.poll_interval == 50 assert ingestor.metadata == { - "model_id": "-", - "model_name": "-", - "workflow_name": "opc_ingestor", - "schema_name": "opc_ingestor", - "pod_id": "localhost1" + 'model_id': '-', + 'model_name': '-', + 'workflow_name': 'opc_ingestor', + 'schema_name': 'opc_ingestor', + 'pod_id': 'localhost1', } notification_handler.assert_called_once_with( - connection_string="mongodb://sientia:sientia@localhost:27017", - database="sientia", + connection_string='mongodb://sientia:sientia@localhost:27017', + database='sientia', logger=ingestor.logger, - project_name="opc_ingestor" + project_name='opc_ingestor', ) @fixture -@patch("ingestor.ingestor.getenv") -@patch("ingestor.ingestor.NotificationHandler") +@patch('ingestor.ingestor.getenv') +@patch('ingestor.ingestor.NotificationHandler') def ingestor(_notification_handler, _getenv): ing = Ingestor() ing.logger = MagicMock() @@ -77,7 +79,7 @@ def ingestor_manager_started(ingestor): shutdown=AsyncMock(), update_opc_servers=AsyncMock(), subscribe_to_tags=AsyncMock(), - unsubscribe_slot=AsyncMock() + unsubscribe_slot=AsyncMock(), ) return ingestor @@ -93,23 +95,23 @@ async def test_shutdown(ingestor_manager_started): async def test_handle_acquired_tags_not_acquired(ingestor_manager_started): await ingestor_manager_started.handle_acquired_tags([]) - ingestor_manager_started.logger.warning.assert_called_once_with( - "No slots available") + ingestor_manager_started.logger.warning.assert_called_once_with('No slots available') ingestor_manager_started.ingestor_manager.update_opc_servers.assert_not_called() ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_not_called() @mark.asyncio async def test_handle_acquired_tags_success(ingestor_manager_started): - await ingestor_manager_started.handle_acquired_tags(["tag1", "tag2"]) + await ingestor_manager_started.handle_acquired_tags(['tag1', 'tag2']) ingestor_manager_started.logger.warning.assert_not_called() ingestor_manager_started.ingestor_manager.update_opc_servers.assert_called_once() ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_called_once_with( - ["tag1", "tag2"]) + ['tag1', 'tag2'] + ) -@patch("ingestor.ingestor.IngestorManager") +@patch('ingestor.ingestor.IngestorManager') @mark.asyncio async def test_prepare_ingestor(ingestor_manager_mock, ingestor): ingestor_manager = ingestor_manager_mock.return_value @@ -120,12 +122,12 @@ async def test_prepare_ingestor(ingestor_manager_mock, ingestor): await ingestor.prepare_ingestor() ingestor_manager_mock.assert_called_once_with( - kafka_servers=ingestor.kafka_servers, + kafka_servers=','.join(ingestor.kafka_servers), redis_data={ - "host": ingestor.redis_host, - "port": ingestor.redis_port, - "username": ingestor.redis_username, - "password": ingestor.redis_password + 'host': ingestor.redis_host, + 'port': ingestor.redis_port, + 'username': ingestor.redis_username, + 'password': ingestor.redis_password, }, lease_ttl=ingestor.lease_ttl, heartbeat_ttl=ingestor.heartbeat_ttl, @@ -141,7 +143,8 @@ async def test_prepare_ingestor(ingestor_manager_mock, ingestor): ingestor_manager.get_slot_leases.assert_called_once() ingestor.handle_acquired_tags.assert_called_once_with( - ingestor_manager.get_slot_leases.return_value) + ingestor_manager.get_slot_leases.return_value + ) def test_manage_slots_has_slots(ingestor_manager_started): @@ -180,8 +183,14 @@ def test_manage_slots_none_available_none_available(ingestor_manager_started): ingestor_manager_started.manage_no_slots(2) - ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with( - 1) + ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with(1) + + +@mark.asyncio +async def test_manage_leases_no_ingestor_manager(ingestor_manager_started): + ingestor_manager_started.ingestor_manager = None + + assert await ingestor_manager_started.manage_leases(2, 2, 5) is None @mark.asyncio @@ -201,8 +210,7 @@ async def test_manage_leases_available_slots_innactive_ingestors(ingestor_manage await ingestor_manager_started.manage_leases(2, 2, 5) - ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with( - 2) + ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with(2) ingestor_manager_started.ingestor_manager.drop_slot_leases.assert_not_called() @@ -211,9 +219,9 @@ async def test_manage_leases_available_slots_innactive_ingestors(ingestor_manage async def test_manage_leases_no_available_slots_extra_sltos(ingestor_manager_started): ingestor_manager_started.handle_acquired_tags = MagicMock() ingestor_manager_started.ingestor_manager.managed_tags = { - "tag1": "server1", - "tag2": "server2", - "tag3": "server3" + 'tag1': 'server1', + 'tag2': 'server2', + 'tag3': 'server3', } await ingestor_manager_started.manage_leases(0, 0, 2) @@ -221,7 +229,8 @@ async def test_manage_leases_no_available_slots_extra_sltos(ingestor_manager_sta ingestor_manager_started.ingestor_manager.get_slot_leases.assert_not_called() ingestor_manager_started.handle_acquired_tags.assert_not_called() ingestor_manager_started.ingestor_manager.drop_slot_leases.assert_called_once_with( - ["tag2", "tag3"]) + ['tag2', 'tag3'] + ) @mark.asyncio @@ -230,16 +239,15 @@ async def test_loop(ingestor_manager_started): ingestor_manager_started.manage_leases = AsyncMock() ingestor_manager_started.update_ingestor_manager = AsyncMock() ingestor_manager_started.ingestor_manager.managed_tags = { - "slot1": "server1", - "slot2": "server2", - "slot3": "server3" + 'slot1': 'server1', + 'slot2': 'server2', + 'slot3': 'server3', } ingestor_manager_started.ingestor_manager.get_active_ingestors = MagicMock( - return_value=["ingestor1", "ingestor2"]) - ingestor_manager_started.ingestor_manager.get_number_of_slots = MagicMock( - return_value=5) - ingestor_manager_started.ingestor_manager.get_number_of_leases = MagicMock( - return_value=1) + return_value=['ingestor1', 'ingestor2'] + ) + ingestor_manager_started.ingestor_manager.get_number_of_slots = MagicMock(return_value=5) + ingestor_manager_started.ingestor_manager.get_number_of_leases = MagicMock(return_value=1) await ingestor_manager_started.loop() @@ -248,10 +256,10 @@ async def test_loop(ingestor_manager_started): ingestor_manager_started.ingestor_manager.get_number_of_slots.assert_called_once() ingestor_manager_started.manage_no_slots.assert_called_once_with( - ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value) + ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value + ) # Explanation: 5 - 2 = 3, 3 - 1 = 2 - ingestor_manager_started.manage_leases.assert_called_once_with( - 4, 3, 2) + ingestor_manager_started.manage_leases.assert_called_once_with(4, 3, 2) ingestor_manager_started.ingestor_manager.update_slot_config.assert_called_once() @@ -262,9 +270,9 @@ async def test_loop_no_managed(ingestor_manager_started): ingestor_manager_started.update_ingestor_manager = AsyncMock() ingestor_manager_started.ingestor_manager.managed_tags = {} ingestor_manager_started.ingestor_manager.get_active_ingestors = MagicMock( - return_value=["ingestor1", "ingestor2"]) - ingestor_manager_started.ingestor_manager.get_number_of_slots = MagicMock( - return_value=5) + return_value=['ingestor1', 'ingestor2'] + ) + ingestor_manager_started.ingestor_manager.get_number_of_slots = MagicMock(return_value=5) await ingestor_manager_started.loop() @@ -273,43 +281,48 @@ async def test_loop_no_managed(ingestor_manager_started): ingestor_manager_started.ingestor_manager.get_number_of_slots.assert_called_once() ingestor_manager_started.manage_no_slots.assert_called_once_with( - ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value) + ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value + ) # Explanation: 5 - 2 = 3, 3 - 1 = 2 - ingestor_manager_started.manage_leases.assert_called_once_with( - ANY, 3, -1) + ingestor_manager_started.manage_leases.assert_called_once_with(ANY, 3, -1) ingestor_manager_started.ingestor_manager.update_slot_config.assert_called_once() - ingestor_manager_started.logger.info.assert_any_call( - "No slots acquired in this loop") + ingestor_manager_started.logger.info.assert_any_call('No slots acquired in this loop') + + +@mark.asyncio +async def test_loop_no_ingestor_manager(ingestor_manager_started): + ingestor_manager_started.ingestor_manager = None + + assert await ingestor_manager_started.loop() is None @mark.asyncio async def test_update_ingestor_manager(ingestor_manager_started): ingestor_manager_started.ingestor_manager.managed_tags = { - "slot_to_create": "new_config", - "slot_to_update": "new_config", - "slot_to_do_nothing": "old_config" + 'slot_to_create': 'new_config', + 'slot_to_update': 'new_config', + 'slot_to_do_nothing': 'old_config', } old_managed_tags = { - "slot_to_update": "old_config", - "slot_to_delete": "old_config", - "slot_to_do_nothing": "old_config" + 'slot_to_update': 'old_config', + 'slot_to_delete': 'old_config', + 'slot_to_do_nothing': 'old_config', } await ingestor_manager_started.update_ingestor_manager(old_managed_tags) ingestor_manager_started.ingestor_manager.update_opc_servers.assert_called_once() ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_has_calls( - [ - call( - {"slot_to_create": "new_config"}), - call( - {"slot_to_update": "new_config"}) - ] + [call({'slot_to_create': 'new_config'}), call({'slot_to_update': 'new_config'})] ) ingestor_manager_started.ingestor_manager.unsubscribe_slot.assert_has_calls( - [ - call("slot_to_update"), - call("slot_to_delete") - ] + [call('slot_to_update'), call('slot_to_delete')] ) + + +@mark.asyncio +async def test_update_ingestor_manager_no_ingestor_manager(ingestor_manager_started): + ingestor_manager_started.ingestor_manager = None + + await ingestor_manager_started.update_ingestor_manager({}) diff --git a/tests/unit/test_metrics.py b/tests/unit/test_metrics.py index d5e5be0..a60a6c7 100644 --- a/tests/unit/test_metrics.py +++ b/tests/unit/test_metrics.py @@ -1,7 +1,7 @@ # tests/unit/test_metrics.py -import pytest from prometheus_client import Counter, Gauge, Histogram + import ingestor.metrics as metrics # --- Test Functions for Each Metric (Corrected for v0.22.0 _name behavior) --- @@ -11,17 +11,16 @@ def test_ingestor_tag_written_count(): """Verify the definition of INGESTOR_TAG_WRITTEN_COUNT.""" assert metrics.TAG_WRITTEN_COUNT is not None assert isinstance(metrics.TAG_WRITTEN_COUNT, Counter) - assert metrics.TAG_WRITTEN_COUNT._name == "ingestor_tag_written_count" - assert set(metrics.TAG_WRITTEN_COUNT._labelnames) == { - "pod_id", "tag_name", "collection_name"} + assert metrics.TAG_WRITTEN_COUNT._name == 'ingestor_tag_written_count' + assert set(metrics.TAG_WRITTEN_COUNT._labelnames) == {'pod_id', 'tag_name', 'collection_name'} def test_app_loop_count(): """Verify the definition of APP_LOOP_COUNT.""" assert metrics.APP_LOOP_COUNT is not None assert isinstance(metrics.APP_LOOP_COUNT, Counter) - assert metrics.APP_LOOP_COUNT._name == "app_main_loop" # REMOVED _total - assert set(metrics.APP_LOOP_COUNT._labelnames) == {"pod_id"} + assert metrics.APP_LOOP_COUNT._name == 'app_main_loop' # REMOVED _total + assert set(metrics.APP_LOOP_COUNT._labelnames) == {'pod_id'} def test_app_loop_duration(): @@ -29,32 +28,32 @@ def test_app_loop_duration(): assert metrics.APP_LOOP_DURATION is not None assert isinstance(metrics.APP_LOOP_DURATION, Histogram) assert ( - metrics.APP_LOOP_DURATION._name == "app_main_loop_duration_seconds" + metrics.APP_LOOP_DURATION._name == 'app_main_loop_duration_seconds' ) # Histograms don't have _total - assert set(metrics.APP_LOOP_DURATION._labelnames) == {"pod_id"} + assert set(metrics.APP_LOOP_DURATION._labelnames) == {'pod_id'} def test_app_errors_total(): """Verify the definition of APP_ERRORS_TOTAL.""" assert metrics.APP_ERRORS_TOTAL is not None assert isinstance(metrics.APP_ERRORS_TOTAL, Counter) - assert metrics.APP_ERRORS_TOTAL._name == "app_errors" # REMOVED _total - assert set(metrics.APP_ERRORS_TOTAL._labelnames) == {"pod_id"} + assert metrics.APP_ERRORS_TOTAL._name == 'app_errors' # REMOVED _total + assert set(metrics.APP_ERRORS_TOTAL._labelnames) == {'pod_id'} def test_app_up(): """Verify the definition of APP_UP.""" assert metrics.APP_UP is not None assert isinstance(metrics.APP_UP, Gauge) - assert metrics.APP_UP._name == "app_up" # Gauges don't have _total - assert set(metrics.APP_UP._labelnames) == {"pod_id"} + assert metrics.APP_UP._name == 'app_up' # Gauges don't have _total + assert set(metrics.APP_UP._labelnames) == {'pod_id'} def test_active_ingestors(): """Verify the definition of ACTIVE_INGESTORS.""" assert metrics.ACTIVE_INGESTORS is not None assert isinstance(metrics.ACTIVE_INGESTORS, Gauge) - assert metrics.ACTIVE_INGESTORS._name == "ingestor_active_total" + assert metrics.ACTIVE_INGESTORS._name == 'ingestor_active_total' assert set(metrics.ACTIVE_INGESTORS._labelnames) == set() @@ -62,7 +61,7 @@ def test_slots_total(): """Verify the definition of SLOTS_TOTAL.""" assert metrics.SLOTS_TOTAL is not None assert isinstance(metrics.SLOTS_TOTAL, Gauge) - assert metrics.SLOTS_TOTAL._name == "ingestor_slots_total" + assert metrics.SLOTS_TOTAL._name == 'ingestor_slots_total' assert set(metrics.SLOTS_TOTAL._labelnames) == set() @@ -70,7 +69,7 @@ def test_leases_total(): """Verify the definition of LEASES_TOTAL.""" assert metrics.LEASES_TOTAL is not None assert isinstance(metrics.LEASES_TOTAL, Gauge) - assert metrics.LEASES_TOTAL._name == "ingestor_leases_total" + assert metrics.LEASES_TOTAL._name == 'ingestor_leases_total' assert set(metrics.LEASES_TOTAL._labelnames) == set() @@ -78,32 +77,32 @@ def test_slots_managed(): """Verify the definition of SLOTS_MANAGED.""" assert metrics.SLOTS_MANAGED is not None assert isinstance(metrics.SLOTS_MANAGED, Gauge) - assert metrics.SLOTS_MANAGED._name == "ingestor_slots_managed_current" - assert set(metrics.SLOTS_MANAGED._labelnames) == {"pod_id"} + assert metrics.SLOTS_MANAGED._name == 'ingestor_slots_managed_current' + assert set(metrics.SLOTS_MANAGED._labelnames) == {'pod_id'} def test_slots_acquired(): """Verify the definition of SLOTS_ACQUIRED.""" assert metrics.SLOTS_ACQUIRED is not None assert isinstance(metrics.SLOTS_ACQUIRED, Counter) - assert metrics.SLOTS_ACQUIRED._name == "ingestor_slots_acquired" # REMOVED _total - assert set(metrics.SLOTS_ACQUIRED._labelnames) == {"pod_id"} + assert metrics.SLOTS_ACQUIRED._name == 'ingestor_slots_acquired' # REMOVED _total + assert set(metrics.SLOTS_ACQUIRED._labelnames) == {'pod_id'} def test_slots_released(): """Verify the definition of SLOTS_RELEASED.""" assert metrics.SLOTS_RELEASED is not None assert isinstance(metrics.SLOTS_RELEASED, Counter) - assert metrics.SLOTS_RELEASED._name == "ingestor_slots_released" # REMOVED _total - assert set(metrics.SLOTS_RELEASED._labelnames) == {"pod_id"} + assert metrics.SLOTS_RELEASED._name == 'ingestor_slots_released' # REMOVED _total + assert set(metrics.SLOTS_RELEASED._labelnames) == {'pod_id'} def test_opc_managers_active(): """Verify the definition of OPC_MANAGERS_ACTIVE.""" assert metrics.OPC_MANAGERS_ACTIVE is not None assert isinstance(metrics.OPC_MANAGERS_ACTIVE, Gauge) - assert metrics.OPC_MANAGERS_ACTIVE._name == "ingestor_opc_managers_active" - assert set(metrics.OPC_MANAGERS_ACTIVE._labelnames) == {"pod_id"} + assert metrics.OPC_MANAGERS_ACTIVE._name == 'ingestor_opc_managers_active' + assert set(metrics.OPC_MANAGERS_ACTIVE._labelnames) == {'pod_id'} def test_opc_subscription_errors(): @@ -111,12 +110,12 @@ def test_opc_subscription_errors(): assert metrics.OPC_SUBSCRIPTION_ERRORS is not None assert isinstance(metrics.OPC_SUBSCRIPTION_ERRORS, Counter) assert ( - metrics.OPC_SUBSCRIPTION_ERRORS._name == "ingestor_opc_subscription_errors" + metrics.OPC_SUBSCRIPTION_ERRORS._name == 'ingestor_opc_subscription_errors' ) # REMOVED _total assert set(metrics.OPC_SUBSCRIPTION_ERRORS._labelnames) == { - "pod_id", - "server", - "slot", + 'pod_id', + 'server', + 'slot', } @@ -124,33 +123,27 @@ def test_opc_connections_total(): """Verify the definition of OPC_CONNECTIONS_TOTAL.""" assert metrics.OPC_CONNECTIONS_TOTAL is not None assert isinstance(metrics.OPC_CONNECTIONS_TOTAL, Counter) - assert ( - metrics.OPC_CONNECTIONS_TOTAL._name == "opc_connections_initiated" - ) # REMOVED _total - assert set(metrics.OPC_CONNECTIONS_TOTAL._labelnames) == { - "pod_id", "server_name"} + assert metrics.OPC_CONNECTIONS_TOTAL._name == 'opc_connections_initiated' # REMOVED _total + assert set(metrics.OPC_CONNECTIONS_TOTAL._labelnames) == {'pod_id', 'server_name'} def test_opc_connections_failed(): """Verify the definition of OPC_CONNECTIONS_FAILED.""" assert metrics.OPC_CONNECTIONS_FAILED is not None assert isinstance(metrics.OPC_CONNECTIONS_FAILED, Counter) - assert ( - metrics.OPC_CONNECTIONS_FAILED._name == "opc_connections_failed" - ) # REMOVED _total - assert set(metrics.OPC_CONNECTIONS_FAILED._labelnames) == { - "pod_id", "server_name"} + assert metrics.OPC_CONNECTIONS_FAILED._name == 'opc_connections_failed' # REMOVED _total + assert set(metrics.OPC_CONNECTIONS_FAILED._labelnames) == {'pod_id', 'server_name'} def test_opc_connection_status(): """Verify the definition of OPC_CONNECTION_STATUS.""" assert metrics.OPC_CONNECTION_STATUS is not None assert isinstance(metrics.OPC_CONNECTION_STATUS, Gauge) - assert metrics.OPC_CONNECTION_STATUS._name == "opc_connection_status" + assert metrics.OPC_CONNECTION_STATUS._name == 'opc_connection_status' assert set(metrics.OPC_CONNECTION_STATUS._labelnames) == { - "pod_id", - "server_name", - "server_url", + 'pod_id', + 'server_name', + 'server_url', } @@ -158,13 +151,11 @@ def test_opc_subscriptions_created(): """Verify the definition of OPC_SUBSCRIPTIONS_CREATED.""" assert metrics.OPC_SUBSCRIPTIONS_CREATED is not None assert isinstance(metrics.OPC_SUBSCRIPTIONS_CREATED, Counter) - assert ( - metrics.OPC_SUBSCRIPTIONS_CREATED._name == "opc_subscriptions_created" - ) # REMOVED _total + assert metrics.OPC_SUBSCRIPTIONS_CREATED._name == 'opc_subscriptions_created' # REMOVED _total assert set(metrics.OPC_SUBSCRIPTIONS_CREATED._labelnames) == { - "pod_id", - "server_name", - "slot_name", + 'pod_id', + 'server_name', + 'slot_name', } @@ -172,101 +163,85 @@ def test_opc_tags_subscribed(): """Verify the definition of OPC_TAGS_SUBSCRIBED.""" assert metrics.OPC_TAGS_SUBSCRIBED is not None assert isinstance(metrics.OPC_TAGS_SUBSCRIBED, Gauge) - assert metrics.OPC_TAGS_SUBSCRIBED._name == "opc_tags_subscribed_current" - assert set(metrics.OPC_TAGS_SUBSCRIBED._labelnames) == { - "pod_id", "server_name"} + assert metrics.OPC_TAGS_SUBSCRIBED._name == 'opc_tags_subscribed_current' + assert set(metrics.OPC_TAGS_SUBSCRIBED._labelnames) == {'pod_id', 'server_name'} def test_opc_cycles_without_data(): """Verify the definition of OPC_CYCLES_WITHOUT_DATA.""" assert metrics.OPC_CYCLES_WITHOUT_DATA is not None assert isinstance(metrics.OPC_CYCLES_WITHOUT_DATA, Gauge) - assert metrics.OPC_CYCLES_WITHOUT_DATA._name == "opc_cycles_without_data" - assert set(metrics.OPC_CYCLES_WITHOUT_DATA._labelnames) == { - "pod_id", "server_name"} + assert metrics.OPC_CYCLES_WITHOUT_DATA._name == 'opc_cycles_without_data' + assert set(metrics.OPC_CYCLES_WITHOUT_DATA._labelnames) == {'pod_id', 'server_name'} def test_opc_reconnections_total(): """Verify the definition of OPC_RECONNECTIONS_TOTAL.""" assert metrics.OPC_RECONNECTIONS_TOTAL is not None assert isinstance(metrics.OPC_RECONNECTIONS_TOTAL, Counter) - assert ( - metrics.OPC_RECONNECTIONS_TOTAL._name == "opc_reconnections_tried" - ) # REMOVED _total - assert set(metrics.OPC_RECONNECTIONS_TOTAL._labelnames) == { - "pod_id", "server_name"} + assert metrics.OPC_RECONNECTIONS_TOTAL._name == 'opc_reconnections_tried' # REMOVED _total + assert set(metrics.OPC_RECONNECTIONS_TOTAL._labelnames) == {'pod_id', 'server_name'} def test_kafka_messages_sent(): """Verify the definition of KAFKA_MESSAGES_SENT.""" assert metrics.KAFKA_MESSAGES_SENT is not None assert isinstance(metrics.KAFKA_MESSAGES_SENT, Counter) - assert metrics.KAFKA_MESSAGES_SENT._name == "kafka_messages_sent" # REMOVED _total - assert set(metrics.KAFKA_MESSAGES_SENT._labelnames) == {"pod_id", "topic"} + assert metrics.KAFKA_MESSAGES_SENT._name == 'kafka_messages_sent' # REMOVED _total + assert set(metrics.KAFKA_MESSAGES_SENT._labelnames) == {'pod_id', 'topic'} def test_kafka_messages_errors(): """Verify the definition of KAFKA_MESSAGES_ERRORS.""" assert metrics.KAFKA_MESSAGES_ERRORS is not None assert isinstance(metrics.KAFKA_MESSAGES_ERRORS, Counter) - assert ( - metrics.KAFKA_MESSAGES_ERRORS._name == "kafka_messages_errors" - ) # REMOVED _total - assert set(metrics.KAFKA_MESSAGES_ERRORS._labelnames) == { - "pod_id", "topic"} + assert metrics.KAFKA_MESSAGES_ERRORS._name == 'kafka_messages_errors' # REMOVED _total + assert set(metrics.KAFKA_MESSAGES_ERRORS._labelnames) == {'pod_id', 'topic'} def test_kafka_connection_status(): """Verify the definition of KAFKA_CONNECTION_STATUS.""" assert metrics.KAFKA_CONNECTION_STATUS is not None assert isinstance(metrics.KAFKA_CONNECTION_STATUS, Gauge) - assert metrics.KAFKA_CONNECTION_STATUS._name == "kafka_connection_status" - assert set(metrics.KAFKA_CONNECTION_STATUS._labelnames) == {"pod_id"} + assert metrics.KAFKA_CONNECTION_STATUS._name == 'kafka_connection_status' + assert set(metrics.KAFKA_CONNECTION_STATUS._labelnames) == {'pod_id'} def test_redis_operations_total(): """Verify the definition of REDIS_OPERATIONS_TOTAL.""" assert metrics.REDIS_OPERATIONS_TOTAL is not None assert isinstance(metrics.REDIS_OPERATIONS_TOTAL, Counter) - assert metrics.REDIS_OPERATIONS_TOTAL._name == "redis_operations" # REMOVED _total - assert set(metrics.REDIS_OPERATIONS_TOTAL._labelnames) == { - "pod_id", "operation"} + assert metrics.REDIS_OPERATIONS_TOTAL._name == 'redis_operations' # REMOVED _total + assert set(metrics.REDIS_OPERATIONS_TOTAL._labelnames) == {'pod_id', 'operation'} def test_redis_operations_errors(): """Verify the definition of REDIS_OPERATIONS_ERRORS.""" assert metrics.REDIS_OPERATIONS_ERRORS is not None assert isinstance(metrics.REDIS_OPERATIONS_ERRORS, Counter) - assert ( - metrics.REDIS_OPERATIONS_ERRORS._name == "redis_operations_errors" - ) # REMOVED _total - assert set(metrics.REDIS_OPERATIONS_ERRORS._labelnames) == { - "pod_id", "operation"} + assert metrics.REDIS_OPERATIONS_ERRORS._name == 'redis_operations_errors' # REMOVED _total + assert set(metrics.REDIS_OPERATIONS_ERRORS._labelnames) == {'pod_id', 'operation'} def test_redis_operations_duration(): """Verify the definition of REDIS_OPERATIONS_DURATION.""" assert metrics.REDIS_OPERATIONS_DURATION is not None assert isinstance(metrics.REDIS_OPERATIONS_DURATION, Histogram) - assert ( - metrics.REDIS_OPERATIONS_DURATION._name == "redis_operations_duration_seconds" - ) - assert set(metrics.REDIS_OPERATIONS_DURATION._labelnames) == { - "pod_id", "operation"} + assert metrics.REDIS_OPERATIONS_DURATION._name == 'redis_operations_duration_seconds' + assert set(metrics.REDIS_OPERATIONS_DURATION._labelnames) == {'pod_id', 'operation'} def test_redis_connection_status(): """Verify the definition of REDIS_CONNECTION_STATUS.""" assert metrics.REDIS_CONNECTION_STATUS is not None assert isinstance(metrics.REDIS_CONNECTION_STATUS, Gauge) - assert metrics.REDIS_CONNECTION_STATUS._name == "redis_connection_status" - assert set(metrics.REDIS_CONNECTION_STATUS._labelnames) == {"pod_id"} + assert metrics.REDIS_CONNECTION_STATUS._name == 'redis_connection_status' + assert set(metrics.REDIS_CONNECTION_STATUS._labelnames) == {'pod_id'} def test_notifications_sent(): """Verify the definition of NOTIFICATIONS_SENT.""" assert metrics.NOTIFICATIONS_SENT is not None assert isinstance(metrics.NOTIFICATIONS_SENT, Counter) - assert metrics.NOTIFICATIONS_SENT._name == "notifications_sent" # REMOVED _total - assert set(metrics.NOTIFICATIONS_SENT._labelnames) == { - "pod_id", "level", "block"} + assert metrics.NOTIFICATIONS_SENT._name == 'notifications_sent' # REMOVED _total + assert set(metrics.NOTIFICATIONS_SENT._labelnames) == {'pod_id', 'level', 'block'} diff --git a/validate.sh b/validate.sh new file mode 100755 index 0000000..5e837c6 --- /dev/null +++ b/validate.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# Model Manager Code Validation Script +# This script runs all code quality checks before committing or deploying + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Args +FIX_MODE=false +while [[ $# -gt 0 ]]; do + case "$1" in + --fix) + FIX_MODE=true + shift + ;; + -h|--help) + echo "Usage: $0 [--fix]" + echo " --fix Apply Ruff auto-fixes (format and lint fixes)." + exit 0 + ;; + *) + echo -e "${RED}Unknown option: $1${NC}" + echo "Usage: $0 [--fix]" + exit 2 + ;; + esac +done + +echo -e "${BLUE}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${NC}" +echo -e "${BLUE}โ•‘ Model Manager - Code Validation Suite โ•‘${NC}" +echo -e "${BLUE}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" +echo "" + +# Check if virtual environment is activated +if [[ -z "${VIRTUAL_ENV}" ]] && [[ -z "${CONDA_DEFAULT_ENV}" ]]; then + echo -e "${YELLOW}โš ๏ธ Warning: No virtual environment detected${NC}" + echo -e "${YELLOW} Consider activating your venv/conda environment${NC}" + echo "" +fi + +# Function to run a validation step +run_step() { + local step_name=$1 + local step_command=$2 + + echo -e "${BLUE}โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”${NC}" + echo -e "${BLUE}โ–ถ ${step_name}${NC}" + echo -e "${BLUE}โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”${NC}" + + if eval "$step_command"; then + echo -e "${GREEN}โœ… ${step_name} - PASSED${NC}" + echo "" + return 0 + else + echo -e "${RED}โŒ ${step_name} - FAILED${NC}" + echo "" + return 1 + fi +} + +# Track failures +FAILED_STEPS=() + +# Step 1: Code Formatting Check (Ruff) +# - default: check only +# - --fix: write changes +if ! run_step "1. Code Formatting (Ruff)" "if \$FIX_MODE; then ruff format ingestor/ tests/; else ruff format --check ingestor/ tests/; fi"; then + FAILED_STEPS+=("Code Formatting") +fi + +# Step 2: Linting (Ruff) +# - default: check only +# - --fix: apply autofixes +if ! run_step "2. Code Linting (Ruff)" "if \$FIX_MODE; then ruff check --fix ingestor/ tests/; else ruff check ingestor/ tests/; fi"; then + FAILED_STEPS+=("Linting") +fi + +# Step 3: Type Checking (mypy) +if ! run_step "3. Type Checking (mypy)" "mypy ingestor/"; then + FAILED_STEPS+=("Type Checking") +fi + +# Step 4: Security Analysis (Bandit) +if ! run_step "4. Security Analysis (Bandit)" "bandit -r ingestor/ -ll -q"; then + FAILED_STEPS+=("Security Analysis") +fi + +# Step 5: Unit Tests (pytest) +if ! run_step "5. Unit Tests (pytest)" "pytest tests/ --cov=ingestor --cov-report=term-missing --cov-report=xml --cov-report=html --cov-fail-under=80 -q"; then + FAILED_STEPS+=("Unit Tests") +fi + +# Summary +echo -e "${BLUE}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${NC}" +echo -e "${BLUE}โ•‘ Validation Summary โ•‘${NC}" +echo -e "${BLUE}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" +echo "" + +if [ ${#FAILED_STEPS[@]} -eq 0 ]; then + echo -e "${GREEN}โœ… All validation checks passed!${NC}" + echo -e "${GREEN} Your code is ready for commit/deployment.${NC}" + echo "" + exit 0 +else + echo -e "${RED}โŒ Validation failed for the following steps:${NC}" + for step in "${FAILED_STEPS[@]}"; do + echo -e "${RED} โ€ข ${step}${NC}" + done + echo "" + echo -e "${YELLOW}๐Ÿ’ก Tips:${NC}" + echo -e "${YELLOW} โ€ข Run 'ruff format ingestor/ tests/' to auto-fix formatting${NC}" + echo -e "${YELLOW} โ€ข Run 'ruff check --fix ingestor/ tests/' to auto-fix linting issues${NC}" + echo -e "${YELLOW} โ€ข Review mypy errors and add type hints where needed${NC}" + echo -e "${YELLOW} โ€ข Check bandit warnings for security issues${NC}" + echo -e "${YELLOW} โ€ข Fix failing tests or improve test coverage${NC}" + echo "" + exit 1 +fi \ No newline at end of file