Code import - branch main

.coveragerc

@@ -0,0 +1,3 @@
+[run]
+omit =
+    ingestor/app.py

.env.example

@@ -0,0 +1,21 @@
+# Kafka
+KAFKA_SERVERS="localhost:9092"
+EXPORT_TO_KAFKA="false"
+
+# Redis
+REDIS_HOST="localhost"
+REDIS_PORT="6379"
+REDIS_USERNAME="redis_user"
+REDIS_PASSWORD="redis_password"
+
+# Pod
+HOSTNAME="localhost"
+POLL_INTERVAL="5"
+LOG_LEVEL="DEBUG"
+
+# Mongo DB
+MONGODB_URL="localhost:27018"
+MONGODB_USERNAME="mongo_user"
+MONGODB_PASSWORD="mongo_password"
+MONGODB_DATABASE="sientia"
+

.github/workflows/quality-gate.yml

@@ -0,0 +1,16 @@
+name: Quality gate
+
+on:
+  pull_request:
+    branches:
+      - main
+    types: [ opened, synchronize, reopened ]
+
+jobs:
+  quality-gate:
+    uses: Aignosi/github_workflow_templates/.github/workflows/dataops-module-quality-gate.yml@main
+    permissions: write-all
+    with:
+      project_name: 'ingestor'
+      repositories: 'sientia-dataops-library'
+    secrets: inherit

.github/workflows/release.yml

@@ -0,0 +1,16 @@
+name: Create Release on Merge to Main
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - main
+
+jobs:
+  release:
+    if: github.event.pull_request.merged == true
+    uses: Aignosi/github_workflow_templates/.github/workflows/dataops-module-release.yml@main
+    permissions: write-all
+    with:
+      project_name: 'ingestor'
+    secrets: inherit

.gitignore

@@ -0,0 +1,184 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+pytest.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or pytest.xmlpackage, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# VSCode
+.vscode/
+
+git_log
+
+.git/
+.ruff_cache/
+.mypy_cache/

README.md

@@ -0,0 +1,396 @@
+# Sientia DataOps OPC Ingestor
+
+A high-performance, scalable OPC UA data ingestion system designed for industrial data collection and real-time streaming. The OPC Ingestor provides enterprise-grade data acquisition from OPC UA servers with automatic load balancing, fault tolerance, and comprehensive monitoring.
+
+##  Features
+
+### Core Functionality
+- **OPC UA Integration**: Native support for OPC UA servers with secure and unsecured connections
+- **Automatic Load Balancing**: Slot-based architecture for horizontal scaling across multiple instances
+- **Real-time Data Streaming**: Kafka integration for historical data streaming
+- **Persistent Storage**: MongoDB integration for historical data persistence
+- **Health Monitoring**: Comprehensive Prometheus metrics and health checks
+- **Fault Tolerance**: Automatic failover, reconnection, and error recovery
+
+### Advanced Capabilities
+- **Certificate-based Security**: Support for X.509 certificates and private keys
+- **Dynamic Tag Management**: Runtime configuration updates without service interruption
+- **Resource Coordination**: Redis-based slot leasing and instance coordination
+- **Notification System**: Integrated alerting and notification management
+- **Performance Optimization**: Configurable polling intervals and data collection frequencies
+
+##  Architecture
+
+The OPC Ingestor uses a modular, manager-based architecture designed for scalability and fault tolerance:
+
+```
+┌────────────────────────────────────────────
+│   Main App      │    │  IngestorManager │    │   OPC Manager   │
+│                 │◄──►│                  │◄──►│                 │
+│ - Signal Hand.  │    │ - Slot Mgmt      │    │ - Connections   │
+│ - Metrics       │    │ - Load Balancing │    │ - Subscriptions │
+│ - Lifecycle     │    │ - Coordination   │    │ - Data Handler  │
+└─────────────────┘    └──────────────────┘    └─────────────────┘
+                                │
+                                ▼
+                       ┌──────────────────┐    ┌─────────────────┐
+                       │  ResourceManager │    │  DataManager    │
+                       │                  │    │                 │
+                       │ - Redis Coord.   │    │ - Kafka Export  │
+                       │ - Slot Leasing   │    │ - MongoDB Store │
+                       │ - Heartbeats     │    │ - Data Pipeline │
+                       └──────────────────┘    └─────────────────┘
+```
+
+### Key Components
+
+- **Ingestor**: Main application orchestrator managing the overall lifecycle
+- **IngestorManager**: Coordinates slot allocation, OPC server management, and load balancing
+- **OPC Manager**: Handles individual OPC UA server connections and tag subscriptions
+- **Data Manager**: Manages data streaming (MongoDB | Kafka)
+- **Resource Manager**: Coordinates resource allocation and instance coordination via Redis
+
+## 📋 Prerequisites
+
+- Python 3.11+
+- Redis server
+- MongoDB server
+- Kafka cluster (optional, for data streaming)
+- OPC UA servers for data collection
+
+**Note**: External dependencies (Redis, MongoDB, Kafka) must be available either through:
+- Port forwarding from a Kubernetes cluster
+- External Docker Compose setup
+- Cloud-managed services
+- Local installations
+
+## ️ Installation
+
+### Local Development Setup
+
+1. **Clone the repository**
+   ```bash
+   git clone <repository-url>
+   cd sientia-dataops-opc-ingestor
+   ```
+
+2. **Create virtual environment**
+   ```bash
+   python3.11 -m venv venv
+   source ./venv/bin/activate
+   ```
+
+3. **Install dependencies**
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+4. **Create environment configuration file**
+   ```bash
+   cp .env.example .env
+   # Edit .env with your connection details
+   ```
+
+5. **Configure external dependencies**
+   
+   You'll need to set up port forwarding or connections to external services. For example:
+   
+   ```bash
+   # Port forwarding from Kubernetes cluster
+   kubectl port-forward svc/redis-master 6379:6379
+   kubectl port-forward svc/mongodb 27017:27017
+   kubectl port-forward svc/kafka 9092:9092
+   
+   # Or connect to external Docker Compose
+   # Ensure services are accessible on localhost with appropriate ports
+   ```
+
+##  Usage
+
+### Running the Ingestor
+
+Use the provided script to run the application locally:
+
+```bash
+# Make script executable (first time only)
+chmod +x run_local.sh
+
+# Run the application
+./run_local.sh
+```
+
+The script will:
+- Activate the virtual environment
+- Load environment variables from `.env`
+- Start the ingestor application
+
+### Running Tests and Coverage
+
+Use the provided script to run tests with coverage:
+
+```bash
+# Make script executable (first time only)
+chmod +x run_coverage.sh
+
+# Run tests with coverage
+./run_coverage.sh
+```
+
+The script will:
+- Activate the virtual environment
+- Run pytest with coverage reporting
+- Generate HTML coverage report
+- Open the coverage report in your browser
+
+### Manual Test Execution
+
+You can also run tests manually:
+
+```bash
+# Activate virtual environment
+source ./venv/bin/activate
+
+# Run all tests
+pytest
+
+# Run with coverage
+pytest --cov=ingestor --cov-report=html
+
+
+### Populating Redis with OPC Configuration
+
+1. **Activate virtual environment**
+   ```bash
+   source ./venv/bin/activate
+   ```
+
+2. **Run the Redis feeder**
+   ```bash
+   python simulator/redis-feeder.py
+   ```
+
+The feeder creates sample OPC tag configurations in Redis that the ingestor can discover and manage.
+
+## 🧪 Testing
+
+### Unit Tests
+```bash
+# Install pytest
+pip install pytest
+
+# Run tests
+pytest
+
+# Run with coverage
+pip install pytest-cov
+pytest --cov=ingestor
+
+# Generate HTML coverage report
+pytest --cov=ingestor --cov-report=html
+```
+
+### Functional Tests
+```bash
+# Run functional tests
+pytest tests/functional/
+```
+
+## 📊 Monitoring and Metrics
+
+The OPC Ingestor exposes comprehensive Prometheus metrics for monitoring:
+
+### Application Metrics
+- `app_up`: Application health status
+- `app_main_loop_total`: Main loop execution count
+- `app_main_loop_duration_seconds`: Loop execution time
+- `app_errors_total`: Error count
+
+### OPC Server Metrics
+- `opc_connection_status`: Server connection status
+- `opc_tags_subscribed_current`: Number of subscribed tags
+- `opc_cycles_without_data`: Data reception health
+- `opc_subscriptions_created_total`: Subscription count
+
+### Resource Management Metrics
+- `ingestor_slots_managed_current`: Slots managed by this instance
+- `ingestor_active_total`: Total active ingestor instances
+- `redis_connection_status`: Redis connection health
+- `kafka_connection_status`: Kafka connection health
+
+### Data Processing Metrics
+- `ingestor_tag_written_count`: Data write operations
+- `kafka_messages_sent_total`: Kafka message count
+- `redis_operations_total`: Redis operation count
+
+## ⚙️ Configuration
+
+### Environment Variables
+
+| Variable | Description | Default | Required |
+|----------|-------------|---------|----------|
+| `KAFKA_SERVERS` | Comma-separated Kafka server addresses | `localhost:9092` | No |
+| `EXPORT_TO_KAFKA` | Enable Kafka data export | `false` | No |
+| `REDIS_HOST` | Redis server hostname | `localhost` | Yes |
+| `REDIS_PORT` | Redis server port | `6379` | Yes |
+| `REDIS_USERNAME` | Redis username | `None` | No |
+| `REDIS_PASSWORD` | Redis password | `None` | No |
+| `LEASE_TTL` | Slot lease time-to-live (seconds) | `10` | No |
+| `HEARTBEAT_TTL` | Heartbeat time-to-live (seconds) | `20` | No |
+| `HOSTNAME` | Pod identifier | `localhost` | No |
+| `POLL_INTERVAL` | Main loop polling interval (seconds) | `5` | No |
+| `MONGODB_URL` | MongoDB server address | `localhost:27017` | Yes |
+| `MONGODB_DATABASE` | MongoDB database name | `sientia` | No |
+| `MONGODB_USERNAME` | MongoDB username | `sientia` | No |
+| `MONGODB_PASSWORD` | MongoDB password | `sientia` | No |
+| `HTTP_METRICS_PORT` | Prometheus metrics port | `9090` | No |
+
+### OPC Server Configuration
+
+OPC servers are configured through Redis with the following structure:
+
+```json
+{
+  "slot:opc_tags:1": {
+    "server1": {
+      "name": "server1",
+      "server_id": "1",
+      "url": "opc.tcp://localhost:4841",
+      "server_uri": "http://opcua-server.simulator",
+      "tags": {
+        "ns=2;i=2": {
+            "aggr_func": "avg",
+            "data_range": [
+               -100,
+               100
+            ],
+            "frequency": "15000",
+            "server_id": "1",
+            "tag_address": "ns=2;i=2",
+            "tag_name": "Counter",
+            "topics": [
+               "raw_scouter-opcua-orchestrated-pipeline",
+               "raw_scouter-basic-sum-model"
+            ]
+      }
+    }
+  }
+}
+```
+
+## 🔧 Development
+
+### Project Structure
+```
+sientia-dataops-opc-ingestor/
+├── ingestor/                 # Main application code
+│   ├── managers/            # Component managers
+│   │   ├── data_manager.py # Data persistence and export
+│   │   ├── ingestor_manager.py # Main coordination
+│   │   ├── opc_manager.py  # OPC UA server management
+│   │   └── resource_manager.py # Resource coordination
+│   ├── app.py              # Main application entry point
+│   ├── ingestor.py         # Core ingestor logic
+│   └── metrics.py          # Prometheus metrics definitions
+├── simulator/               # OPC simulation and testing tools
+├── tests/                   # Test suite
+├── docker-compose.yaml      # Infrastructure services
+└── requirements.txt         # Python dependencies
+```
+
+### Adding New Features
+
+1. **Follow the manager pattern** for new components
+2. **Add comprehensive docstrings** for all public methods
+3. **Include Prometheus metrics** for monitoring
+4. **Add unit tests** for new functionality
+5. **Update this README** with new features and configuration
+
+## 🐛 Troubleshooting
+
+### Common Issues
+
+1. **OPC Connection Failures**
+   - Verify server URLs and network connectivity
+   - Check certificate paths and security settings
+   - Review server logs for authentication issues
+
+2. **Redis Connection Issues**
+   - Verify Redis server is running and accessible
+   - Check authentication credentials
+   - Ensure proper network configuration
+
+3. **Kafka Export Failures**
+   - Verify Kafka cluster is running
+   - Check broker addresses and network connectivity
+   - Review topic configuration and permissions
+
+4. **Performance Issues**
+   - Monitor Prometheus metrics for bottlenecks
+   - Adjust polling intervals and lease TTLs
+   - Review OPC server performance and network latency
+
+### Debug Mode
+
+Enable debug logging by setting the log level in your environment:
+```bash
+export LOG_LEVEL=DEBUG
+```
+
+##  Performance Tuning
+
+### Key Parameters
+
+- **`POLL_INTERVAL`**: Main loop frequency (lower = more responsive, higher = less CPU)
+- **`LEASE_TTL`**: Slot lease duration (lower = faster failover, higher = more stable)
+- **`HEARTBEAT_TTL`**: Instance health check frequency
+- **Tag frequency**: OPC tag collection rate (Hz)
+
+### Scaling Considerations
+
+- **Horizontal Scaling**: Deploy multiple ingestor instances for high availability
+- **Load Distribution**: Use Redis-based slot allocation for automatic load balancing
+- **Resource Limits**: Monitor CPU, memory, and network usage
+- **Database Performance**: Optimize MongoDB indexes and Kafka partitioning
+
+## 🤝 Contributing
+
+1. Fork the repository
+2. Create a feature branch
+3. Make your changes with comprehensive testing
+4. Update documentation and docstrings
+5. Submit a pull request
+
+### Code Quality Standards
+
+- Follow PEP 8 style guidelines
+- Include comprehensive docstrings for all public methods
+- Maintain test coverage above 80%
+- Use type hints where appropriate
+- Follow the established architectural patterns
+
+## 📄 License
+
+This project is licensed under the terms specified in the LICENSE file.
+
+## 🆘 Support
+
+For support and questions:
+- Check the troubleshooting section above
+- Review the metrics and logs for error patterns
+- Open an issue in the project repository
+- Contact the development team
+
+---
+
+**Note**: This OPC Ingestor is designed for production use in industrial environments. Ensure proper security configuration and network isolation for production deployments.
+
+This comprehensive documentation provides:
+
+1. **Complete feature overview** with architectural details
+2. **Detailed installation and setup instructions**
+3. **Comprehensive configuration documentation**
+4. **Performance tuning and troubleshooting guides**
+5. **Development guidelines and contribution standards**
+6. **Updated docstrings** for all major classes and methods
+
+The documentation now serves as a complete reference for users, developers, and operators of the OPC Ingestor system.

encrypt.py

@@ -0,0 +1,113 @@
+import os
+import argparse
+from pathspec import PathSpec
+import yaml # type: ignore
+from typing import Any
+
+'''
+Usage:
+    python .\encrypt.py path_to_dir output_file --ignore ignore_file --chunk-size 100000
+'''
+
+
+def load_ignore_patterns(ignore_file, include_library):
+    # Ensure the .gitignore file exists
+    if not os.path.exists(ignore_file):
+        raise FileNotFoundError(f"Ignore file not found at {ignore_file}")
+
+    # Load and parse the .gitignore patterns
+    with open(ignore_file, 'r') as file:
+        patterns = file.readlines()
+        if not include_library:
+            patterns.append('**/deploy/library/')
+
+        spec = PathSpec.from_lines('gitwildmatch', patterns)
+        return spec
+
+
+def is_ignored(file_path, spec):
+    """Check if a file should be ignored based on the ignore patterns."""
+    return spec.match_file(file_path) if spec else False
+
+
+def encode_file_tree_to_yaml(directory, ignore_file, include_library):
+    """Encode the file tree into a single YAML file."""
+    ignore_patterns = load_ignore_patterns(
+        ignore_file, include_library) if ignore_file else None
+    file_tree: dict[str, Any] = {}
+
+    for root, dirs, files in os.walk(directory):
+        # Skip ignored directories
+        dirs[:] = [d for d in dirs if not is_ignored(
+            os.path.join(root, d), ignore_patterns)]
+
+        for file in files:
+            file_path = os.path.join(root, file)
+
+            # Skip ignored files
+            if is_ignored(file_path, ignore_patterns):
+                continue
+
+            # Read file content
+            try:
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    content = f.read()
+            except Exception as e:
+                print(f"Error reading file {file_path}: {e}")
+                raise
+
+            # Create nested dictionary structure
+            path_parts = os.path.relpath(file_path, directory).split(os.sep)
+            current_level = file_tree
+
+            # all except the last part (the file name)
+            for part in path_parts[:-1]:
+                current_level = current_level.setdefault(part, {})
+
+            # Add the file and its content
+            current_level[path_parts[-1]] = content
+    return yaml.dump(file_tree, default_flow_style=False)
+
+
+def chunk_and_write_file_tree_to_yaml(yaml_content, output_file, chunk_size=None):
+    """Chunk the YAML content and write it to the output file."""
+
+    chunks = [yaml_content] if chunk_size is None else [
+        yaml_content[i:i + chunk_size] for i in range(0, len(yaml_content), chunk_size)]
+
+    for i, chunk in enumerate(chunks):
+        chunk_file = f"{output_file}_{i}.yaml"
+        # Write the file tree to the output YAML file
+        with open(chunk_file, 'w', encoding='utf-8') as yaml_file:
+            yaml_file.write(chunk)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Encrypts file tree to yaml file")
+    parser.add_argument("input_directory", help="Directory to encode")
+    parser.add_argument("output_yaml_file", help="Output YAML file")
+    parser.add_argument("--ignore", default=None,
+                        help="Path to the ignore file")
+    parser.add_argument("--chunk-size", type=int, default=None,
+                        help="Chunk size for the output YAML file")
+    parser.add_argument("--library", type=bool, default=False,
+                        help="Incude the library in the output YAML file")
+
+    # Parse arguments
+    args = parser.parse_args()
+
+    # Example usage
+    directory_to_encode = args.input_directory
+    ignore_file_path = args.ignore
+    output_yaml_file = args.output_yaml_file
+    include_library = args.library
+
+    content = encode_file_tree_to_yaml(
+        directory_to_encode, ignore_file_path, include_library)
+    chunk_and_write_file_tree_to_yaml(
+        content, output_yaml_file, args.chunk_size)
+
+
+if __name__ == "__main__":
+    main()

ingestor/app.py

@@ -0,0 +1,150 @@
+import asyncio
+import os
+import signal
+import traceback
+from threading import Event
+from time import time
+
+from prometheus_client import start_http_server
+
+import ingestor.metrics as metrics
+from ingestor.ingestor import Ingestor
+
+exit_signal = Event()
+POD_ID = os.getenv('HOSTNAME', 'localhost')
+
+
+async def main():  # NOSONAR
+    """
+    Main asynchronous function that orchestrates the OPC Ingestor application.
+
+    This function performs the following operations:
+    1. Starts the Prometheus metrics server for monitoring
+    2. Initializes the Ingestor instance
+    3. Prepares the ingestor (connects to services, acquires slot leases)
+    4. Runs the main processing loop until shutdown is requested
+    5. Handles graceful shutdown and cleanup
+
+    The main loop continuously:
+    - Processes OPC data from subscribed tags
+    - Manages slot leases and resource allocation
+    - Monitors OPC server connections
+    - Records metrics for monitoring and observability
+
+    Environment Variables:
+        HOSTNAME: Pod identifier for metrics labeling (default: "localhost")
+        HTTP_METRICS_PORT: Port for Prometheus metrics server (default: 9090)
+
+    Raises:
+        Exception: If ingestor preparation fails, the application will exit
+    """
+    start_prometheus_server()
+    ingestor = Ingestor()
+    try:
+        await ingestor.prepare_ingestor()
+    except Exception as e:
+        metrics.APP_ERRORS_TOTAL.labels(pod_id=POD_ID).inc()  # Increment errors
+        ingestor.logger.error(f'Failed to prepare ingestor: {e}')
+        exit_signal.set()
+    ingestor.logger.info('Ingestor prepared. Starting main loop.')
+
+    while not exit_signal.is_set():  # NOSONAR
+        start_time = time()  # Start loop timer
+        try:
+            await ingestor.loop()
+            metrics.APP_LOOP_COUNT.labels(pod_id=POD_ID).inc()  # Increment loop counter
+
+            # Use asyncio.sleep instead of exit_signal.wait for better async compatibility
+            await asyncio.sleep(ingestor.poll_interval)  # NOSONAR
+
+        except KeyboardInterrupt:  # Handle Ctrl+C gracefully
+            print('KeyboardInterrupt received. Setting exit_signal flag.')
+            exit_signal.set()
+        except Exception:
+            print('Exception in main loop. Setting exit_signal flag.')
+            traceback.print_exc()
+            metrics.APP_ERRORS_TOTAL.labels(pod_id=POD_ID).inc()  # Increment errors
+            exit_signal.set()
+        finally:
+            # Record loop duration
+            duration = time() - start_time
+            metrics.APP_LOOP_DURATION.labels(pod_id=POD_ID).observe(duration)
+
+    await ingestor.shutdown()
+    metrics.APP_UP.labels(pod_id=POD_ID).set(0)  # Mark app as DOWN
+
+    ingestor.logger.info('Main loop exit_signaled.')
+
+    # Give Prometheus a chance to scrape one last time before exiting (optional)
+    await asyncio.sleep(5)
+
+    os._exit(0)
+
+
+def signal_handler(_signum, _frame):
+    """
+    Signal handler for graceful application shutdown.
+
+    This function handles system signals (SIGINT, SIGTERM, SIGHUP) by setting
+    the exit_signal flag, which triggers the main loop to complete its current
+    iteration and then shut down gracefully.
+
+    Args:
+        _signum: The signal number received
+        _frame: The current stack frame (unused)
+    """
+    print(f'Received signal {_signum}. Setting exit_signal flag.')
+    exit_signal.set()
+
+
+def start_prometheus_server():
+    """
+    Starts the Prometheus metrics HTTP server.
+
+    This function initializes a Prometheus metrics server on the configured port
+    to expose application metrics for monitoring and alerting. The server provides
+    metrics about application health, performance, and operational status.
+
+    Environment Variables:
+        HTTP_METRICS_PORT: Port number for the metrics server (default: 9090)
+
+    Raises:
+        Exception: If the server fails to start, the application will exit
+    """
+    try:
+        port = int(os.getenv('HTTP_METRICS_PORT', 9090))
+        start_http_server(port)
+        print(f'Prometheus server started on port {port}.')
+        metrics.APP_UP.labels(pod_id=POD_ID).set(1)  # Mark app as UP
+    except Exception as e:
+        print(f'Failed to start Prometheus server: {e}')
+        os._exit(1)
+
+
+def run_async_main():
+    """
+    Run the async main function with proper event loop setup.
+
+    This function sets up the asyncio event loop and runs the main async function.
+    It handles KeyboardInterrupt gracefully and ensures proper cleanup of the event loop.
+
+    The function is designed to work with both direct execution and containerized
+    environments, providing consistent behavior across different deployment scenarios.
+    """
+    try:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        loop.run_until_complete(main())
+    except KeyboardInterrupt:
+        print('KeyboardInterrupt received in main thread.')
+        exit_signal.set()
+    finally:
+        loop.close()
+
+
+if __name__ == '__main__':
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGHUP, signal_handler)
+
+    run_async_main()

ingestor/ingestor.py

@@ -0,0 +1,456 @@
+from copy import deepcopy
+from os import getenv
+from typing import Any
+
+from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+from sientia_do.observability.logger import get_logger
+from sientia_do.observability.metrics_controller import MetricsController
+from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+
+import ingestor.metrics as metrics
+from ingestor.managers.ingestor_manager import IngestorManager
+
+
+class Ingestor(SientiaMonitoring):
+    """
+    Main OPC Ingestor class that orchestrates data collection from OPC UA servers.
+
+    The Ingestor is responsible for:
+    - Managing slot leases for load balancing across multiple instances
+    - Connecting to and monitoring OPC UA servers
+    - Subscribing to OPC tags and collecting real-time data
+    - Distributing data to Kafka and MongoDB
+    - Providing health monitoring and metrics collection
+
+    The ingestor uses a slot-based architecture where each slot represents
+    a collection of OPC tags that can be managed by a single ingestor instance.
+    This allows for horizontal scaling and load distribution.
+
+    Environment Variables:
+        KAFKA_SERVERS: Comma-separated list of Kafka server addresses (default: "localhost:9092")
+        EXPORT_TO_KAFKA: Enable/disable Kafka export (default: "false")
+        REDIS_HOST: Redis server hostname (default: "localhost")
+        REDIS_PORT: Redis server port (default: 6379)
+        REDIS_USERNAME: Redis username (optional)
+        REDIS_PASSWORD: Redis password (optional)
+        LEASE_TTL: Time-to-live for slot leases in seconds (default: 10)
+        HEARTBEAT_TTL: Time-to-live for heartbeats in seconds (default: 20)
+        HOSTNAME: Pod identifier (default: "localhost")
+        POLL_INTERVAL: Main loop polling interval in seconds (default: 5)
+        MONGODB_URL: MongoDB server address (default: "localhost:27017")
+        MONGODB_USERNAME: MongoDB username (default: "sientia")
+        MONGODB_PASSWORD: MongoDB password (default: "sientia")
+        MONGODB_DATABASE: MongoDB database name (default: "sientia")
+
+    Attributes:
+        export_to_kafka (bool): Whether to export data to Kafka
+        redis_host (str): Redis server hostname
+        redis_port (int): Redis server port
+        redis_username (str): Redis username (optional)
+        redis_password (str): Redis password (optional)
+        lease_ttl (int): Time-to-live for slot leases in seconds
+        heartbeat_ttl (int): Time-to-live for heartbeat signals in seconds
+        pod_id (str): Identifier for the current pod or host
+        poll_interval (int): Interval in seconds for polling operations
+        mongo_database (str): MongoDB database name
+        mongo_connection_string (str): Complete MongoDB connection string
+        kafka_servers (list): List of Kafka server addresses
+        logger: Logger instance for application logging
+        notification_handler: Handler for sending notifications
+        metadata (dict): Application metadata for notifications and tracking
+        ingestor_manager: Manager instance for coordinating operations
+    """
+
+    def __init__(self):
+        """
+        Initializes the ingestor with configuration values retrieved from environment variables.
+
+        Sets up all necessary connections and configurations for:
+        - Kafka connectivity (if enabled)
+        - Redis for slot management and coordination
+        - MongoDB for data persistence and notifications
+        - OPC UA server management
+        - Metrics collection and monitoring
+        """
+
+        kafka_servers = getenv('KAFKA_SERVERS', 'localhost:9092')
+        export_to_kafka: bool = getenv('EXPORT_TO_KAFKA', 'false') == 'true'
+
+        self.export_to_kafka = export_to_kafka
+        self.redis_host = getenv('REDIS_HOST', 'localhost')
+        self.redis_port = int(getenv('REDIS_PORT', '6379'))
+        self.redis_username = getenv('REDIS_USERNAME', None)
+        self.redis_password = getenv('REDIS_PASSWORD', None)
+        self.lease_ttl = int(getenv('LEASE_TTL', '10'))
+        self.heartbeat_ttl = int(getenv('HEARTBEAT_TTL', '20'))
+        self.pod_id = getenv('HOSTNAME', 'localhost')
+        self.poll_interval = int(getenv('POLL_INTERVAL', '5'))
+        mongo_url = getenv('MONGODB_URL', 'localhost:27017')
+        mongo_username = getenv('MONGODB_USERNAME', 'sientia')
+        mongo_password = getenv('MONGODB_PASSWORD', 'sientia')
+        self.mongo_database = getenv('MONGODB_DATABASE', 'sientia')
+        self.mongo_connection_string = f'mongodb://{mongo_username}:{mongo_password}@{mongo_url}'
+
+        self.kafka_servers = kafka_servers.split(',')
+        self.logger = get_logger(__name__)
+        self.notification_handler = NotificationHandler(
+            connection_string=self.mongo_connection_string,
+            database=self.mongo_database,
+            logger=self.logger,
+            project_name='opc_ingestor',
+        )
+        self.metrics_controller = MetricsController(logger=self.logger)
+
+        SientiaMonitoring.__init__(
+            self,
+            logger=self.logger,
+            notification_handler=self.notification_handler,
+            metrics_controller=self.metrics_controller,
+        )
+
+        self.metadata = {
+            'model_id': '-',
+            'model_name': '-',
+            'workflow_name': 'opc_ingestor',
+            'schema_name': 'opc_ingestor',
+            'pod_id': self.pod_id,
+        }
+        self.ingestor_manager: IngestorManager | None = None
+
+    async def shutdown(self):
+        """
+        Gracefully shuts down the ingestor and all its components.
+
+        This method ensures proper cleanup of:
+        - OPC UA connections and subscriptions
+        - Resource managers and data connections
+        - Active slot leases and heartbeats
+
+        Should be called before application termination to prevent resource leaks.
+        """
+        if self.ingestor_manager:
+            await self.ingestor_manager.shutdown()
+
+    async def handle_acquired_tags(self, acquired):
+        """
+        Handles the acquired tags by subscribing to them if available.
+
+        This method processes the tags that have been allocated to this ingestor
+        instance through the slot leasing system. It updates OPC server configurations
+        and establishes subscriptions to the allocated tags.
+
+        Args:
+            acquired (list): A list of acquired tags to be processed. If the list
+                             is empty or None, no action is taken other than logging
+                             a warning.
+
+        Behavior:
+            - If no tags are acquired, logs a warning about no slots being available
+            - If tags are acquired, updates OPC server configurations and subscribes
+              to the allocated tags for data collection
+        """
+
+        if not acquired:
+            self.logger.warning('No slots available')
+
+        else:
+            # Subscribe to acquired slots
+            if self.ingestor_manager:
+                self.ingestor_manager.update_opc_servers()
+                await self.ingestor_manager.subscribe_to_tags(acquired)
+
+    async def prepare_ingestor(self):
+        """
+        Prepares the ingestor by initializing all components and acquiring initial slot leases.
+
+        This method performs the following steps:
+        1. Initializes the IngestorManager with all necessary configuration parameters
+        2. Declares the ingestor as active in the coordination system
+        3. Acquires slot leases for tag management
+        4. Processes the acquired tags and establishes OPC subscriptions
+
+        The preparation phase is critical for establishing the ingestor's role in
+        the distributed system and ensuring it can begin processing OPC data.
+
+        Raises:
+            Exception: If any error occurs during the initialization or lease acquisition process.
+                      This will cause the application to exit as the ingestor cannot function
+                      without proper initialization.
+        """
+
+        self.ingestor_manager = IngestorManager(
+            kafka_servers=','.join(self.kafka_servers),
+            redis_data={
+                'host': self.redis_host,
+                'port': self.redis_port,
+                'username': self.redis_username,
+                'password': self.redis_password,
+            },
+            lease_ttl=self.lease_ttl,
+            heartbeat_ttl=self.heartbeat_ttl,
+            poll_interval=self.poll_interval,
+            mongo_connection_string=self.mongo_connection_string,
+            mongo_database=self.mongo_database,
+            metadata=self.metadata,
+            logger=self.logger,
+            notification_handler=self.notification_handler,
+            metrics_controller=self.metrics_controller,
+            export_to_kafka=self.export_to_kafka,
+        )
+        assert self.ingestor_manager is not None
+
+        # Declare ingestor active
+        await self.ingestor_manager.declare_active()
+
+        # Get slot lease
+        acquired = await self.ingestor_manager.get_slot_leases()
+        self.logger.info(f'Acquired slots: {acquired}')
+
+        await self.handle_acquired_tags(acquired)
+
+        await self.emit_metric(
+            metric_object=metrics.SLOTS_MANAGED,
+            method='set',
+            value=len(self.ingestor_manager.managed_tags),
+            tags={
+                'pod_id': self.pod_id,
+            },
+        )
+
+    async def manage_no_slots(self, number_of_slots: int):
+        """
+        Manages the scenario where there are no slots assigned to the ingestor.
+
+        This method handles the case where an ingestor is active but has no
+        allocated slots. It attempts to acquire a slot lease if slots are
+        available in the system.
+
+        Args:
+            number_of_slots (int): The number of available slots in the system.
+
+        Behavior:
+            - Only attempts to acquire slots if the ingestor is currently active
+              (has no managed tags) and there are slots available
+            - Requests a single slot lease to begin processing
+        """
+
+        if self.ingestor_manager and not self.ingestor_manager.managed_tags and number_of_slots > 0:
+            # This ingestor is active and has no slots, so we need to try to
+
+            # Get slot lease
+            await self.ingestor_manager.get_slot_leases(1)
+
+    async def manage_leases(self, available_slots: int, lacking_ingestors: int, slot_diff: int):
+        """
+        Manages the allocation and deallocation of slot leases for ingestors.
+
+        This method implements the load balancing logic for distributing OPC tag
+        processing across multiple ingestor instances. It ensures optimal resource
+        utilization and fair distribution of work.
+
+        Args:
+            available_slots (int): The number of slots currently available for allocation.
+            lacking_ingestors (int): The number of ingestors that are active and without slots.
+            slot_diff (int): The difference between the total slots and the required slots.
+
+        Behavior:
+            - If there are available slots and lacking ingestors, attempts to acquire
+              slot leases for the available slots and processes the acquired tags.
+            - If there are no lacking ingestors but there are extra slots (slot_diff > 0),
+              releases the extra slot leases to ensure proper allocation.
+
+        Logs:
+            - Logs the number of available slots when attempting to acquire leases.
+            - Logs the number of extra slots when releasing leases.
+        """
+        if not self.ingestor_manager:
+            return
+
+        if available_slots > 0 and lacking_ingestors > 0:
+            # Some ingestors are inactive, so there are "available_slots" slots available
+            self.logger.info(f'Slots available: {available_slots}')
+
+            # Get slot lease
+            await self.ingestor_manager.get_slot_leases(available_slots)
+
+        elif lacking_ingestors <= 0 and slot_diff > 0:
+            self.logger.info(f'Extra slots available: {slot_diff}')
+            # There's enough slots for all ingestors, but this ingestor has more than one slot
+            # So we need to drop the extra leases
+
+            overleases = list(self.ingestor_manager.managed_tags.keys())[1:]
+
+            await self.ingestor_manager.drop_slot_leases(overleases)
+
+            for lease in overleases:
+                await self.ingestor_manager.unsubscribe_slot(lease)
+                self.ingestor_manager.managed_tags.pop(lease)
+
+            await self.emit_metric(
+                metric_object=metrics.SLOTS_MANAGED,
+                method='set',
+                value=len(self.ingestor_manager.managed_tags),
+                tags={
+                    'pod_id': self.pod_id,
+                },
+            )
+
+    async def update_ingestor_manager(self, old_managed_tags: dict[str, Any]):
+        """
+        Updates the ingestor manager with new managed tags and handles configuration changes.
+
+        This method compares the current managed tags with the previous state and
+        performs necessary operations to maintain synchronization:
+        - Subscribes to newly allocated tags
+        - Resubscribes to tags with changed configurations
+        - Unsubscribes from deallocated tags
+
+        Args:
+            old_managed_tags (Dict[str, Any]): The previous state of managed tags.
+
+        Behavior:
+            - Compares current and previous tag configurations
+            - Establishes subscriptions for new tags
+            - Updates subscriptions for modified tags
+            - Removes subscriptions for deallocated tags
+            - Updates metrics to reflect current state
+        """
+
+        if not self.ingestor_manager:
+            return
+
+        self.logger.debug(f'Current managed tags: {self.ingestor_manager.managed_tags}')
+
+        await self.ingestor_manager.update_opc_servers()
+        new_managed_tags = deepcopy(self.ingestor_manager.managed_tags)
+
+        self.logger.debug(
+            f'Comparing new managed tags {new_managed_tags} with old managed tags {old_managed_tags}'
+        )
+
+        keys = set(new_managed_tags) | set(old_managed_tags)
+
+        changes = {
+            k: (new_managed_tags.get(k), old_managed_tags.get(k))
+            for k in keys
+            if new_managed_tags.get(k) != old_managed_tags.get(k)
+        }
+
+        self.logger.debug(f'Changes: {changes}')
+
+        for slot, config in new_managed_tags.items():
+            if slot not in old_managed_tags:
+                self.logger.info(f'Subscribing to new slot {slot}')
+                await self.ingestor_manager.subscribe_to_tags({slot: config})
+                continue
+
+            if config != old_managed_tags[slot]:
+                self.logger.info(f'Resubscribing to slot {slot}')
+                await self.ingestor_manager.unsubscribe_slot(slot)
+                await self.ingestor_manager.subscribe_to_tags({slot: config})
+
+        for slot in old_managed_tags.keys():
+            if slot not in new_managed_tags:
+                self.logger.info(f'Unsubscribing from slot {slot}')
+                await self.ingestor_manager.unsubscribe_slot(slot)
+
+        await self.emit_metric(
+            metric_object=metrics.SLOTS_MANAGED,
+            method='set',
+            value=len(self.ingestor_manager.managed_tags),
+            tags={
+                'pod_id': self.pod_id,
+            },
+        )
+
+    async def loop(self):
+        """
+        Executes the main processing loop for managing ingestors and slots.
+
+        This method is the core of the ingestor's operation, performing the following
+        tasks in each iteration:
+        1. Declares the ingestor as active to maintain its presence in the system
+        2. Polls for slot updates and manages resource allocation
+        3. Handles scenarios where no slots are available
+        4. Manages slot leases based on system load and available resources
+        5. Updates OPC server configurations and checks server integrity
+        6. Synchronizes managed tags with the current system state
+
+        The loop implements a sophisticated load balancing algorithm that:
+        - Distributes OPC tag processing across multiple ingestor instances
+        - Ensures optimal resource utilization
+        - Maintains system stability during scaling operations
+        - Provides real-time monitoring and metrics collection
+
+        This method is intended to be called repeatedly to ensure the ingestor
+        manager operates correctly and maintains synchronization with the slots
+        and OPC servers.
+        """
+
+        if not self.ingestor_manager:
+            return
+
+        await self.ingestor_manager.declare_active()
+
+        self.logger.info('Polling for slot updates...')
+        # Get active ingestors
+
+        current_managed_tags = deepcopy(self.ingestor_manager.managed_tags)
+
+        ingestors = await self.ingestor_manager.get_active_ingestors()
+        number_of_ingestors = len(ingestors)
+        number_of_leases = await self.ingestor_manager.get_number_of_leases()
+        number_of_slots = await self.ingestor_manager.get_number_of_slots()
+
+        # Update active ingestors gauge
+        await self.emit_metric(
+            metric_object=metrics.ACTIVE_INGESTORS,
+            method='set',
+            value=number_of_ingestors,
+            tags={
+                'pod_id': self.pod_id,
+            },
+        )
+
+        # Handle no slots
+        self.logger.info('Managing no slots...')
+        await self.manage_no_slots(number_of_slots)
+
+        available_slots = number_of_slots - number_of_leases
+        lacking_ingestors = number_of_slots - number_of_ingestors
+        slot_diff = len(self.ingestor_manager.managed_tags) - 1
+
+        self.logger.info('Managing leases...')
+        await self.manage_leases(available_slots, lacking_ingestors, slot_diff)
+
+        # Update managed slots gauge
+        await self.emit_metric(
+            metric_object=metrics.SLOTS_MANAGED,
+            method='set',
+            value=len(self.ingestor_manager.managed_tags),
+            tags={
+                'pod_id': self.pod_id,
+            },
+        )
+
+        self.logger.debug(
+            f'Active ingestors: {ingestors}, '
+            f'Number of slots: {number_of_slots}, '
+            f'Number of leases: {number_of_leases}, '
+            f'Managed tags: {self.ingestor_manager.managed_tags}, '
+            f'Managed servers: {self.ingestor_manager.opc_managers}'
+        )
+        if not self.ingestor_manager.managed_tags:
+            # No slots acquired
+            self.logger.info('No slots acquired in this loop')
+
+        # Update opc servers
+        self.logger.info('Updating slot config...')
+        await self.ingestor_manager.update_slot_config()
+
+        # Check OPC cycles
+        self.logger.info('Checking OPC servers integrity...')
+        await self.ingestor_manager.check_opc_servers_integrity()
+
+        self.logger.info('Updating managed tags...')
+        await self.update_ingestor_manager(current_managed_tags)

ingestor/managers/data_manager.py

@@ -0,0 +1,288 @@
+import json
+import os
+import traceback
+from time import sleep
+
+from kafka import KafkaProducer
+from kafka.errors import NoBrokersAvailable
+from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+from sientia_do.notifications.models import NotificationLevel
+from sientia_do.observability.logger import Logger
+from sientia_do.observability.metrics_controller import MetricsController
+from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+from sientia_do.repository.mongodb_repository import MongoDBRepository
+from sientia_do.temporal.constants import now
+
+import ingestor.metrics as metrics
+
+
+class DataManager(SientiaMonitoring):
+    """
+    Manages data persistence and export operations for the OPC Ingestor.
+
+    The DataManager is responsible for:
+    - Storing OPC data in MongoDB for historical analysis and persistence
+    - Exporting data to Kafka for real-time streaming and downstream processing
+    - Managing database connections and ensuring data integrity
+    - Providing data access interfaces for other components
+
+    The manager supports both MongoDB and Kafka operations, with Kafka export
+    being optional and configurable. It implements retry logic for connection
+    failures and provides comprehensive error handling and notification.
+
+    Args:
+        kafka_servers (str): Comma-separated string of Kafka server addresses
+        mongo_connection_string (str): MongoDB connection string
+        mongo_database (str): MongoDB database name
+        export_to_kafka (bool): Whether to enable Kafka export functionality
+        metadata (dict): Application metadata for notifications and tracking
+        logger (Logger): Logger instance for application logging
+        notification_handler (NotificationHandler): Handler for sending notifications
+
+    Attributes:
+        pod_id (str): Pod identifier for metrics labeling
+        kafka_producer (KafkaProducer): Kafka producer instance for data export
+        export_to_kafka (bool): Whether Kafka export is enabled
+        connection_string (str): MongoDB connection string
+        database (str): MongoDB database name
+        mongo_client (MongoClient): MongoDB client instance
+        metadata (dict): Application metadata
+    """
+
+    def __init__(
+        self,
+        kafka_servers: str,
+        mongo_connection_string: str,
+        mongo_database: str,
+        export_to_kafka: bool,
+        metadata: dict,
+        logger: Logger,
+        notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
+    ) -> None:
+        """
+        Initializes the DataManager instance with Kafka and MongoDB connections.
+
+        This constructor attempts to establish connections to the specified services:
+        1. Kafka: Initializes producer with retry logic (up to 3 attempts)
+        2. MongoDB: Establishes connection and verifies server availability
+
+        The initialization process includes:
+        - Kafka producer setup with JSON serialization
+        - MongoDB client initialization and connection testing
+        - Metrics recording for connection status
+        - Error handling with notifications
+
+        Args:
+            kafka_servers (str): Comma-separated string of Kafka server addresses
+            mongo_connection_string (str): MongoDB connection string
+            mongo_database (str): MongoDB database name
+            export_to_kafka (bool): Whether to enable Kafka export
+            metadata (dict): Application metadata
+            logger (Logger): Logger instance
+            notification_handler (NotificationHandler): Notification handler
+
+        Raises:
+            NoBrokersAvailable: If the connection to Kafka servers fails after 3 attempts.
+
+        Metrics:
+            - KAFKA_CONNECTION_STATUS: Set to 1 on successful connection, 0 on failure
+        """
+
+        self.pod_id = os.getenv('HOSTNAME', 'localhost')
+        self.kafka_producer = None
+        self.export_to_kafka = export_to_kafka
+
+        SientiaMonitoring.__init__(
+            self,
+            logger=logger,
+            metrics_controller=metrics_controller,
+            notification_handler=notification_handler,
+        )
+
+        if self.export_to_kafka:
+            for i in range(0, 3):
+                logger.info(
+                    f'Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}'
+                )
+                try:
+                    self.kafka_producer = KafkaProducer(
+                        bootstrap_servers=kafka_servers,
+                        value_serializer=lambda v: json.dumps(v).encode(
+                            'utf-8'
+                        ),  # Serialize JSON messages
+                        key_serializer=lambda k: str(k).encode('utf-8') if k else None,
+                    )
+                    # Kafka connected
+                    metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1)
+                    break
+                except NoBrokersAvailable:
+                    logger.error(f'Kafka servers {kafka_servers} are not available. Retrying...')
+                    sleep(5)
+            else:
+                # Kafka not connected
+                metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
+                logger.error(
+                    f'Failed to connect to Kafka servers {kafka_servers} after 3 attempts.'
+                )
+                raise NoBrokersAvailable(
+                    f'Failed to connect to Kafka servers {kafka_servers} after 3 attempts.'
+                )
+
+            logger.info(f'DataManager initialized with Kafka servers: {kafka_servers}')
+
+        logger.info(
+            f'Trying to initializing DataManager with MongoDB servers: {mongo_connection_string}'
+        )
+
+        self.connection_string = mongo_connection_string
+        self.database = mongo_database
+
+        self.mongo_repository = MongoDBRepository(
+            connection_string=self.connection_string,
+            database_name=self.database,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+        self.metadata = metadata
+
+        logger.info(f'DataManager initialized with MongoDB servers: {self.connection_string}')
+
+    def shutdown(self):
+        """
+        Gracefully shuts down the DataManager and closes all connections.
+
+        This method ensures proper cleanup of:
+        - Kafka producer connection with message flushing
+        - MongoDB client connection
+        - Metrics recording for connection status
+
+        The method handles connection closure gracefully, logging any errors
+        that occur during shutdown while ensuring all resources are properly released.
+        """
+        if self.kafka_producer:
+            try:
+                self.kafka_producer.flush(timeout=10)
+                self.kafka_producer.close()
+                # Mark as disconnected
+                metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
+            except Exception as e:
+                self.logger.error(f'Error closing Kafka producer: {e}')
+        else:
+            self.logger.warning('Kafka producer is already closed or not initialized.')
+
+        try:
+            self.mongo_repository.close()
+        except Exception as e:
+            self.logger.error(f'Error closing MongoDB client: {e}')
+
+    def __del__(self):
+        self.shutdown()
+
+    def delivery_report(self, msg):
+        """
+        Callback for successful Kafka message delivery reports.
+
+        This method is called by the Kafka producer when a message is successfully
+        delivered to a topic. It logs the delivery details including topic, partition,
+        and offset information for debugging and monitoring purposes.
+
+        Args:
+            msg: Kafka message object containing delivery details
+        """
+        self.logger.debug(
+            f'Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}'
+        )
+
+    def delivery_error(self, err):
+        """
+        Callback for Kafka message delivery error reports.
+
+        This method is called by the Kafka producer when a message delivery fails.
+        It logs the error details for debugging and monitoring purposes.
+
+        Args:
+            err: Error information from the failed delivery attempt
+        """
+        self.logger.error(f'Delivery failed for record : {err}')
+
+    async def publish(self, topic: str, data: dict) -> None:
+        """
+        Publishes a message to a specified Kafka topic.
+
+        Args:
+            topic (str): The name of the Kafka topic to which the message will be published.
+            data (dict): The message data to be sent to the Kafka topic.
+
+        Returns:
+            None
+
+        Raises:
+            Exception: If there is an error during message delivery, it will be handled by the `delivery_error` callback.
+        """
+
+        if self.export_to_kafka and self.kafka_producer:
+            try:
+                self.logger.debug(f'Publishing message to topic {topic}: {data}')
+                self.kafka_producer.send(topic=topic, value=data).add_callback(
+                    self.delivery_report
+                ).add_errback(self.delivery_error)
+
+                self.kafka_producer.flush(timeout=10)
+                await self.emit_metric(
+                    metric_object=metrics.KAFKA_MESSAGES_SENT,
+                    tags={
+                        'pod_id': self.pod_id,
+                        'topic': topic,
+                    },
+                )
+
+            except Exception as e:
+                await self.emit_metric(
+                    metric_object=metrics.KAFKA_MESSAGES_ERRORS,
+                    tags={
+                        'pod_id': self.pod_id,
+                        'topic': topic,
+                    },
+                )
+                trace = traceback.format_exc()
+                await self.send_notification_async(
+                    metadata=self.metadata,
+                    notification_id=f'KAFKA_PRODUCER_ERROR_{topic}',
+                    message=f'Error publishing message to topic {topic}: {e}',
+                    block='kafka_producer',
+                    level=NotificationLevel.ERROR,
+                    attachment_content=trace,
+                )
+                self.logger.error(trace)
+
+        try:
+            await self.mongo_repository.insert(
+                collection_name=topic,
+                document={**data, 'inserted_at': now()},
+                metadata=self.metadata,
+            )
+            self.logger.debug(f'Message inserted into MongoDB collection {topic}: {data}')
+
+            await self.emit_metric(
+                metric_object=metrics.TAG_WRITTEN_COUNT,
+                tags={
+                    'pod_id': self.pod_id,
+                    'tag_name': data['name'],
+                    'collection_name': topic,
+                },
+            )
+
+        except Exception as e:
+            trace = traceback.format_exc()
+            await self.send_notification_async(
+                metadata=self.metadata,
+                notification_id=f'MONGO_PRODUCER_ERROR_{topic}',
+                message=f'Error inserting message to MongoDB: {e}',
+                block='mongo_producer',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+            self.logger.error(trace)

ingestor/managers/ingestor_manager.py

@@ -0,0 +1,689 @@
+import asyncio
+import traceback
+from copy import deepcopy
+
+from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+from sientia_do.notifications.models import NotificationLevel
+from sientia_do.observability.logger import Logger
+from sientia_do.observability.metrics_controller import MetricsController
+from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+
+import ingestor.metrics as metrics
+from ingestor.managers.data_manager import DataManager
+from ingestor.managers.opc_manager import OpcManager
+from ingestor.managers.resource_manager import ResourceManager
+
+
+class IngestorManager(SientiaMonitoring):
+    """
+    Central coordinator for managing OPC data ingestion operations.
+
+    The IngestorManager orchestrates the interaction between different components:
+    - DataManager: Handles data persistence and Kafka export
+    - OPC Managers: Manage individual OPC UA server connections
+    - ResourceManager: Coordinates slot leasing and load balancing
+
+    This class implements a slot-based architecture where:
+    - Each slot represents a collection of OPC tags from one or more servers
+    - Slots are distributed across multiple ingestor instances for load balancing
+    - Dynamic slot allocation ensures optimal resource utilization
+
+    Key Responsibilities:
+    - Slot lease management and distribution
+    - OPC server connection lifecycle management
+    - Tag subscription coordination
+    - System health monitoring and integrity checks
+    - Load balancing across multiple ingestor instances
+
+    Args:
+        kafka_servers (str): Comma-separated list of Kafka server addresses
+        redis_data (dict): Redis connection parameters (host, port, username, password)
+        lease_ttl (int): Time-to-live for slot leases in seconds
+        heartbeat_ttl (int): Time-to-live for heartbeat signals in seconds
+        poll_interval (int): Main loop polling interval in seconds
+        mongo_connection_string (str): MongoDB connection string
+        mongo_database (str): MongoDB database name
+        metadata (dict): Application metadata for notifications and tracking
+        logger (Logger): Logger instance for application logging
+        notification_handler (NotificationHandler): Handler for sending notifications
+        export_to_kafka (bool): Whether to export data to Kafka
+
+    Attributes:
+        data_manager (DataManager): Manages data persistence and Kafka export
+        opc_managers (dict): Dictionary of OPC managers keyed by server name
+        resource_manager (ResourceManager): Manages Redis-based resource coordination
+        number_of_slots (int): Total number of slots configured in the system
+        poll_interval (int): Main loop polling interval
+        managed_tags (dict): Currently managed tags organized by slot
+        opc_servers (dict): OPC server configurations
+        metadata (dict): Application metadata
+    """
+
+    def __init__(
+        self,
+        kafka_servers: str,
+        redis_data: dict,
+        lease_ttl: int,
+        heartbeat_ttl: int,
+        poll_interval: int,
+        mongo_connection_string: str,
+        mongo_database: str,
+        metadata: dict,
+        logger: Logger,
+        notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
+        export_to_kafka: bool = False,
+    ):
+        redis_host: str = redis_data['host']
+        redis_port: int = int(redis_data['port'])
+        redis_username: str | None = redis_data.get('username', None)
+        redis_password: str | None = redis_data.get('password', None)
+
+        SientiaMonitoring.__init__(
+            self,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+        self.data_manager = DataManager(
+            kafka_servers=kafka_servers,
+            mongo_connection_string=mongo_connection_string,
+            mongo_database=mongo_database,
+            export_to_kafka=export_to_kafka,
+            metadata=metadata,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+        self.opc_managers: dict = {}
+        self.resource_manager = ResourceManager(
+            host=redis_host,
+            port=redis_port,
+            lease_ttl=lease_ttl,
+            heartbeat_ttl=heartbeat_ttl,
+            metadata=metadata,
+            logger=logger,
+            notification_handler=notification_handler,
+            username=redis_username,
+            password=redis_password,
+            metrics_controller=metrics_controller,
+        )
+        self.number_of_slots = 0
+        self.poll_interval = poll_interval
+        self.managed_tags: dict = {}
+        self.opc_servers: dict = {}
+
+        self.metadata = metadata
+
+    async def initialize_opc_from_config(self, server_config: dict) -> OpcManager | None:
+        """
+        Initializes an OPC Manager instance using the provided server configuration.
+
+        This method creates and configures an OPC Manager for a specific OPC UA server,
+        establishing the connection and preparing it for tag subscriptions.
+
+        Args:
+            server_config (dict): A dictionary containing the OPC server configuration.
+                Expected keys include:
+                    - 'name' (str): The name of the OPC server.
+                    - 'url' (str): The URL of the OPC server.
+                    - 'server_uri' (str): The URI of the OPC server.
+                    - 'cert_path' (str, optional): Path to the client certificate file.
+                    - 'private_key_path' (str, optional): Path to the private key file.
+                    - 'server_cert_path' (str, optional): Path to the server certificate file.
+
+        Returns:
+            OpcManager | None: An initialized OpcManager instance if successful,
+            otherwise None if an error occurs during initialization.
+
+        Raises:
+            Exception: If OPC manager initialization fails, the error is logged and
+                      a notification is sent, but the method returns None to allow
+                      the system to continue operating with other servers.
+        """
+
+        try:
+            self.logger.info(f'Initializing OpcManager at {server_config["url"]}')
+            manager = OpcManager(
+                name=server_config['name'],
+                url=server_config['url'],
+                subscription_period_ms=server_config['subscription_period_ms'],
+                data_manager=self.data_manager,
+                logger=self.logger,
+                server_uri=server_config['server_uri'],
+                notification_handler=self.notification_handler,
+                metadata=self.metadata,
+                cert_path=server_config.get('cert_path'),
+                private_key_path=server_config.get('private_key_path'),
+                server_cert_path=server_config.get('server_cert_path'),
+                metrics_controller=self.metrics_controller,
+            )
+
+            manager.config = server_config
+            await manager.connect()
+        except Exception as e:
+            trace = traceback.format_exc()
+            self.send_notification(
+                metadata=self.metadata,
+                notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}',
+                message=f'Error initializing OPC manager: {e}',
+                block='opc_manager',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+            self.logger.error(trace)
+
+            return None
+
+        return manager
+
+    async def shutdown(self):
+        """
+        Gracefully shuts down the IngestorManager and all its components.
+
+        This method ensures proper cleanup of:
+        - All OPC manager instances and their connections
+        - Data manager connections and resources
+        - Active subscriptions and server connections
+
+        The shutdown process is performed asynchronously to allow proper cleanup
+        of all managed resources before termination.
+        """
+        for _server_name, server in self.opc_managers.items():
+            await server.shutdown()
+        self.data_manager.shutdown()
+
+    def __del__(self):
+        asyncio.run(self.shutdown())
+
+    async def remove_server(self, server: str):
+        """
+        Removes an OPC server from the ingestor.
+        """
+        if server in self.opc_managers:
+            await self.opc_managers[server].shutdown()
+            del self.opc_managers[server]
+        for slot, _config in self.managed_tags.items():
+            self.managed_tags[slot].pop(server, None)
+
+    async def update_opc_servers(self):
+        """
+        Updates the OPC (OLE for Process Control) server connections managed by the ingestor.
+        This method ensures that the OPC servers defined in `self.managed_tags` are properly
+        initialized and updated. It performs the following tasks:
+        - Registers new OPC servers based on the configuration in `self.managed_tags`.
+        - Updates existing OPC server instances if their configuration has changed.
+        - Disconnects and removes OPC servers that are no longer present in `self.managed_tags`.
+        Steps:
+        1. Iterates through the `self.managed_tags` dictionary to identify and register servers.
+        2. Initializes new OPC server instances if they are not already managed.
+        3. Reinitializes OPC server instances if their configuration has changed.
+        4. Disconnects and removes OPC servers that are no longer registered.
+        Attributes:
+            self.managed_tags (dict): A nested dictionary containing slot and server configurations.
+            self.opc_managers (dict): A dictionary mapping server names to their
+                OPC manager instances.
+            self.data_manager: An object responsible for managing data operations.
+            self.logger: A logging object for recording warnings and other messages.
+        Raises:
+            Any exceptions raised during OPC server initialization or disconnection.
+        Logs:
+            - Warnings for servers that are no longer found in `self.managed_tags`.
+        """
+
+        registered_servers = []
+        current_managed_tags = deepcopy(self.managed_tags)
+        for _slot, slot_config in current_managed_tags.items():
+            for server, server_config in slot_config.items():
+                registered_servers.append(server)
+                server_config = deepcopy(server_config)
+                server_config.pop('tags', None)
+                server_instance = self.opc_managers.get(server, None)
+                if server_instance is None:
+                    self.logger.info(f'Initializing OPC manager for server {server}')
+                    server_instance = await self.initialize_opc_from_config(server_config)
+
+                elif server_instance.config != server_config:
+                    self.logger.warning(f'Reinitializing OPC manager for server {server}')
+                    await server_instance.shutdown()
+                    del self.opc_managers[server]
+                    server_instance = await self.initialize_opc_from_config(server_config)
+                else:
+                    self.logger.debug(
+                        f'OPC manager for server {server} is already initialized and up to date'
+                    )
+
+                if server_instance is not None:
+                    self.opc_managers[server] = server_instance
+                else:
+                    self.logger.warning(
+                        f'Failed to initialize OPC manager for server {server}, '
+                        f'removing server from managed tags.'
+                    )
+
+                    await self.remove_server(server)
+
+        servers = list(self.opc_managers.keys())
+        for server in servers:
+            if server not in registered_servers:
+                self.logger.warning(
+                    f'Server {server} not found in managed tags. Desconnecting from server.'
+                )
+                await self.remove_server(server)
+
+        await self.emit_metric(
+            metric_object=metrics.OPC_MANAGERS_ACTIVE,
+            method='set',
+            value=len(self.opc_managers),
+            tags={
+                'pod_id': self.pod_id,
+            },
+        )
+
+    async def check_opc_servers_integrity(self):
+        """
+        Checks the integrity of the OPC servers and updates the OPC servers if necessary.
+
+        This method performs health checks on all managed OPC servers by:
+        - Checking cycle counts for data reception
+        - Monitoring connection health and data flow
+        - Triggering reconnection for lost servers
+        - Updating metrics for active OPC managers
+
+        Side Effects:
+            - Updates cycle monitoring for all nodes
+            - Removes lost servers from managed tags
+            - Updates OPC manager metrics
+        """
+        to_disconnect: list[str] = []
+        for server, opc_manager in self.opc_managers.items():
+            await opc_manager.check_cycles()
+
+            is_lost = await opc_manager.check_opc_listenning()
+            if is_lost:
+                self.logger.warning(f'OPC server {server} is lost. Server will be disconnected.')
+
+                to_disconnect.append(server)
+
+        for server in to_disconnect:
+            await self.remove_server(server)
+
+        await self.emit_metric(
+            metric_object=metrics.OPC_MANAGERS_ACTIVE,
+            method='set',
+            value=len(self.opc_managers),
+            tags={
+                'pod_id': self.pod_id,
+            },
+        )
+
+    async def declare_active(self):
+        """
+        Declares the ingestor as active by sending a heartbeat signal to the resource manager.
+
+        This method ensures that the ingestor is marked as active by invoking the
+        `ingestor_heartbeat` method of the associated resource manager. The heartbeat
+        mechanism enables load balancers and monitoring systems to track active instances.
+
+        Side Effects:
+            - Updates Redis with current instance heartbeat
+            - Enables load balancing and health monitoring
+        """
+
+        await self.resource_manager.ingestor_heartbeat()
+
+    async def get_active_ingestors(self) -> list[str]:
+        """
+        Retrieve a list of active ingestors.
+
+        This method fetches all ingestors from the resource manager and returns them.
+        If no ingestors are found, an empty list is returned.
+
+        Returns:
+            List[str]: A list of active ingestor names, or an empty list if none are found.
+
+        The method queries Redis for all active ingestor heartbeats and extracts
+        the pod identifiers for load balancing and coordination purposes.
+        """
+
+        ingestors = await self.resource_manager.get_all_ingestors()
+        return ingestors if ingestors else []
+
+    async def get_number_of_leases(self) -> int:
+        """
+        Retrieves the number of leases managed by the resource manager.
+
+        This method fetches all available leases from the resource manager,
+        calculates their count, and updates the `number_of_slots` attribute.
+
+        Returns:
+            int: The total number of leases. Returns 0 if no leases are available.
+
+        Side Effects:
+            - Updates internal slot count tracking
+            - Updates Prometheus metrics for total leases
+        """
+
+        leases = await self.resource_manager.get_all_leases()
+        self.number_of_slots = len(leases) if leases else 0
+        await self.emit_metric(
+            metric_object=metrics.LEASES_TOTAL,
+            method='set',
+            value=self.number_of_slots,
+            tags={
+                'pod_id': self.pod_id,
+            },
+        )
+        return self.number_of_slots
+
+    async def get_number_of_slots(self) -> int:
+        """
+        Retrieves the number of slots managed by the resource manager.
+
+        This method fetches all available slots from the resource manager,
+        calculates their count, and updates the `number_of_slots` attribute.
+
+        Returns:
+            int: The total number of slots. Returns 0 if no slots are available.
+
+        Side Effects:
+            - Updates internal slot count tracking
+            - Updates Prometheus metrics for total slots
+        """
+
+        slots = await self.resource_manager.get_all_slots()
+        self.number_of_slots = len(slots) if slots else 0
+        await self.emit_metric(
+            metric_object=metrics.SLOTS_TOTAL,
+            method='set',
+            value=self.number_of_slots,
+            tags={
+                'pod_id': self.pod_id,
+            },
+        )
+        return self.number_of_slots
+
+    async def get_slot_leases(self, max_slots: int = 1) -> dict:
+        """
+        Acquires a specified number of resource slots by leasing them from the resource manager.
+
+        This method implements the slot acquisition logic for load balancing:
+        - Iterates through available slots and attempts to lease them
+        - Logs the leasing of each slot
+        - Updates the `managed_tags` attribute with the acquired slots
+        - Stops leasing once the specified `max_slots` are acquired
+
+        Args:
+            max_slots (int): The maximum number of slots to lease. Defaults to 1.
+
+        Returns:
+            Dict: A dictionary where the keys are the slot identifiers (as strings)
+                  and the values are the leased slot details.
+
+        Behavior:
+            - Attempts to lease slots sequentially starting from slot 1
+            - Skips slots that cannot be retrieved after leasing
+            - Logs warnings if unable to acquire the requested number of slots
+            - Updates metrics for acquired slots and managed slots count
+
+        Notes:
+            - If a slot is leased but its details cannot be retrieved
+              (i.e., `get_tag_slot` returns None), that slot is skipped.
+        """
+
+        acquired = {}
+        for i in range(1, self.number_of_slots + 1):
+            if await self.resource_manager.lease_tag(str(i)):
+                self.logger.info(f'Leased slot {i}')
+                slots = await self.resource_manager.get_tag_slot(str(i))
+                if slots is None:
+                    continue
+                acquired[str(i)] = slots
+                await self.emit_metric(
+                    metric_object=metrics.SLOTS_ACQUIRED,
+                    method='inc',
+                    value=1,
+                    tags={
+                        'pod_id': self.pod_id,
+                    },
+                )
+
+                if len(acquired) >= max_slots:
+                    self.managed_tags.update(acquired)
+                    await self.emit_metric(
+                        metric_object=metrics.SLOTS_MANAGED,
+                        method='set',
+                        value=len(self.managed_tags),
+                        tags={
+                            'pod_id': self.pod_id,
+                        },
+                    )
+                    return acquired
+
+        self.logger.warning(
+            f'Unable to acquire {max_slots} slots. Only {acquired} slots were leased.'
+        )
+        self.managed_tags.update(acquired)
+        await self.emit_metric(
+            metric_object=metrics.SLOTS_MANAGED,
+            method='set',
+            value=len(self.managed_tags),
+            tags={
+                'pod_id': self.pod_id,
+            },
+        )
+        return acquired
+
+    async def unsubscribe_slot(self, slot: str):
+        """
+        Unsubscribes a specific slot from all associated OPC servers.
+
+        This method removes all subscriptions for a given slot across all
+        OPC servers that were managing it. It ensures clean cleanup of
+        resources when slots are released or reconfigured.
+
+        Args:
+            slot (str): The name of the slot to unsubscribe.
+
+        Raises:
+            KeyError: If the specified slot does not exist in the managed tags.
+
+        Side Effects:
+            - Removes subscriptions from all OPC servers for the specified slot
+            - Cleans up subscription resources on the OPC servers
+        """
+
+        for server in self.managed_tags[slot].keys():
+            if server in self.opc_managers:
+                await self.opc_managers[server].unsubscribe(slot)
+
+    async def update_slot_config(self):
+        """
+        Updates the configuration of managed slots by renewing their leases,
+        fetching the latest configurations, and handling any changes or removals.
+
+        This method performs the following steps:
+        1. Renews the lease for each managed slot using the resource manager.
+        2. Fetches the latest configuration for each slot.
+        3. Logs and removes slots whose configurations are no longer available.
+        4. Updates the configuration of slots if changes are detected.
+        5. Unsubscribes and re-subscribes to slots with updated configurations.
+        6. Removes slots from the managed tags if they are no longer valid.
+        7. Updates the OPC servers after processing all slots.
+
+        Side Effects:
+            - Modifies the `managed_tags` dictionary to reflect the latest slot configurations.
+            - Updates OPC server subscriptions based on the current state of managed slots.
+
+        Raises:
+            None explicitly, but relies on the behavior of `resource_manager` and
+            other dependencies for error handling.
+
+        Logging:
+            - Logs warnings for removed slots.
+            - Logs informational messages for updated slot configurations.
+        """
+
+        removed_slots: list[str] = []
+        for slot, _slot_config in self.managed_tags.items():
+            await self.resource_manager.renew_tag_lease(slot)
+            update = await self.resource_manager.get_tag_slot(slot)
+            if update is None:
+                removed_slots.append(slot)
+                continue
+
+            self.managed_tags[slot] = update
+
+        for slot in removed_slots:
+            self.managed_tags.pop(slot, None)
+
+    async def drop_slot_leases(self, ids: list[str]) -> None:
+        """
+        Releases the leases associated with the specified slot IDs.
+
+        This method iterates through a list of slot IDs and calls the
+        `drop_tag_lease` method of the `resource_manager` to release
+        the lease for each ID. It's used during load balancing and
+        graceful shutdown scenarios.
+
+        Args:
+            ids (List[str]): A list of slot IDs for which the leases
+                             should be released.
+
+        Returns:
+            None
+
+        Side Effects:
+            - Releases Redis-based leases for specified slots
+            - Updates metrics for released slots count
+        """
+        for lease_id in ids:
+            await self.resource_manager.drop_tag_lease(lease_id)
+            await self.emit_metric(
+                metric_object=metrics.SLOTS_RELEASED,
+                method='inc',
+                value=1,
+                tags={
+                    'pod_id': self.pod_id,
+                },
+            )
+
+    async def manage_server(self, slot: str, server: str, server_config: dict, tags: dict) -> int:
+        """
+        Manages the subscription of tags to a specified OPC server and slot.
+
+        This method ensures that the specified server and slot have an active subscription
+        for the provided tags. If the server or slot is not properly configured, or if
+        subscription fails, appropriate error handling is performed.
+
+        Args:
+            slot (str): The slot identifier for the subscription.
+            server (str): The name of the OPC server.
+            server_config (dict): Configuration dictionary for the server, which includes
+                the tags to be subscribed under the key 'tags'.
+            tags (dict): A dictionary of tags to be subscribed.
+
+        Returns:
+            int: Status code indicating the result of the operation:
+                - 0: Subscription was successful.
+                - 1: Server not found in `opc_managers`.
+                - 2: Subscription creation or tag subscription failed.
+
+        Logs:
+            - Logs informational messages about the subscription process.
+            - Logs errors if the server is not found, subscription creation fails, or
+              tag subscription fails.
+            - Logs a warning if a subscription is removed due to failure.
+
+        Raises:
+            Exception: Any unexpected exceptions during subscription creation or tag
+            subscription are logged but not propagated.
+
+        Side Effects:
+            - Creates or updates OPC subscriptions
+            - Manages tag subscriptions on OPC servers
+            - Updates error metrics and notifications
+        """
+
+        self.logger.info(f'Subscribing to tags from {slot}:{server}')
+        tags_to_sub = server_config.get('tags')
+        if server not in self.opc_managers:
+            self.logger.error(f'Server {server} not found in opc_managers.')
+            return 1
+        if slot not in self.opc_managers[server].subscriptions:
+            try:
+                await self.opc_managers[server].create_subscription(slot)
+            except Exception as e:
+                self.logger.error(f'Failed to create subscription for slot {slot}: {e}')
+                return 2
+        try:
+            self.logger.info(tags_to_sub)
+            await self.opc_managers[server].subscribe(
+                slot, deepcopy(tags_to_sub), self.poll_interval
+            )
+            self.logger.info(tags_to_sub)
+        except Exception as e:
+            await self.emit_metric(
+                metric_object=metrics.OPC_SUBSCRIPTION_ERRORS,
+                tags={
+                    'pod_id': self.pod_id,
+                    'server': server,
+                    'slot': slot,
+                },
+            )
+            trace = traceback.format_exc()
+            await self.send_notification_async(
+                metadata=self.metadata,
+                notification_id=f'OPC_SUBSCRIPTION_ERROR_{slot}:{server}',
+                message=f'Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}',
+                block='opc_manager',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+            self.logger.error(trace)
+
+            self.logger.warning(f'Removing subscription from server {server} for slot {slot}')
+            await self.opc_managers[server].unsubscribe(slot)
+            return 2
+        return 0
+
+    async def subscribe_to_tags(self, tags: dict) -> None:
+        """
+        Subscribes to a set of tags and manages their configurations.
+
+        This method processes a dictionary of tags, iterating through each slot and server
+        configuration. It attempts to manage the server configurations and removes any
+        servers that return a specific response code.
+
+        Args:
+            tags (Dict): A dictionary containing tag configurations. The structure is
+                         expected to be {slot: {server: server_config}}.
+
+        Side Effects:
+            - Logs the provided tags for debugging purposes.
+            - Updates the `managed_tags` attribute by removing servers that meet the
+              removal criteria.
+            - Establishes OPC subscriptions for all configured tags.
+
+        Removal Criteria:
+            - If the `manage_server` method returns a response code of 2 for a given
+              slot and server, that server is removed from the `managed_tags` attribute.
+
+        The method ensures that only successfully configured servers remain in the
+        managed tags, maintaining system stability and preventing subscription errors.
+        """
+
+        to_remove = []
+        self.logger.info(tags)
+        for slot, slot_config in tags.items():
+            for server, server_config in slot_config.items():
+                response = await self.manage_server(slot, server, server_config, tags)
+                if response == 2:
+                    to_remove.append([slot, server])
+
+        for slot, server in to_remove:
+            if server in self.opc_managers:
+                await self.opc_managers[server].shutdown()
+                del self.opc_managers[server]
+            self.managed_tags[slot].pop(server, None)

ingestor/managers/opc_manager.py

@@ -0,0 +1,590 @@
+import asyncio
+import json
+import traceback
+from pathlib import Path
+
+from asyncua import Client
+from asyncua.crypto.security_policies import SecurityPolicyBasic256
+from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+from sientia_do.notifications.models import NotificationLevel
+from sientia_do.observability.logger import Logger
+from sientia_do.observability.metrics_controller import MetricsController
+from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, OPC_TIMEZONE
+
+import ingestor.metrics as metrics
+from ingestor.managers.data_manager import DataManager
+
+
+class OpcManager(SientiaMonitoring):
+    """
+    Manages OPC UA server connections and tag subscriptions.
+
+    The OpcManager is responsible for:
+    - Establishing and maintaining secure connections to OPC UA servers
+    - Managing tag subscriptions and data collection
+    - Handling server reconnection and error recovery
+    - Processing OPC data and forwarding it to the data manager
+    - Monitoring connection health and performance metrics
+
+    The manager supports both secure and unsecured connections, with optional
+    certificate-based authentication for enhanced security.
+
+    Args:
+        name (str): Unique identifier for the OPC server
+        url (str): OPC UA server endpoint URL
+        data_manager (DataManager): Manager for data persistence and export
+        logger (Logger): Logger instance for application logging
+        server_uri (str): OPC UA server application URI
+        notification_handler (NotificationHandler): Handler for sending notifications
+        metadata (dict): Application metadata for notifications and tracking
+        cert_path (str, optional): Path to client certificate file for secure connections
+        private_key_path (str, optional): Path to client private key file
+        server_cert_path (str, optional): Path to server certificate file for validation
+
+    Attributes:
+        url (str): OPC UA server endpoint URL
+        name (str): Unique identifier for the OPC server
+        server_uri (str): OPC UA server application URI
+        data_queue (dict): Queue for buffering OPC data before processing
+        non_receive_count (int): Counter for cycles without data reception
+        client (Client): OPC UA client instance
+        cert_path (str): Path to client certificate file
+        private_key_path (str): Path to client private key file
+        server_cert_path (str): Path to server certificate file
+        nodes (dict): Dictionary of OPC node references
+        subscriptions (dict): Active OPC subscriptions
+        data_manager (DataManager): Manager for data persistence and export
+        metadata (dict): Application metadata
+    """
+
+    def __init__(
+        self,
+        name: str,
+        url: str,
+        subscription_period_ms: int,
+        data_manager: DataManager,
+        logger: Logger,
+        server_uri: str,
+        notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
+        metadata: dict,
+        cert_path: str | None = None,
+        private_key_path: str | None = None,
+        server_cert_path: str | None = None,
+    ):
+        self.url = url
+        self.name = name
+        self.server_uri = server_uri
+        self.data_queue: dict = {}
+        self.non_receive_count = 0
+        self.client: Client | None = None
+        self.subscription_period_ms = subscription_period_ms
+        self.cert_path = cert_path
+        self.private_key_path = private_key_path
+        self.server_cert_path = server_cert_path
+        self.nodes: dict = {}
+        self.subscriptions: dict = {}
+        self.data_manager = data_manager
+        self.metadata = metadata
+
+        SientiaMonitoring.__init__(
+            self,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+        metrics.OPC_CONNECTION_STATUS.labels(
+            pod_id=self.pod_id, server_name=self.name, server_url=self.url
+        ).set(0)
+        metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(0)
+
+    def __str__(self):
+        """
+        String representation of the OPC Manager.
+
+        Returns:
+            str: Human-readable representation showing server details and current state.
+        """
+        return (
+            f'OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n'
+            f'nodes={self.nodes}, subscriptions={self.subscriptions}'
+        )
+
+    def __del__(self):
+        asyncio.run(self.shutdown())
+
+    async def shutdown(self):
+        """
+        Comprehensive cleanup method for graceful shutdown.
+
+        This method ensures proper cleanup of all OPC UA resources:
+        - Closes active subscriptions
+        - Disconnects from the OPC server
+        - Releases allocated resources
+
+        Should be called before the application terminates to prevent resource leaks
+        and ensure clean disconnection from OPC servers.
+        """
+        try:
+            await self.disconnect()
+        except Exception as e:
+            self.logger.error(f'Error during cleanup: {e}')
+
+    async def set_security(self):
+        """
+        Configures the security settings for the OPC UA client.
+
+        This method sets up the security policy, certificates, and timeouts
+        required for establishing a secure connection with the OPC UA server.
+        It implements Basic256 security policy with certificate-based authentication.
+
+        Raises:
+            ValueError: If either the certificate path or private key path is not provided.
+
+        Security Settings:
+            - Security Policy: Basic256
+            - Secure Channel Timeout: 10,000,000 ms
+            - Session Timeout: 10,000,000 ms
+
+        The method configures:
+            - Client application URI
+            - Certificate-based authentication
+            - Server certificate validation (if provided)
+            - Connection timeouts for stability
+        """
+
+        if not all([self.cert_path, self.private_key_path]):
+            raise ValueError(
+                'Certificate and private key paths must be provided for secure connection.'
+            )
+        cert = str(Path(self.cert_path)) if self.cert_path else None
+        private_key = str(Path(self.private_key_path)) if self.private_key_path else None
+        server_cert = str(Path(self.server_cert_path)) if self.server_cert_path else None
+
+        if self.client:
+            self.client.application_uri = self.server_uri
+            self.logger.info('Setting security...')
+            await self.client.set_security(
+                SecurityPolicyBasic256,
+                certificate=cert,
+                private_key=private_key,
+                server_certificate=server_cert,
+            )
+            self.client.secure_channel_timeout = 10000000
+            self.client.session_timeout = 10000000
+
+    async def connect(self):
+        """
+        Establishes a connection to the OPC server.
+
+        This method initializes the OPC client using the provided URL and
+        sets up security if a certificate path is specified. It then
+        attempts to connect to the server and logs the connection status.
+
+        The connection process includes:
+        1. Client initialization with server URL
+        2. Security configuration (if certificates are provided)
+        3. Connection establishment
+        4. Metrics recording for monitoring
+
+        Raises:
+            Exception: If the connection to the OPC server fails.
+
+        Metrics:
+            - OPC_CONNECTIONS_TOTAL: Incremented on connection attempt
+            - OPC_CONNECTION_STATUS: Set to 1 on successful connection
+        """
+
+        await self.emit_metric(
+            metric_object=metrics.OPC_CONNECTIONS_TOTAL,
+            tags={
+                'pod_id': self.pod_id,
+                'server_name': self.name,
+            },
+        )
+        try:
+            self.client = Client(self.url, timeout=10, watchdog_intervall=3600000)
+            assert self.client is not None  # Informa ao mypy que client não é None
+
+            self.client.name = self.pod_id
+            self.client.application_name = self.pod_id
+            pod_uri = self.pod_id.replace('-', ':')
+            self.client.application_uri = pod_uri
+            self.client.product_uri = pod_uri
+
+            if self.cert_path:
+                await self.set_security()
+            self.logger.info(f'Starting connection to {self.name}...')
+            await self.client.connect()
+            await self.emit_metric(
+                metric_object=metrics.OPC_CONNECTION_STATUS,
+                method='set',
+                value=1,
+                tags={
+                    'pod_id': self.pod_id,
+                    'server_name': self.name,
+                    'server_url': self.url,
+                },
+            )
+            self.logger.info(f'Connection to {self.name} successful.')
+        except Exception:
+            await self.disconnect()
+
+            raise
+
+    async def create_subscription(self, name: str):
+        """
+        Creates a subscription with the specified monitoring period.
+
+        This method establishes a subscription to monitor data changes or events
+        from the OPC UA server. If the client is not connected, an exception is raised.
+
+        Args:
+            name (str): The name identifier for the subscription
+            period (int, optional): The monitoring period in milliseconds. Defaults to 500 ms.
+
+        Raises:
+            ValueError: If the client is not connected.
+
+        Side Effects:
+            - Sets the `self.period` attribute to the specified or default period.
+            - Creates a subscription and assigns it to `self.subscriptions[name]`.
+            - Logs the creation of the subscription.
+            - Increments subscription creation metrics.
+        """
+
+        if not self.client:
+            raise ValueError('Client not connected. Call connect first.')
+        try:
+            self.subscriptions[name] = await self.client.create_subscription(
+                self.subscription_period_ms, self
+            )
+            self.logger.info(f'Subscription {name} created on {self.name}.')
+            await self.emit_metric(
+                metric_object=metrics.OPC_SUBSCRIPTIONS_CREATED,
+                method='inc',
+                value=1,
+                tags={
+                    'pod_id': self.pod_id,
+                    'server_name': self.name,
+                    'slot_name': name,
+                },
+            )
+        except Exception as e:
+            self.logger.error(f'Failed to create subscription {name} on {self.name}: {e}')
+            raise
+
+    async def subscribe(self, subscription: str, nodes: dict, collect_period: int):
+        """
+        Subscribes to a set of OPC UA nodes for data change notifications.
+
+        This method adds the specified nodes to the subscription and configures
+        their data collection rules based on the provided collection period and
+        node-specific frequency.
+
+        Args:
+            subscription (str): The name of the subscription to use
+            nodes (dict): A dictionary where keys are node identifiers and values are
+                configurations for each node. Each configuration must include a 'frequency'
+                key indicating the frequency of data collection in Hz.
+            collect_period (int): The data collection period in seconds.
+
+        Raises:
+            ValueError: If the subscription has not been created by calling
+                `create_subscription` prior to this method.
+
+        Side Effects:
+            - Updates internal node tracking and cycle rules
+            - Establishes data change monitoring for specified nodes
+            - Updates metrics for subscribed tags count
+        """
+
+        if not self.subscriptions.get(subscription):
+            raise ValueError('Subscription not created. Call create_subscription first.')
+
+        self.logger.info(f'Subscribing to {subscription} on {self.name}...')
+        self.logger.info(f'Subscribing to nodes: {nodes}')
+        assert self.client is not None  # Informa ao mypy que client não é None
+        addr_nodes = [self.client.get_node(n) for n in nodes]
+        self.logger.debug(f'Addr nodes: {addr_nodes}')
+        self.nodes.update(nodes)
+        self.logger.debug(f'Nodes: {self.nodes}')
+
+        self.collect_period = collect_period
+
+        for node, config in self.nodes.items():
+            self.nodes[node]['cycle_rule'] = {
+                'cycle_increment': collect_period * 1000 / float(config['frequency']),
+                'cycle_count': 0,
+            }
+
+        await self.subscriptions[subscription].subscribe_data_change(addr_nodes)
+
+        await self.emit_metric(
+            metric_object=metrics.OPC_TAGS_SUBSCRIBED,
+            method='set',
+            value=len(self.nodes),
+            tags={
+                'pod_id': self.pod_id,
+                'server_name': self.name,
+            },
+        )
+
+    async def unsubscribe(self, subscription: str):
+        """
+        Unsubscribes from a given subscription.
+
+        This method removes the specified subscription and cleans up associated
+        resources. It handles cases where the subscription doesn't exist gracefully.
+
+        Args:
+            subscription (str): The name of the subscription to unsubscribe from.
+
+        Logs:
+            - A warning if the specified subscription does not exist.
+            - An info message upon successful unsubscription.
+
+        Behavior:
+            - If the subscription exists, it is deleted and removed from the
+              subscriptions dictionary.
+            - If the subscription does not exist, no action is taken.
+        """
+
+        if not self.subscriptions.get(subscription):
+            self.logger.warning(f"Subscription '{subscription}' not found. Cannot unsubscribe.")
+            return
+        await self.subscriptions[subscription].delete()
+        del self.subscriptions[subscription]
+        self.logger.info(f'Unsubscribed from {subscription}.')
+
+    async def disconnection_fallback(self) -> list:
+        """
+        Tries 5 times to disconnect from the OPC UA server, with a delay of 100ms x try.
+        """
+
+        assert self.client is not None
+        error_stack = []
+        for i in range(5):
+            try:
+                self.logger.info(f'Disconnecting from OPC UA server, attempt {i + 1} of 5')
+                await self.client.disconnect()
+                return []
+            except Exception as e:
+                self.logger.error(
+                    f'Failed to disconnect from OPC UA serve  in attempt {i + 1} of 5: {e}'
+                )
+                error_stack.append(
+                    {
+                        'attempt': i + 1,
+                        'error': str(e),
+                        'traceback': traceback.format_exc(),
+                    }
+                )
+                await asyncio.sleep(0.1 * i)
+        return error_stack
+
+    async def disconnect(self):
+        """
+        Disconnects from the OPC UA server.
+
+        This method handles the disconnection process by deleting all subscriptions
+        and disconnecting the client from the OPC UA server. It logs the disconnection
+        process and handles any exceptions that may occur during cleanup.
+
+        Raises:
+            Exception: If an error occurs while deleting the subscription or disconnecting
+                       from the OPC UA server, it logs the error details.
+
+        Side Effects:
+            - Deletes all active subscriptions
+            - Disconnects the OPC client
+            - Updates connection status metrics
+            - Clears internal client reference
+        """
+
+        self.logger.warning('Disconnecting from OPC server')
+        if self.client is None:
+            self.logger.warning('Client already disconnected.')
+            return
+        try:
+            for sub in self.subscriptions:
+                await self.subscriptions[sub].delete()
+            self.logger.warning('Deleted all subscriptions.')
+        except Exception as sub_error:
+            self.logger.error(f'Failed to clean up subscription: {sub_error}')
+
+        errors = await self.disconnection_fallback()
+
+        if errors:
+            await self.send_notification_async(
+                metadata=self.metadata,
+                notification_id=f'OPC_DISCONNECTION_ERROR_{self.name}',
+                message=f'Failed to disconnect from OPC UA server {self.name} after 5 attempts',
+                block='opc_manager',
+                level=NotificationLevel.ERROR,
+                attachment_content=json.dumps(errors, indent=4),
+            )
+        else:
+            self.logger.warning('Disconnected from OPC UA server.')
+
+        del self.client
+        self.client = None
+        await self.emit_metric(
+            metric_object=metrics.OPC_CONNECTION_STATUS,
+            method='set',
+            value=0,
+            tags={
+                'pod_id': self.pod_id,
+                'server_name': self.name,
+                'server_url': self.url,
+            },
+        )
+        await self.emit_metric(
+            metric_object=metrics.OPC_TAGS_SUBSCRIBED,
+            method='set',
+            value=0,
+            tags={
+                'pod_id': self.pod_id,
+                'server_name': self.name,
+            },
+        )
+
+    async def datachange_notification(self, node, _val, data):
+        """
+        Handles data change notifications for monitored OPC UA nodes.
+
+        This method is triggered when a monitored node's value changes. It processes
+        the notification, updates internal state, and publishes the data to the
+        appropriate topics.
+
+        Args:
+            node (NodeId): The OPC UA node that triggered the data change notification.
+            _val (Any): The new value of the node (unused in this implementation).
+            data (DataChangeNotification): The data change notification object containing
+                details about the change.
+
+        Behavior:
+            - Extracts the value and source timestamp from the monitored item.
+            - Resets the cycle count for the node's cycle rule.
+            - Resets the non-receive count.
+            - Constructs a data dictionary containing the tag, tag name, timestamp, and value.
+            - Publishes the data to all topics associated with the node.
+        """
+
+        # get data value
+        monitored_item = data.monitored_item
+        value = monitored_item.Value.Value.Value
+        # source_timestamp
+        source_timestamp = monitored_item.Value.SourceTimestamp.replace(tzinfo=OPC_TIMEZONE)
+        tag = str(node)
+
+        self.logger.debug(
+            f'Data change notification received for tag:'
+            f'{tag} after {self.nodes[tag]["cycle_rule"]["cycle_count"]} cycles'
+        )
+
+        data = {
+            'tag': tag,
+            'name': self.nodes[str(node)]['tag_name'],
+            'timestamp': source_timestamp.strftime(DATETIME_FORMAT_WITH_TZ),
+            'value': value,
+        }
+
+        for topic in self.nodes[tag]['topics']:
+            await self.data_manager.publish(topic, data)
+
+        self.nodes[tag]['cycle_rule']['cycle_count'] = 0
+        self.non_receive_count = 0
+        await self.emit_metric(
+            metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
+            method='set',
+            value=0,
+            tags={
+                'pod_id': self.pod_id,
+                'server_name': self.name,
+            },
+        )
+
+    async def check_cycles(self):
+        """
+        Checks the cycle counts for all monitored nodes and sends
+        notifications if thresholds are exceeded.
+
+        This method iterates through all monitored nodes and updates their cycle counts based on
+        configured increments. If a node's cycle count exceeds a threshold (5 cycles), it triggers
+        a warning notification.
+
+        Side Effects:
+            - Updates cycle counts for all monitored nodes
+            - Sends warning notifications for nodes exceeding cycle thresholds
+        """
+        for node, config in self.nodes.items():
+            self.nodes[node]['cycle_rule']['cycle_count'] += config['cycle_rule']['cycle_increment']
+            if self.nodes[node]['cycle_rule']['cycle_count'] >= 5:
+                name = config['tag_name']
+                cycles = self.nodes[node]['cycle_rule']['cycle_count']
+                await self.send_notification_async(
+                    metadata=self.metadata,
+                    notification_id=f'TAG_{node}:{name}_LISTENNING_STOPPED',
+                    message=f'{cycles} cycles without receive from {node}:{name}',
+                    block='opc_manager',
+                    level=NotificationLevel.WARNING,
+                )
+
+    async def check_opc_listenning(self) -> bool:
+        """
+        Checks the OPC connection and triggers notifications if the connection is lost.
+
+        This method monitors the data reception health by tracking cycles without
+        data. It sends notifications at different thresholds and can trigger
+        reconnection attempts.
+
+        Returns:
+            bool: True if the connection is lost and reconnection should be attempted,
+                  False otherwise.
+
+        Side Effects:
+            - Increments non-receive count
+            - Updates metrics for cycles without data
+            - Sends warning notifications at 5 cycles
+            - Sends error notifications and triggers reconnection at 15 cycles
+        """
+
+        self.non_receive_count += 1
+        await self.emit_metric(
+            metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
+            method='set',
+            value=self.non_receive_count,
+            tags={
+                'pod_id': self.pod_id,
+                'server_name': self.name,
+            },
+        )
+        if self.non_receive_count >= 5:
+            await self.send_notification_async(
+                metadata=self.metadata,
+                notification_id=f'OPC_LISTENNING_STOPPED__{self.name}',
+                message=f'{self.non_receive_count} cycles without '
+                f'receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}',
+                block='opc_manager',
+                level=NotificationLevel.ERROR,
+            )
+        if self.non_receive_count >= 15:
+            await self.emit_metric(
+                metric_object=metrics.OPC_RECONNECTIONS_TOTAL,
+                tags={
+                    'pod_id': self.pod_id,
+                    'server_name': self.name,
+                },
+            )
+            await self.send_notification_async(
+                metadata=self.metadata,
+                notification_id=f'OPC_CONNECTION_RETRY__{self.name}',
+                message=f'Retrying to connect to server {self.name}',
+                block='opc_manager',
+                level=NotificationLevel.ERROR,
+            )
+            return True
+        return False

ingestor/managers/resource_manager.py

@@ -0,0 +1,315 @@
+from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+from sientia_do.observability.logger import Logger
+from sientia_do.observability.metrics_controller import MetricsController
+from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+from sientia_do.repository.redis_repository import RedisRepository
+
+
+class ResourceManager(SientiaMonitoring):
+    """
+    Manages Redis-based resource coordination and slot leasing for the OPC Ingestor.
+
+    The ResourceManager is responsible for:
+    - Coordinating slot allocation across multiple ingestor instances
+    - Managing lease lifecycles and heartbeats for load balancing
+    - Providing distributed locking and resource management
+    - Monitoring Redis operations and connection health
+
+    The manager implements a sophisticated slot leasing system that enables:
+    - Dynamic load distribution across multiple ingestor instances
+    - Automatic failover and recovery from instance failures
+    - Fair resource allocation based on system capacity
+    - Real-time monitoring of system health and performance
+
+    Args:
+        host (str): Redis server hostname
+        port (int): Redis server port
+        lease_ttl (int): Time-to-live for slot leases in seconds
+        heartbeat_ttl (int): Time-to-live for heartbeat signals in seconds
+        metadata (dict): Application metadata for notifications and tracking
+        logger (Logger): Logger instance for application logging
+        notification_handler (NotificationHandler): Handler for sending notifications
+        username (str, optional): Redis username for authentication
+        password (str, optional): Redis password for authentication
+
+    Attributes:
+        redis (Redis): Redis client instance
+        lease_ttl (int): Time-to-live for slot leases
+        heartbeat_ttl (int): Time-to-live for heartbeat signals
+        metadata (dict): Application metadata
+    """
+
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        lease_ttl: int,
+        heartbeat_ttl: int,
+        metadata: dict,
+        logger: Logger,
+        notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
+        username: str | None = None,
+        password: str | None = None,
+    ) -> None:
+        """
+        Initializes the ResourceManager with Redis connection and configuration.
+
+        This constructor establishes a connection to Redis and verifies connectivity
+        by performing a ping operation. It sets up the connection with optional
+        authentication and records the connection status in metrics.
+
+        Args:
+            host (str): Redis server hostname
+            port (int): Redis server port
+            lease_ttl (int): Time-to-live for slot leases in seconds
+            heartbeat_ttl (int): Time-to-live for heartbeat signals in seconds
+            metadata (dict): Application metadata
+            logger (Logger): Logger instance
+            notification_handler (NotificationHandler): Notification handler
+            username (str, optional): Redis username for authentication
+            password (str, optional): Redis password for authentication
+
+        Raises:
+            Exception: If Redis connection fails, the error is logged and metrics
+                      are updated before re-raising the exception.
+
+        Metrics:
+            - REDIS_CONNECTION_STATUS: Set to 1 on successful connection, 0 on failure
+        """
+        SientiaMonitoring.__init__(
+            self,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+        try:
+            self.redis_repository = RedisRepository(
+                host=host,
+                port=port,
+                username=username,
+                password=password,
+                logger=logger,
+                notification_handler=notification_handler,
+                metrics_controller=metrics_controller,
+            )
+            self.redis_repository.redis_client.ping()
+        except Exception as e:
+            self.logger.error(f'Failed to connect to Redis: {e}')
+            raise
+
+        self.lease_ttl = lease_ttl
+        self.heartbeat_ttl = heartbeat_ttl
+        self.metadata = metadata
+
+    async def get_tag_slot(self, tag_id: str) -> dict | None:
+        """
+        Retrieve the tag slot information for a given ID.
+
+        This method constructs the Redis key for a tag slot and retrieves
+        the associated configuration information.
+
+        Args:
+            id (str): The unique identifier of the tag slot to retrieve.
+
+        Returns:
+            dict: A dictionary containing the tag slot information associated
+                  with the given ID, or None if not found.
+
+        The method constructs the key using the pattern "slot:opc_tags:{id}"
+        and delegates to the get() method for the actual Redis operation.
+        """
+
+        self.info(f'Getting tag slot for tag_id: {tag_id}', metadata=self.metadata)
+
+        slot = await self.redis_repository.get(f'slot:opc_tags:{tag_id}', metadata=self.metadata)
+        self.info(f'Tag slot for tag_id: {tag_id} is: {slot}', metadata=self.metadata)
+        return slot
+
+    async def ingestor_heartbeat(self) -> None:
+        """
+        Sends a heartbeat signal to Redis to indicate that the ingestor is active.
+
+        This method sets a key in Redis with a specific format that includes the
+        ingestor's pod ID. The key is set with a value of 1 and an expiration
+        time defined by `self.heartbeat_ttl`. This allows monitoring systems to
+        track the activity and health of the ingestor.
+
+        The heartbeat mechanism enables:
+        - Load balancers to identify active ingestor instances
+        - Health monitoring systems to detect failed instances
+        - Automatic failover and recovery mechanisms
+
+        Metrics:
+            - REDIS_OPERATIONS_TOTAL: Incremented with operation="set"
+            - REDIS_OPERATIONS_DURATION: Records timing for heartbeat operations
+        """
+
+        await self.redis_repository.set(
+            f'heartbeat:ingestor:{self.pod_id}', 1, ttl=self.heartbeat_ttl, metadata=self.metadata
+        )
+
+    async def lease_tag(self, tag_id: str) -> bool:
+        """
+        Attempts to lease a tag by setting a key in Redis with a specified TTL.
+
+        This method uses the Redis `SET` command with the `NX` option to ensure that
+        the key is only set if it does not already exist. The key is set with an
+        expiration time defined by `lease_ttl`. This implements a distributed
+        locking mechanism for tag allocation.
+
+        Args:
+            tag_id (str): The unique identifier of the tag to be leased.
+
+        Returns:
+            bool: True if the lease was successfully acquired, False if the tag
+                  is already leased by another ingestor.
+
+        The leasing mechanism ensures:
+        - Only one ingestor can process a specific tag at a time
+        - Automatic lease expiration prevents deadlocks
+        - Fair distribution of tags across available ingestor instances
+
+        Metrics:
+            - REDIS_OPERATIONS_TOTAL: Incremented with operation="set_nx"
+            - REDIS_OPERATIONS_DURATION: Records timing for lease operations
+        """
+
+        return await self.redis_repository.set(
+            f'lease:opc_tags:{tag_id}',
+            self.pod_id,
+            ttl=self.lease_ttl,
+            nx=True,
+            metadata=self.metadata,
+        )
+
+    async def renew_tag_lease(self, tag_id: str) -> bool:
+        """
+        Renews the lease for a specific OPC tag if the current pod holds the lease.
+
+        This method checks if the current pod (identified by `self.pod_id`) holds
+        the lease for the given OPC tag. If so, it extends the lease by resetting
+        its expiration time in Redis to the configured lease TTL.
+
+        Args:
+            tag_id (str): The identifier of the OPC tag whose lease is to be renewed.
+
+        Returns:
+            bool: True if the lease was successfully renewed, False if the current
+                  pod doesn't hold the lease or renewal failed.
+
+        Lease renewal is essential for:
+        - Maintaining continuous tag processing without interruptions
+        - Preventing lease expiration during long-running operations
+        - Ensuring system stability and reliability
+
+        Metrics:
+            - REDIS_OPERATIONS_TOTAL: Incremented with operation="get" and "expire"
+            - REDIS_OPERATIONS_DURATION: Records timing for renewal operations
+        """
+
+        current = await self.redis_repository.get(
+            f'lease:opc_tags:{tag_id}', metadata=self.metadata
+        )
+        if current == self.pod_id:
+            await self.redis_repository.expire(
+                f'lease:opc_tags:{tag_id}', self.lease_ttl, metadata=self.metadata
+            )
+            return True
+        return False
+
+    async def drop_tag_lease(self, tag_id: str) -> None:
+        """
+        Drops the lease for a specific OPC tag.
+
+        This method removes the lease for the given OPC tag by deleting the
+        corresponding key in Redis. This is typically called when an ingestor
+        is shutting down or when it needs to release a tag for reallocation.
+
+        Args:
+            tag_id (str): The identifier of the OPC tag whose lease is to be dropped.
+
+        Lease dropping enables:
+        - Graceful shutdown of ingestor instances
+        - Dynamic reallocation of tags for load balancing
+        - Recovery from failed or unresponsive ingestor instances
+
+        Metrics:
+            - REDIS_OPERATIONS_TOTAL: Incremented with operation="delete"
+            - REDIS_OPERATIONS_DURATION: Records timing for lease dropping operations
+        """
+
+        await self.redis_repository.delete(f'lease:opc_tags:{tag_id}', metadata=self.metadata)
+
+    async def get_all_ingestors(self) -> list[str]:
+        """
+        Retrieves all active ingestors from Redis.
+
+        This method fetches all keys in Redis that match the pattern for ingestor
+        heartbeats and returns a list of active ingestor identifiers. The method
+        uses the pattern "heartbeat:ingestor:*" to find all active instances.
+
+        Returns:
+            List[str]: A list of active ingestor identifiers, extracted from
+                      the Redis keys by removing the "heartbeat:ingestor:" prefix.
+
+        This information is used for:
+        - Load balancing calculations
+        - System health monitoring
+        - Resource allocation decisions
+
+        Metrics:
+            - REDIS_OPERATIONS_TOTAL: Incremented with operation="keys"
+            - REDIS_OPERATIONS_DURATION: Records timing for ingestor discovery
+        """
+
+        return await self.redis_repository.keys('heartbeat:ingestor:*', metadata=self.metadata)
+
+    async def get_all_slots(self) -> list[str]:
+        """
+        Retrieves all available slots from Redis.
+
+        This method fetches all keys in Redis that match the pattern for OPC tag
+        slots and returns a list of slot identifiers. The method uses the pattern
+        "slot:opc_tags:*" to find all configured slots.
+
+        Returns:
+            List[str]: A list of slot identifiers, extracted from the Redis keys
+                      by removing the "slot:opc_tags:" prefix.
+
+        Slot information is used for:
+        - Resource allocation planning
+        - Load balancing across ingestor instances
+        - System capacity monitoring
+
+        Metrics:
+            - REDIS_OPERATIONS_TOTAL: Incremented with operation="keys"
+            - REDIS_OPERATIONS_DURATION: Records timing for slot discovery
+        """
+
+        return await self.redis_repository.keys('slot:opc_tags:*', metadata=self.metadata)
+
+    async def get_all_leases(self) -> list[str]:
+        """
+        Retrieves all active leases from Redis.
+
+        This method fetches all keys in Redis that match the pattern for OPC tag
+        leases and returns a list of lease identifiers. The method uses the pattern
+        "lease:opc_tags:*" to find all active leases.
+
+        Returns:
+            List[str]: A list of lease identifiers, extracted from the Redis keys
+                      by removing the "lease:opc_tags:" prefix.
+
+        Lease information is used for:
+        - Current resource utilization monitoring
+        - Load balancing calculations
+        - System health and performance analysis
+
+        Metrics:
+            - REDIS_OPERATIONS_TOTAL: Incremented with operation="keys"
+            - REDIS_OPERATIONS_DURATION: Records timing for lease discovery
+        """
+
+        return await self.redis_repository.keys('lease:opc_tags:*', metadata=self.metadata)

ingestor/metrics.py

@@ -0,0 +1,158 @@
+"""
+Prometheus metrics configuration for the OPC Ingestor application.
+
+This module defines all the metrics used for monitoring and observability
+of the OPC Ingestor system. It includes metrics for:
+
+- Application health and performance
+- OPC server connections and subscriptions
+- Data processing and storage operations
+- Resource management and load balancing
+- Error tracking and notification systems
+
+All metrics follow Prometheus naming conventions and include appropriate
+labels for multi-dimensional analysis and alerting.
+"""
+
+from prometheus_client import Counter, Gauge, Histogram
+
+# Metric label definitions for consistent labeling across all metrics
+POD_ID_LABEL = ['pod_id']
+SERVER_LABELS = ['pod_id', 'server_name', 'server_url']
+KAFKA_LABELS = ['pod_id', 'topic']
+REDIS_LABELS = ['pod_id', 'operation']
+NOTIFICATION_LABELS = ['pod_id', 'level', 'block']
+
+MAIN_LABELS = ['pod_id']
+
+# --- Reliability Metrics ---
+TAG_WRITTEN_COUNT = Counter(
+    'ingestor_tag_written_count',
+    'Number of writing process to the collection',
+    [*MAIN_LABELS, 'tag_name', 'collection_name'],
+)
+
+# --- General Application Metrics ---
+APP_LOOP_COUNT = Counter(
+    'app_main_loop_total',
+    'Total number of times the application main loop has run',
+    POD_ID_LABEL,
+)
+APP_LOOP_DURATION = Histogram(
+    'app_main_loop_duration_seconds',
+    'Duration of the application main loop in seconds',
+    POD_ID_LABEL,
+)
+APP_ERRORS_TOTAL = Counter(
+    'app_errors_total',
+    'Total number of unhandled errors in the main loop',
+    POD_ID_LABEL,
+)
+APP_UP = Gauge(
+    'app_up',
+    'Indicates if the application is running (1) or shutting down (0)',
+    POD_ID_LABEL,
+)
+
+# --- Ingestor Manager Metrics ---
+ACTIVE_INGESTORS = Gauge(
+    'ingestor_active_total',
+    'Number of active ingestors reported by Redis',
+    POD_ID_LABEL,
+)
+SLOTS_TOTAL = Gauge(
+    'ingestor_slots_total',
+    'Total number of slots configured in Redis',
+    POD_ID_LABEL,
+)
+LEASES_TOTAL = Gauge(
+    'ingestor_leases_total',
+    'Total number of leases (allocated slots) in Redis',
+    POD_ID_LABEL,
+)
+SLOTS_MANAGED = Gauge(
+    'ingestor_slots_managed_current',
+    'Number of slots currently managed by this ingestor instance',
+    POD_ID_LABEL,
+)
+SLOTS_ACQUIRED = Counter(
+    'ingestor_slots_acquired_total',
+    'Total number of slots acquired by this instance',
+    POD_ID_LABEL,
+)
+SLOTS_RELEASED = Counter(
+    'ingestor_slots_released_total',
+    'Total number of slots released by this instance',
+    POD_ID_LABEL,
+)
+
+# --- OPC Manager Metrics ---
+OPC_MANAGERS_ACTIVE = Gauge(
+    'ingestor_opc_managers_active',
+    'Number of active OPC Managers in this instance',
+    POD_ID_LABEL,
+)
+OPC_SUBSCRIPTION_ERRORS = Counter(
+    'ingestor_opc_subscription_errors_total',
+    'Errors when trying to subscribe to OPC tags',
+    ['pod_id', 'server', 'slot'],
+)
+
+OPC_CONNECTIONS_TOTAL = Counter(
+    'opc_connections_initiated_total',
+    'Total connection attempts to OPC servers',
+    ['pod_id', 'server_name'],
+)
+OPC_CONNECTIONS_FAILED = Counter(
+    'opc_connections_failed_total',
+    'Total failed connection attempts to OPC servers',
+    ['pod_id', 'server_name'],
+)
+OPC_CONNECTION_STATUS = Gauge(
+    'opc_connection_status',
+    'Connection status with the OPC server (1=connected, 0=disconnected)',
+    SERVER_LABELS,
+)
+OPC_SUBSCRIPTIONS_CREATED = Counter(
+    'opc_subscriptions_created_total',
+    'Total OPC subscriptions created',
+    ['pod_id', 'server_name', 'slot_name'],
+)
+OPC_TAGS_SUBSCRIBED = Gauge(
+    'opc_tags_subscribed_current',
+    'Current number of OPC tags subscribed on a server',
+    ['pod_id', 'server_name'],
+)
+OPC_CYCLES_WITHOUT_DATA = Gauge(
+    'opc_cycles_without_data',
+    'Current number of cycles without receiving data from a server',
+    ['pod_id', 'server_name'],
+)
+OPC_RECONNECTIONS_TOTAL = Counter(
+    'opc_reconnections_tried_total',
+    'Reconnection attempts to an OPC server after a loss',
+    ['pod_id', 'server_name'],
+)
+
+# --- Data Manager (Kafka) Metrics ---
+KAFKA_MESSAGES_SENT = Counter(
+    'kafka_messages_sent_total', 'Total messages sent to Kafka', KAFKA_LABELS
+)
+KAFKA_MESSAGES_ERRORS = Counter(
+    'kafka_messages_errors_total',
+    'Total errors sending messages to Kafka',
+    KAFKA_LABELS,
+)
+KAFKA_CONNECTION_STATUS = Gauge(
+    'kafka_connection_status',
+    'Connection status with Kafka (1=connected, 0=disconnected)',
+    POD_ID_LABEL,
+)
+
+
+# --- Notification Metrics ---
+NOTIFICATIONS_SENT = Counter(
+    'notifications_sent_total',
+    'Total number of notifications sent',
+    NOTIFICATION_LABELS,
+)

pyproject.toml

@@ -0,0 +1,155 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ingestor"
+version = "0.0.0"
+description = "Sientia DataOps Ingestor - OPC Tag Ingestor"
+readme = "README.md"
+requires-python = ">=3.11"
+authors = [
+    {name = "Aignosi", email = "dev@aignosi.com"}
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+exclude = [
+    ".git",
+    ".venv",
+    "venv",
+    "__pycache__",
+    "*.pyc",
+    ".pytest_cache",
+    "htmlcov",
+]
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # pyflakes
+    "I",   # isort
+    "B",   # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "UP",  # pyupgrade
+    "N",   # pep8-naming
+    "YTT", # flake8-2020
+    "S",   # flake8-bandit
+    "BLE", # flake8-blind-except
+    "A",   # flake8-builtins
+    "C90", # mccabe complexity
+]
+
+ignore = [
+    "BLE001",  # ignore blind except, we need to send notifications with any error
+    "E501",  # line too long (handled by formatter)
+    "S101",  # use of assert (needed for tests)
+    "S105",  # possible hardcoded password (false positives)
+    "S106",  # possible hardcoded password (false positives)
+    "N802",  # function name should be lowercase (temporal decorators)
+    "N806",  # variable in function should be lowercase
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = [
+    "S101",  # assert allowed in tests
+    "S105",  # hardcoded passwords ok in tests
+    "S106",  # hardcoded passwords ok in tests
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 15
+
+[tool.ruff.format]
+quote-style = "single"
+indent-style = "space"
+line-ending = "auto"
+
+[tool.mypy]
+python_version = "3.11"
+warn_return_any = false
+warn_unused_configs = true
+disallow_untyped_defs = false
+disallow_incomplete_defs = false
+check_untyped_defs = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = false
+warn_no_return = true
+strict_equality = true
+ignore_missing_imports = true
+
+# Ignore missing imports for external packages
+[[tool.mypy.overrides]]
+module = "temporalio.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "sientia_do.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "mlflow.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "prometheus_client.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "sientia.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "pandas.*"
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-v",
+    "--strict-markers"
+]
+markers = [
+    "asyncio: marks tests as async",
+    "integration: marks tests as integration tests",
+    "unit: marks tests as unit tests",
+]
+
+[tool.coverage.run]
+source = ["model_manager"]
+omit = [
+    "*/tests/*",
+    "*/venv/*",
+    "*/__pycache__/*",
+    "*/site-packages/*",
+]
+branch = true
+
+[tool.coverage.report]
+precision = 2
+show_missing = true
+skip_covered = false
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "def __str__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "class .*\\bProtocol\\):",
+    "@(abc\\.)?abstractmethod",
+]
+
+[tool.coverage.html]
+directory = "htmlcov"
+
+[tool.bandit]
+exclude_dirs = ["tests", "venv", ".venv"]
+skips = ["B101", "B601"]  # Skip assert and shell injection in controlled environments

requirements-dev.txt

@@ -0,0 +1,19 @@
+# Development and Testing Dependencies
+# These packages are only needed for development, testing, and code quality checks
+# Install with: pip install -r requirements-dev.txt
+
+# Code Quality & Linting
+ruff>=0.1.0              # Fast Python linter and formatter (replaces flake8, black, isort)
+mypy>=1.7.0              # Static type checker
+bandit>=1.7.5            # Security vulnerability scanner
+pandas-stubs>=2.0.0      # Type stubs for pandas
+types-requests>=2.31.0   # Type stubs for requests
+
+# Testing
+pytest>=7.4.0            # Testing framework
+pytest-cov>=4.1.0        # Coverage plugin for pytest
+pytest-asyncio>=0.21.0   # Async test support (already in main requirements)
+
+# Development Tools
+ipython>=8.12.0          # Enhanced Python shell
+ipdb>=0.13.13            # IPython debugger

requirements.txt

@@ -0,0 +1,5 @@
+asyncua==1.1.5
+redis
+git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.5.3
+prometheus_client
+pymongo

run_coverage.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Exit on any error
+set -e
+
+echo "Activating virtual environment..."
+source ./venv/bin/activate
+
+pytest --cov=ingestor --cov-report=html
+
+xdg-open htmlcov/index.html

run_local.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Exit on any error
+set -e
+
+echo "Activating virtual environment..."
+source ./venv/bin/activate
+
+echo "Loading environment variables from .env..."
+if [ -f .env ]; then
+    export $(cat .env | grep -v '^#' | xargs)
+    echo "Environment variables loaded from .env"
+else
+    echo "Warning: .env file not found. Continuing without environment variables."
+fi
+
+echo "Starting ingestor application..."
+python -m ingestor.app

sonar-project.properties

@@ -0,0 +1,11 @@
+sonar.projectKey=Aignosi_sientia-dataops-opc-ingestor_7a3d9693-a2bf-4699-bd82-df68d9a854f3
+sonar.projectName=sientia-dataops-opc-ingestor
+sonar.sources=ingestor
+sonar.tests=tests
+sonar.qualitygate.wait=true
+sonar.qualitygate.timeout=300
+sonar.python.coverage.reportPaths=coverage.xml
+sonar.python.xunit.reportPath=pytest.xml
+sonar.python.version=3.11
+sonar.projectVersion=1.2.0
+sonar.coverage.exclusions=ingestor/app.py

tests/functional/conftest.py

@@ -0,0 +1,47 @@
+# import subprocess
+# from time import sleep
+# from typing import Generator
+# import uuid
+# from kafka import KafkaConsumer
+# import pytest
+# from redis import Redis
+
+
+# @pytest.fixture(scope="session", autouse=True)
+# def docker_compose():
+#     """Sobe os containers antes dos testes e derruba depois."""
+#     print("\n🚀 Subindo Docker Compose...")
+#     subprocess.run(["docker", "compose", "up", "-d"], check=True)
+
+#     print("⏳ Aguardando containers ficarem prontos...")
+#     sleep(15)  # ajuste conforme necessário
+
+#     yield  # os testes rodam aqui
+
+#     print("\n🧹 Derrubando Docker Compose...")
+#     subprocess.run(["docker", "compose", "down"], check=True)
+
+
+# @pytest.fixture()
+# def redis_client():
+#     redis = Redis(host="localhost", port=6379, decode_responses=True)
+#     redis.flushdb()
+
+#     yield redis
+
+#     # Limpa o banco de dados após os testes
+#     redis.flushdb()
+#     redis.close()
+
+
+# def kafka_searcher(topic) -> Generator[KafkaConsumer, None, None]:
+#     consumer = KafkaConsumer(
+#         topic,
+#         bootstrap_servers="localhost:9092",
+#         group_id=f"test-group-{uuid.uuid4()}",
+#         auto_offset_reset="earliest",   # Começa a consumir apenas mensagens novas
+#         enable_auto_commit=True,
+#     )
+
+#     yield consumer
+#     consumer.close()

tests/functional/test_single_node.py

@@ -0,0 +1,118 @@
+# import json
+# import subprocess
+# from time import sleep
+
+# from tests.functional.conftest import kafka_searcher
+
+
+# new_data = {
+#     "slot:opc_tags:1": {
+#         "server1": {
+#             "name": "server1",
+#             "url": "opc.tcp://simulator:4840",
+#             "server_uri": "http://opcua-server.simulator",
+#             "tags": {
+#                 'ns=2;i=2': {
+#                     'tag_name': 'Counter',
+#                     'frequency': 1000,
+#                     'topics': [],
+#                 },
+#                 'ns=2;i=3': {
+#                     'tag_name': 'Rollout',
+#                     'frequency': 1000,
+#                     "topics": [],
+#                 },
+#                 'ns=2;i=4': {
+#                     'tag_name': 'Square',
+#                     'frequency': 1000,
+#                     "topics": [],
+#                 },
+#             }
+#         }
+#     },
+#     "slot:opc_tags:2": {
+#         "server2": {
+#             "name": "server2",
+#             "url": "opc.tcp://simulator:4840",
+#             "server_uri": "http://opcua-server.simulator",
+#             "tags": {
+#                 'ns=2;i=2': {
+#                     'tag_name': 'Counter',
+#                     'frequency': 1000,
+#                     'topics': [],
+#                 },
+#                 'ns=2;i=3': {
+#                     'tag_name': 'Rollout',
+#                     'frequency': 1000,
+#                     "topics": [],
+#                 },
+#                 'ns=2;i=4': {
+#                     'tag_name': 'Square',
+#                     'frequency': 1000,
+#                     "topics": [],
+#                 },
+#             }
+#         }
+#     }
+# }
+
+
+# def test_simple(redis_client):
+#     new_data['slot:opc_tags:1']['server1']['tags']['ns=2;i=2']['topics'] = [
+#         'test_topic_1']
+#     redis_client.set("slot:opc_tags:1",
+#                      json.dumps(new_data['slot:opc_tags:1']))
+
+#     sleep(20)  # Espera o Ingestor processar os dados
+
+#     # Check if lease is in Redis
+#     assert redis_client.get("lease:opc_tags:1") == 'ingestor'
+#     assert redis_client.get("heartbeat:ingestor:ingestor") == '1'
+
+#     # Check if data is in Kafka
+
+#     kafka = next(kafka_searcher('test_topic_1'))
+#     sleep(1)
+#     messages = kafka.poll(timeout_ms=10000)
+
+#     assert messages, "Expected messages in Kafka, but got none."
+
+
+# def test_simple_double_slot(redis_client):
+
+#     new_data['slot:opc_tags:1']['server1']['tags']['ns=2;i=2']['topics'] = [
+#         'test_topic_double_slot1']
+#     redis_client.set("slot:opc_tags:1",
+#                      json.dumps(new_data['slot:opc_tags:1']))
+
+#     sleep(20)  # Espera o Ingestor processar os dados
+
+#     assert redis_client.get("lease:opc_tags:1") == 'ingestor'
+#     assert redis_client.get("heartbeat:ingestor:ingestor") == '1'
+
+#     # Check if data is in Kafka
+#     kafka1 = next(kafka_searcher('test_topic_double_slot1'))
+#     messages = kafka1.poll(timeout_ms=10000)
+
+#     assert messages, "Expected messages in test_topic_double_slot1, but got none."
+
+#     new_data['slot:opc_tags:2']['server2']['tags']['ns=2;i=2']['topics'] = [
+#         'test_topic_double_slot2']
+#     redis_client.set("slot:opc_tags:2",
+#                      json.dumps(new_data['slot:opc_tags:2']))
+
+#     sleep(20)  # Espera o Ingestor processar os dados
+
+#     # Check if lease is in Redis
+#     assert redis_client.get("lease:opc_tags:2") == 'ingestor'
+#     assert redis_client.get("lease:opc_tags:1") == 'ingestor'
+#     assert redis_client.get("heartbeat:ingestor:ingestor") == '1'
+
+#     # Check if data is in Kafka
+#     kafka2 = next(kafka_searcher('test_topic_double_slot2'))
+#     messages = kafka2.poll(timeout_ms=10000)
+
+#     assert messages, "Expected messages in test_topic_double_slot2, but got none."
+
+#     messages = kafka1.poll(timeout_ms=10000)
+#     assert messages, "Expected messages in test_topic_double_slot1, but got none."

tests/unit/managers/test_data_manager.py

@@ -0,0 +1,283 @@
+from unittest.mock import ANY, AsyncMock, MagicMock, patch
+
+from kafka.errors import NoBrokersAvailable
+from pytest import fixture, mark
+from sientia_do.notifications.models import NotificationLevel
+
+from ingestor.managers.data_manager import DataManager
+
+metadata = {
+    'metadata': {
+        'model_id': 'test_model',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+        'schema_name': 'test_schedule',
+        'pod_id': 'localhost',
+    },
+}
+
+
+@fixture
+@patch('ingestor.managers.data_manager.KafkaProducer')
+@patch('ingestor.managers.data_manager.MongoDBRepository')
+def data_manager(mongodb_repository, kafka):
+    data_manager = DataManager(
+        kafka_servers='localhost:9092',
+        mongo_connection_string='mongodb://localhost:27017',
+        mongo_database='sientia',
+        export_to_kafka=True,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metadata=metadata['metadata'],
+        metrics_controller=MagicMock(),
+    )
+
+    data_manager.send_notification = MagicMock()
+    data_manager.send_notification_async = AsyncMock()
+    data_manager.emit_metric = AsyncMock()
+    return data_manager
+
+
+@patch('ingestor.managers.data_manager.KafkaProducer')
+@patch('ingestor.managers.data_manager.MongoDBRepository')
+def test___init___success(mongodb_repository, kafka):
+    logger_mock = MagicMock()
+
+    data_manager = DataManager(
+        metadata=metadata['metadata'],
+        kafka_servers='localhost:9092',
+        mongo_connection_string='mongodb://localhost:27017',
+        mongo_database='sientia',
+        export_to_kafka=True,
+        logger=logger_mock,
+        notification_handler=MagicMock(),
+        metrics_controller=MagicMock(),
+    )
+
+    kafka.assert_called_once_with(
+        bootstrap_servers='localhost:9092', value_serializer=ANY, key_serializer=ANY
+    )
+    assert data_manager.kafka_producer is not None
+    logger_mock.info.assert_any_call(
+        'Trying (0) to initializing DataManager with Kafka servers: localhost:9092'
+    )
+    logger_mock.info.assert_any_call('DataManager initialized with Kafka servers: localhost:9092')
+    logger_mock.error.assert_not_called()
+    assert logger_mock.info.call_count == 4
+
+
+@patch('ingestor.managers.data_manager.KafkaProducer')
+@patch('ingestor.managers.data_manager.MongoDBRepository')
+def test___init___second_attempt(mongodb_repository, kafka):
+    kafka.side_effect = [NoBrokersAvailable, MagicMock()]
+    logger_mock = MagicMock()
+
+    data_manager = DataManager(
+        kafka_servers='localhost:9092',
+        mongo_connection_string='mongodb://localhost:27017',
+        mongo_database='sientia',
+        export_to_kafka=True,
+        logger=logger_mock,
+        notification_handler=MagicMock(),
+        metadata=metadata['metadata'],
+        metrics_controller=MagicMock(),
+    )
+
+    kafka.assert_any_call(
+        bootstrap_servers='localhost:9092', value_serializer=ANY, key_serializer=ANY
+    )
+    assert kafka.call_count == 2
+    assert data_manager.kafka_producer is not None
+    logger_mock.info.assert_any_call(
+        'Trying (0) to initializing DataManager with Kafka servers: localhost:9092'
+    )
+    logger_mock.info.assert_any_call(
+        'Trying (1) to initializing DataManager with Kafka servers: localhost:9092'
+    )
+    logger_mock.info.assert_any_call('DataManager initialized with Kafka servers: localhost:9092')
+    logger_mock.error.assert_called_once_with(
+        'Kafka servers localhost:9092 are not available. Retrying...'
+    )
+    assert logger_mock.info.call_count == 5
+
+
+@patch('ingestor.managers.data_manager.KafkaProducer')
+@patch('ingestor.managers.data_manager.MongoDBRepository')
+def test___init___failure_max_attempts(mongodb_repository, kafka):
+    kafka.side_effect = NoBrokersAvailable
+    logger_mock = MagicMock()
+
+    try:
+        DataManager(
+            kafka_servers='localhost:9092',
+            mongo_connection_string='mongodb://localhost:27017',
+            mongo_database='sientia',
+            export_to_kafka=True,
+            logger=logger_mock,
+            notification_handler=MagicMock(),
+            metadata=metadata['metadata'],
+            metrics_controller=MagicMock(),
+        )
+    except NoBrokersAvailable as e:
+        assert (
+            str(e)
+            == 'NoBrokersAvailable: Failed to connect to Kafka servers localhost:9092 after 3 attempts.'
+        )
+
+        assert kafka.call_count == 3
+        logger_mock.info.assert_any_call(
+            'Trying (0) to initializing DataManager with Kafka servers: localhost:9092'
+        )
+        logger_mock.info.assert_any_call(
+            'Trying (1) to initializing DataManager with Kafka servers: localhost:9092'
+        )
+        logger_mock.info.assert_any_call(
+            'Trying (2) to initializing DataManager with Kafka servers: localhost:9092'
+        )
+        logger_mock.error.assert_called_with(
+            'Failed to connect to Kafka servers localhost:9092 after 3 attempts.'
+        )
+        assert logger_mock.info.call_count == 3
+
+    else:
+        raise AssertionError('Expected NoBrokersAvailable exception was not raised.')
+
+
+def test_shutdown_has_producer(data_manager):
+    flush_mock = MagicMock()
+    close_mock = MagicMock()
+
+    data_manager.kafka_producer.flush = flush_mock
+    data_manager.kafka_producer.close = close_mock
+
+    data_manager.shutdown()
+    flush_mock.assert_called_once()
+    close_mock.assert_called_once()
+
+
+def test_shutdown_no_producer(data_manager):
+    data_manager.kafka_producer = None
+
+    data_manager.shutdown()
+
+    data_manager.logger.warning.assert_any_call(
+        'Kafka producer is already closed or not initialized.'
+    )
+
+
+def test_shutdown_exception(data_manager):
+    data_manager.kafka_producer.flush = MagicMock(side_effect=Exception('Test error'))
+    data_manager.kafka_producer.close = MagicMock()
+
+    data_manager.shutdown()
+    data_manager.logger.error.assert_called_once_with('Error closing Kafka producer: Test error')
+
+
+def test_shutdown_exception_mongo(data_manager):
+    data_manager.mongo_repository.close = MagicMock(side_effect=Exception('Test error'))
+
+    data_manager.shutdown()
+
+    data_manager.logger.error.assert_called_once_with('Error closing MongoDB client: Test error')
+
+
+def test___del__(data_manager):
+    data_manager.shutdown = MagicMock()
+    data_manager.__del__()
+    data_manager.shutdown.assert_called_once()
+
+
+def test_delivery_report(data_manager):
+    msg = MagicMock()
+    msg.topic = 'test_topic'
+    msg.partition = 0
+    msg.offset = 1
+
+    data_manager.delivery_report(msg)
+
+    data_manager.logger.debug.assert_called_once_with(
+        f'Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}'
+    )
+
+
+def test_delivery_error(data_manager):
+    err = 'Test error'
+    data_manager.delivery_error(err)
+
+    data_manager.logger.error.assert_called_once_with(f'Delivery failed for record : {err}')
+
+
+@mark.asyncio
+async def test_publish(data_manager):
+    topic = 'test_topic'
+    data = {'key': 'value'}
+
+    # Mock the send method of the Kafka producer
+    send_mock = MagicMock()
+    data_manager.kafka_producer.send = send_mock
+
+    # Call the publish method
+    await data_manager.publish(topic, data)
+
+    # Check if the send method was called with the correct arguments
+    send_mock.assert_called_once_with(topic=topic, value=data)
+
+    send_mock.return_value.add_callback.assert_called_once()
+
+    data_manager.kafka_producer.flush.assert_called_once()
+
+
+def test_publish_no_kafka(data_manager):
+    data_manager.export_to_kafka = False
+    topic = 'test_topic'
+    data = {'key': 'value'}
+
+    data_manager.publish(topic, data)
+
+    data_manager.kafka_producer.send.assert_not_called()
+
+
+@patch('ingestor.managers.data_manager.traceback')
+@mark.asyncio
+async def test_publish_error(traceback, data_manager):
+    topic = 'test_topic'
+    data = {'key': 'value', 'name': 'test_tag'}
+
+    # Mock the send method of the Kafka producer to raise an exception
+    send_mock = MagicMock(side_effect=Exception('Test error'))
+    data_manager.kafka_producer.send = send_mock
+
+    data_manager.mongo_repository = AsyncMock()
+
+    # Call the publish method
+    await data_manager.publish(topic, data)
+
+    # Check if the send method was called with the correct arguments
+    send_mock.assert_called_once_with(topic=topic, value=data)
+
+    # Check if the error was logged
+    data_manager.send_notification_async.assert_called_once_with(
+        notification_id=f'KAFKA_PRODUCER_ERROR_{topic}',
+        message=f'Error publishing message to topic {topic}: Test error',
+        block='kafka_producer',
+        level=NotificationLevel.ERROR,
+        attachment_content=traceback.format_exc.return_value,
+        metadata=metadata['metadata'],
+    )
+
+
+@mark.asyncio
+async def test_publish_error_mongo(data_manager):
+    data_manager.export_to_kafka = False
+    data_manager.mongo_repository.insert = AsyncMock(side_effect=Exception('Test error'))
+
+    await data_manager.publish('test_topic', {'key': 'value'})
+
+    data_manager.send_notification_async.assert_called_once_with(
+        notification_id='MONGO_PRODUCER_ERROR_test_topic',
+        message='Error inserting message to MongoDB: Test error',
+        block='mongo_producer',
+        level=NotificationLevel.ERROR,
+        attachment_content=ANY,
+        metadata=metadata['metadata'],
+    )

tests/unit/managers/test_ingestor_manager.py

@@ -0,0 +1,654 @@
+from unittest.mock import AsyncMock, MagicMock, call, patch
+
+from pytest import fixture, mark
+from sientia_do.notifications.models import NotificationLevel
+
+from ingestor.managers.ingestor_manager import IngestorManager
+
+metadata = {
+    'metadata': {
+        'model_id': 'test_model',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+        'schema_name': 'test_schedule',
+    },
+}
+
+
+@fixture
+@patch('ingestor.managers.ingestor_manager.DataManager')
+@patch('ingestor.managers.ingestor_manager.ResourceManager')
+def ingestor_manager(data_manager_mock, resource_manager_mock):
+    ingestor = IngestorManager(
+        kafka_servers='localhost:9092',
+        redis_data={'host': 'localhost', 'port': 6379},
+        lease_ttl=60,
+        heartbeat_ttl=60,
+        poll_interval=5,
+        mongo_connection_string='mongodb://localhost:27017',
+        mongo_database='sientia',
+        export_to_kafka=False,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metadata=metadata['metadata'],
+        metrics_controller=MagicMock(),
+    )
+
+    ingestor.send_notification = MagicMock()
+    ingestor.send_notification_async = AsyncMock()
+    ingestor.emit_metric = AsyncMock()
+
+    return ingestor
+
+
+@patch('ingestor.managers.ingestor_manager.OpcManager')
+@patch('ingestor.managers.ingestor_manager.DataManager')
+@patch('ingestor.managers.ingestor_manager.ResourceManager')
+@patch('ingestor.managers.ingestor_manager.NotificationHandler')
+def test___init__(
+    notification_handler_mock, resource_manager_mock, data_manager_mock, opc_manager_mock
+):
+    ingestor = IngestorManager(
+        kafka_servers='localhost:9092',
+        redis_data={'host': 'localhost', 'port': 6379},
+        lease_ttl=60,
+        heartbeat_ttl=60,
+        poll_interval=5,
+        mongo_connection_string='mongodb://localhost:27017',
+        mongo_database='sientia',
+        export_to_kafka=False,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metadata=metadata['metadata'],
+        metrics_controller=MagicMock(),
+    )
+
+    opc_manager_mock.assert_not_called()
+    data_manager_mock.assert_called_once_with(
+        kafka_servers='localhost:9092',
+        mongo_connection_string='mongodb://localhost:27017',
+        mongo_database='sientia',
+        export_to_kafka=False,
+        metadata=metadata['metadata'],
+        logger=ingestor.logger,
+        notification_handler=ingestor.notification_handler,
+        metrics_controller=ingestor.metrics_controller,
+    )
+    resource_manager_mock.assert_called_once_with(
+        host='localhost',
+        port=6379,
+        lease_ttl=60,
+        heartbeat_ttl=60,
+        metadata=metadata['metadata'],
+        logger=ingestor.logger,
+        notification_handler=ingestor.notification_handler,
+        username=None,
+        password=None,
+        metrics_controller=ingestor.metrics_controller,
+    )
+    assert ingestor.poll_interval == 5
+    assert ingestor.managed_tags == {}
+    assert ingestor.opc_servers == {}
+    assert ingestor.opc_managers == {}
+    assert ingestor.data_manager == data_manager_mock.return_value
+    assert ingestor.resource_manager == resource_manager_mock.return_value
+
+
+@mark.asyncio
+@patch('ingestor.managers.ingestor_manager.OpcManager')
+async def test_initialize_opc_from_config(opc_manager, ingestor_manager):
+    server_config = {
+        'name': 'server1',
+        'url': 'opc.tcp://localhost:4840',
+        'subscription_period_ms': 1000,
+        'server_uri': 'http://opcua-server.simulator',
+        'cert_path': '/path/to/cert',
+        'private_key_path': '/path/to/private_key',
+        'server_cert_path': '/path/to/server_cert',
+        'pod_id': 'test_pod',
+    }
+
+    opc_manager.return_value = MagicMock(connect=AsyncMock())
+    result = await ingestor_manager.initialize_opc_from_config(server_config)
+
+    opc_manager.assert_called_once_with(
+        name=server_config['name'],
+        url=server_config['url'],
+        data_manager=ingestor_manager.data_manager,
+        logger=ingestor_manager.logger,
+        subscription_period_ms=server_config['subscription_period_ms'],
+        server_uri=server_config['server_uri'],
+        notification_handler=ingestor_manager.notification_handler,
+        cert_path=server_config['cert_path'],
+        private_key_path=server_config['private_key_path'],
+        server_cert_path=server_config['server_cert_path'],
+        metadata=metadata['metadata'],
+        metrics_controller=ingestor_manager.metrics_controller,
+    )
+
+    assert result == opc_manager.return_value
+    result.connect.assert_called_once()
+
+
+@mark.asyncio
+@patch('ingestor.managers.ingestor_manager.OpcManager')
+@patch('ingestor.managers.ingestor_manager.traceback')
+async def test_initialize_opc_from_config_exception(traceback_mock, opc_manager, ingestor_manager):
+    server_config = {
+        'name': 'server1',
+        'url': 'opc.tcp://localhost:4840',
+        'subscription_period_ms': 1000,
+        'server_uri': 'http://opcua-server.simulator',
+        'cert_path': '/path/to/cert',
+        'private_key_path': '/path/to/private_key',
+        'server_cert_path': '/path/to/server_cert',
+    }
+
+    ingestor_manager.logger.error = MagicMock()
+    opc_manager.side_effect = Exception('Initialization error')
+
+    result = await ingestor_manager.initialize_opc_from_config(server_config)
+
+    assert result is None
+
+    traceback_mock.format_exc.assert_called_once()
+    ingestor_manager.send_notification.assert_called_once_with(
+        metadata=metadata['metadata'],
+        notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}',
+        message='Error initializing OPC manager: Initialization error',
+        block='opc_manager',
+        level=NotificationLevel.ERROR,
+        attachment_content=traceback_mock.format_exc.return_value,
+    )
+
+
+@mark.asyncio
+async def test_shutdown(ingestor_manager):
+    ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()}
+
+    ingestor_manager.data_manager.shutdown = MagicMock()
+
+    await ingestor_manager.shutdown()
+
+    ingestor_manager.opc_managers['server1'].shutdown.assert_called_once()
+    ingestor_manager.opc_managers['server2'].shutdown.assert_called_once()
+
+    ingestor_manager.data_manager.shutdown.assert_called_once()
+
+
+@patch('ingestor.managers.ingestor_manager.asyncio')
+def test___del__(asyncio_mock, ingestor_manager):
+    ingestor_manager.shutdown = MagicMock()
+    ingestor_manager.__del__()
+    asyncio_mock.run.assert_called_once_with(ingestor_manager.shutdown.return_value)
+
+
+@mark.asyncio
+async def test_remove_server(ingestor_manager):
+    server1 = AsyncMock()
+    server2 = AsyncMock()
+    ingestor_manager.opc_managers = {'server1': server1, 'server2': server2}
+    ingestor_manager.managed_tags = {
+        'slot1': {'server1': {'config': 'config1'}, 'server2': {'config': 'config2'}}
+    }
+    await ingestor_manager.remove_server('server1')
+    server1.shutdown.assert_called_once()
+    server2.shutdown.assert_not_called()
+    assert 'server1' not in ingestor_manager.opc_managers
+    assert 'server2' in ingestor_manager.opc_managers
+    assert ingestor_manager.managed_tags == {'slot1': {'server2': {'config': 'config2'}}}
+
+
+@mark.asyncio
+async def test_remove_server_not_found(ingestor_manager):
+    server1 = AsyncMock()
+    server2 = AsyncMock()
+    ingestor_manager.opc_managers = {'server1': server1, 'server2': server2}
+    ingestor_manager.managed_tags = {
+        'slot1': {'server1': {'config': 'config1'}, 'server2': {'config': 'config2'}}
+    }
+    await ingestor_manager.remove_server('server3')
+    server1.shutdown.assert_not_called()
+    server2.shutdown.assert_not_called()
+    assert 'server1' in ingestor_manager.opc_managers
+    assert 'server2' in ingestor_manager.opc_managers
+    assert ingestor_manager.managed_tags == {
+        'slot1': {'server1': {'config': 'config1'}, 'server2': {'config': 'config2'}}
+    }
+
+
+@mark.asyncio
+@patch('ingestor.managers.ingestor_manager.OpcManager')
+@patch('ingestor.managers.ingestor_manager.metrics')
+async def test_update_opc_servers(metrics, opc_manager, ingestor_manager):
+    manager1 = MagicMock(config={'config': 'config1'})
+    manager2 = MagicMock(config={'config': 'config2'})
+    manager3 = MagicMock(config={'config': 'config3'})
+
+    async def mock_initialize_from_config(config):
+        if config == {'config': 'config1'}:
+            return manager1
+        elif config == {'config': 'config2'}:
+            return manager2
+        elif config == {'config': 'config3'}:
+            return manager3
+        else:
+            return None
+
+    ingestor_manager.initialize_opc_from_config = AsyncMock(side_effect=mock_initialize_from_config)
+    ingestor_manager.remove_server = AsyncMock()
+
+    ingestor_manager.managed_tags = {
+        'slot1': {
+            'server1': {'config': 'config1'},
+            'server2': {'config': 'config2'},
+            'server5': {'config': 'config5'},
+        },
+        'slot2': {
+            'server3': {'config': 'config3'},
+            'server1': {'config': 'config1'},
+        },
+    }
+
+    mock = AsyncMock(config={'config': 'old_config2'})
+    ingestor_manager.opc_managers['server3'] = AsyncMock(config={'config': 'config3'})
+    ingestor_manager.opc_managers['server2'] = mock
+    ingestor_manager.opc_managers['server4'] = AsyncMock()
+
+    await ingestor_manager.update_opc_servers()
+
+    ingestor_manager.initialize_opc_from_config.assert_any_call({'config': 'config1'})
+    ingestor_manager.initialize_opc_from_config.assert_any_call({'config': 'config2'})
+    ingestor_manager.initialize_opc_from_config.assert_any_call({'config': 'config5'})
+
+    assert ingestor_manager.initialize_opc_from_config.call_count == 3
+
+    assert ingestor_manager.opc_managers['server1'].config == {'config': 'config1'}
+    assert ingestor_manager.opc_managers['server2'].config == {'config': 'config2'}
+    assert ingestor_manager.opc_managers['server3'].config == {'config': 'config3'}
+
+    ingestor_manager.remove_server.assert_any_call('server4')
+    ingestor_manager.remove_server.assert_any_call('server5')
+    assert ingestor_manager.remove_server.call_count == 2
+
+    assert ingestor_manager.opc_managers['server2'] != mock
+
+    ingestor_manager.emit_metric.assert_called_once_with(
+        metric_object=metrics.OPC_MANAGERS_ACTIVE,
+        method='set',
+        value=len(ingestor_manager.opc_managers),
+        tags={'pod_id': ingestor_manager.pod_id},
+    )
+
+
+@mark.asyncio
+@patch('ingestor.managers.ingestor_manager.metrics')
+async def test_check_opc_servers_integrity_all_healthy(metrics, ingestor_manager):
+    # Setup mock OPC managers
+    opc_manager1 = MagicMock()
+    opc_manager1.check_cycles = AsyncMock(return_value=None)
+    opc_manager1.check_opc_listenning = AsyncMock(return_value=False)
+    opc_manager1.config = {'config': 'config1'}
+
+    opc_manager2 = MagicMock()
+    opc_manager2.check_cycles = AsyncMock(return_value=None)
+    opc_manager2.check_opc_listenning = AsyncMock(return_value=False)
+    opc_manager2.config = {'config': 'config2'}
+
+    ingestor_manager.opc_managers = {'server1': opc_manager1, 'server2': opc_manager2}
+
+    # Mock the initialize_opc_from_config method
+    ingestor_manager.initialize_opc_from_config = MagicMock()
+
+    # Call the method
+    await ingestor_manager.check_opc_servers_integrity()
+
+    # Verify that check_cycles and check_opc_listenning were called for each server
+    opc_manager1.check_cycles.assert_called_once()
+    opc_manager1.check_opc_listenning.assert_called_once()
+    opc_manager2.check_cycles.assert_called_once()
+    opc_manager2.check_opc_listenning.assert_called_once()
+
+    # Verify that no reinitialization was needed
+    ingestor_manager.initialize_opc_from_config.assert_not_called()
+
+    ingestor_manager.emit_metric.assert_called_once_with(
+        metric_object=metrics.OPC_MANAGERS_ACTIVE,
+        method='set',
+        value=len(ingestor_manager.opc_managers),
+        tags={'pod_id': ingestor_manager.pod_id},
+    )
+
+
+@mark.asyncio
+async def test_check_opc_servers_integrity_server_lost(ingestor_manager):
+    # Setup mock OPC manager that will be lost
+    opc_manager = AsyncMock()
+    opc_manager.check_cycles.return_value = None
+    opc_manager.check_opc_listenning.return_value = True  # Server is lost
+    opc_manager.config = {'config': 'config1'}
+
+    ingestor_manager.opc_managers = {'server1': opc_manager}
+    ingestor_manager.managed_tags = {'slot1': MagicMock()}
+
+    # Call the method
+    await ingestor_manager.check_opc_servers_integrity()
+
+    assert 'server1' not in ingestor_manager.opc_managers
+    ingestor_manager.managed_tags['slot1'].pop.assert_called_once_with('server1', None)
+
+
+def test_check_opc_servers_integrity_server_lost_with_tags(ingestor_manager):
+    # Setup mock OPC manager that will be lost
+    opc_manager = MagicMock()
+    opc_manager.check_cycles.return_value = None
+    opc_manager.check_opc_listenning.return_value = True  # Server is lost
+    opc_manager.config = {'config': 'config1'}
+
+    ingestor_manager.opc_managers = {'server1': opc_manager}
+
+    # Setup managed tags
+    ingestor_manager.managed_tags = {
+        'slot1': {'server1': {'config': 'config1', 'tags': {'tag1': 'value1'}}}
+    }
+
+    # Mock the initialize_opc_from_config method to return a new manager
+    new_manager = MagicMock()
+    ingestor_manager.initialize_opc_from_config = MagicMock(return_value=new_manager)
+
+    # Call the method
+    ingestor_manager.check_opc_servers_integrity()
+
+
+@mark.asyncio
+async def test_declare_active(ingestor_manager):
+    ingestor_manager.resource_manager.ingestor_heartbeat = AsyncMock()
+    await ingestor_manager.declare_active()
+    ingestor_manager.resource_manager.ingestor_heartbeat.assert_called_once()
+
+
+@mark.asyncio
+async def test_get_active_ingestors(ingestor_manager):
+    ingestor_manager.resource_manager.get_all_ingestors = AsyncMock()
+    await ingestor_manager.get_active_ingestors()
+    ingestor_manager.resource_manager.get_all_ingestors.assert_called_once()
+
+
+@mark.asyncio
+async def test_get_active_ingestors_empty(ingestor_manager):
+    ingestor_manager.resource_manager.get_all_ingestors = AsyncMock(return_value=None)
+    result = await ingestor_manager.get_active_ingestors()
+    assert result == []
+    ingestor_manager.resource_manager.get_all_ingestors.assert_called_once()
+
+
+@patch('ingestor.managers.ingestor_manager.metrics')
+@mark.asyncio
+async def test_get_number_of_leases_success(metrics, ingestor_manager):
+    ingestor_manager.resource_manager.get_all_leases = AsyncMock(return_value=['lease1', 'lease2'])
+    result = await ingestor_manager.get_number_of_leases()
+    assert result == 2
+
+    ingestor_manager.emit_metric.assert_called_once_with(
+        metric_object=metrics.LEASES_TOTAL,
+        method='set',
+        value=2,
+        tags={'pod_id': ingestor_manager.pod_id},
+    )
+    ingestor_manager.resource_manager.get_all_leases.assert_called_once()
+
+
+@patch('ingestor.managers.ingestor_manager.metrics')
+@mark.asyncio
+async def test_get_number_of_slots_success(metrics, ingestor_manager):
+    ingestor_manager.resource_manager.get_all_slots = AsyncMock(return_value=['slot1', 'slot2'])
+    result = await ingestor_manager.get_number_of_slots()
+    assert result == 2
+    ingestor_manager.resource_manager.get_all_slots.assert_called_once()
+    ingestor_manager.emit_metric.assert_called_once_with(
+        metric_object=metrics.SLOTS_TOTAL,
+        method='set',
+        value=2,
+        tags={'pod_id': ingestor_manager.pod_id},
+    )
+
+
+@patch('ingestor.managers.ingestor_manager.metrics')
+@mark.asyncio
+async def test_get_number_of_slots_empty(metrics, ingestor_manager):
+    ingestor_manager.resource_manager.get_all_slots = AsyncMock(return_value=None)
+    result = await ingestor_manager.get_number_of_slots()
+    assert result == 0
+    ingestor_manager.resource_manager.get_all_slots.assert_called_once()
+    ingestor_manager.emit_metric.assert_called_once_with(
+        metric_object=metrics.SLOTS_TOTAL,
+        method='set',
+        value=0,
+        tags={'pod_id': ingestor_manager.pod_id},
+    )
+
+
+@mark.asyncio
+async def test_get_slot_leases_1_success(ingestor_manager):
+    ingestor_manager.resource_manager.lease_tag = AsyncMock(return_value=True)
+    ingestor_manager.resource_manager.get_tag_slot = AsyncMock(return_value={'tags': ['tag1']})
+
+    ingestor_manager.number_of_slots = 1
+    result = await ingestor_manager.get_slot_leases()
+
+    assert result == {'1': {'tags': ['tag1']}}
+
+
+@mark.asyncio
+async def test_get_slot_leases_2_success(ingestor_manager):
+    ingestor_manager.resource_manager.lease_tag = AsyncMock(side_effect=[True, True])
+    ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
+        side_effect=[{'tags': ['tag1']}, {'tags': ['tag2']}]
+    )
+
+    ingestor_manager.number_of_slots = 2
+    result = await ingestor_manager.get_slot_leases(max_slots=2)
+
+    assert result == {'1': {'tags': ['tag1']}, '2': {'tags': ['tag2']}}
+
+
+@mark.asyncio
+async def test_get_slot_leases_2_1_none(ingestor_manager):
+    ingestor_manager.resource_manager.lease_tag = AsyncMock(side_effect=[True, True])
+    ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
+        side_effect=[None, {'tags': ['tag1']}]
+    )
+
+    ingestor_manager.number_of_slots = 1
+    result = await ingestor_manager.get_slot_leases(max_slots=1)
+
+    assert result == {}
+
+
+@mark.asyncio
+async def test_get_slot_leases_1_failure(ingestor_manager):
+    ingestor_manager.resource_manager.lease_tag = AsyncMock(return_value=False)
+    ingestor_manager.resource_manager.get_tag_slot = AsyncMock(return_value={'tags': ['tag1']})
+
+    result = await ingestor_manager.get_slot_leases()
+    ingestor_manager.resource_manager.get_tag_slot.assert_not_called()
+
+    assert result == {}
+
+
+@mark.asyncio
+async def test_unsubscribe_slot(ingestor_manager):
+    ingestor_manager.managed_tags = {
+        'slot1': {'server1': {'tags': 'config1'}, 'server2': {'tags': 'config2'}},
+        'slot2': {'server3': {'tags': 'config3'}, 'server1': {'tags': 'config1'}},
+    }
+    ingestor_manager.opc_managers = {
+        'server1': AsyncMock(),
+        'server2': AsyncMock(),
+        'server3': AsyncMock(),
+    }
+    await ingestor_manager.unsubscribe_slot('slot1')
+
+    ingestor_manager.opc_managers['server1'].unsubscribe.assert_called_once_with('slot1')
+    ingestor_manager.opc_managers['server2'].unsubscribe.assert_called_once_with('slot1')
+    ingestor_manager.opc_managers['server3'].unsubscribe.assert_not_called()
+
+
+@mark.asyncio
+async def test_update_slot_config(ingestor_manager):
+    ingestor_manager.managed_tags = {
+        'slot1': {'config': 'old_config'},
+        'slot2': {'config': 'new_config'},
+        'slot3': {'config': 'old_config'},
+    }
+
+    ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
+        side_effect=[{'config': 'updated_config'}, {'config': 'new_config'}, None]
+    )
+    ingestor_manager.resource_manager.renew_tag_lease = AsyncMock()
+
+    await ingestor_manager.update_slot_config()
+
+    assert ingestor_manager.managed_tags['slot1'] == {'config': 'updated_config'}
+    assert ingestor_manager.managed_tags['slot2'] == {'config': 'new_config'}
+    assert 'slot3' not in ingestor_manager.managed_tags
+
+    ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot1')
+    ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot3')
+    ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot2')
+    assert ingestor_manager.resource_manager.renew_tag_lease.call_count == 3
+
+
+@patch('ingestor.managers.ingestor_manager.metrics')
+@mark.asyncio
+async def test_drop_slot_leases(metrics, ingestor_manager):
+    ingestor_manager.resource_manager.drop_tag_lease = AsyncMock()
+    await ingestor_manager.drop_slot_leases(['1', '2'])
+
+    ingestor_manager.resource_manager.drop_tag_lease.assert_any_call('1')
+    ingestor_manager.resource_manager.drop_tag_lease.assert_any_call('2')
+
+    ingestor_manager.emit_metric.assert_has_calls(
+        [
+            call(
+                metric_object=metrics.SLOTS_RELEASED,
+                method='inc',
+                value=1,
+                tags={'pod_id': ingestor_manager.pod_id},
+            ),
+            call(
+                metric_object=metrics.SLOTS_RELEASED,
+                method='inc',
+                value=1,
+                tags={'pod_id': ingestor_manager.pod_id},
+            ),
+        ]
+    )
+
+
+@mark.asyncio
+async def test_manage_server_no_server(ingestor_manager):
+    ingestor_manager.opc_managers = {'server1': MagicMock(), 'server2': MagicMock()}
+    server_config = {'tags': 'config1'}
+
+    result = await ingestor_manager.manage_server('slot1', 'server3', server_config, server_config)
+
+    assert result == 1
+    ingestor_manager.opc_managers['server1'].create_subscription.assert_not_called()
+    ingestor_manager.opc_managers['server1'].subscribe.assert_not_called()
+
+
+@mark.asyncio
+async def test_manage_server_create_subscription_failure(ingestor_manager):
+    ingestor_manager.opc_managers = {'server1': MagicMock(), 'server2': MagicMock()}
+    ingestor_manager.subscriptions = {'server1': MagicMock()}
+    server_config = {'tags': 'config1'}
+
+    ingestor_manager.opc_managers['server1'].create_subscription.side_effect = Exception(
+        'Subscription error'
+    )
+
+    result = await ingestor_manager.manage_server('slot1', 'server1', server_config, server_config)
+
+    assert result == 2
+    ingestor_manager.opc_managers['server1'].create_subscription.assert_called_once_with('slot1')
+    ingestor_manager.opc_managers['server1'].subscribe.assert_not_called()
+
+
+@mark.asyncio
+async def test_manage_server(ingestor_manager):
+    ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()}
+    ingestor_manager.subscriptions = {'server1': AsyncMock()}
+    server_config = {'tags': 'config1'}
+
+    result = await ingestor_manager.manage_server('slot1', 'server1', server_config, server_config)
+
+    assert result == 0
+    ingestor_manager.opc_managers['server1'].create_subscription.assert_called_once_with('slot1')
+    ingestor_manager.opc_managers['server1'].subscribe.assert_called_once_with(
+        'slot1', 'config1', ingestor_manager.poll_interval
+    )
+
+
+@patch('ingestor.managers.ingestor_manager.traceback')
+@mark.asyncio
+async def test_manage_server_subscribe_failure(traceback_mock, ingestor_manager):
+    ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()}
+    ingestor_manager.subscriptions = {'server1': AsyncMock()}
+    server_config = {'tags': 'config1'}
+
+    ingestor_manager.opc_managers['server1'].subscribe.side_effect = Exception('Subscription error')
+
+    result = await ingestor_manager.manage_server('slot1', 'server1', server_config, server_config)
+
+    assert result == 2
+    ingestor_manager.opc_managers['server1'].create_subscription.assert_called_once_with('slot1')
+    ingestor_manager.opc_managers['server1'].subscribe.assert_called_once_with(
+        'slot1', 'config1', ingestor_manager.poll_interval
+    )
+    ingestor_manager.opc_managers['server1'].unsubscribe.assert_called_once_with('slot1')
+
+    traceback_mock.format_exc.assert_called_once()
+
+    ingestor_manager.send_notification_async.assert_called_once_with(
+        metadata=metadata['metadata'],
+        notification_id='OPC_SUBSCRIPTION_ERROR_slot1:server1',
+        message="Failed to subscribe to tags from slot1:server1\n{'tags': 'config1'}: Subscription error",
+        block='opc_manager',
+        level=NotificationLevel.ERROR,
+        attachment_content=traceback_mock.format_exc.return_value,
+    )
+
+    ingestor_manager.logger.warning.assert_any_call(
+        'Removing subscription from server server1 for slot slot1'
+    )
+
+
+@mark.asyncio
+async def test_subscribe_to_tags(ingestor_manager):
+    ingestor_manager.manage_server = AsyncMock(side_effect=[0, 1, 2])
+    ingestor_manager.managed_tags = {'slot1': MagicMock(), 'slot2': MagicMock()}
+
+    ingestor_manager.opc_managers = {
+        'server1': AsyncMock(),
+        'server2': AsyncMock(),
+        'server3': AsyncMock(),
+    }
+    ingestor_manager.subscriptions = {'server1': AsyncMock()}
+    tags = {
+        'slot1': {
+            'server1': {'tags': 'config1'},
+            'server2': {'tags': 'config2'},
+            'server3': {'tags': 'config3'},
+        }
+    }
+
+    await ingestor_manager.subscribe_to_tags(tags)
+
+    ingestor_manager.manage_server.assert_any_call('slot1', 'server1', {'tags': 'config1'}, tags)
+    ingestor_manager.manage_server.assert_any_call('slot1', 'server2', {'tags': 'config2'}, tags)
+    ingestor_manager.manage_server.assert_any_call('slot1', 'server3', {'tags': 'config3'}, tags)
+
+    assert ingestor_manager.manage_server.call_count == 3
+
+    ingestor_manager.managed_tags['slot1'].pop.assert_called_once_with('server3', None)

tests/unit/managers/test_opc_manager.py

@@ -0,0 +1,627 @@
+import json
+from datetime import datetime
+from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
+
+import pytest
+from asyncua.crypto.security_policies import SecurityPolicyBasic256
+from pytest import fixture, mark
+from sientia_do.notifications.models import NotificationLevel
+
+from ingestor.managers.opc_manager import OpcManager
+
+tags = {
+    'ns=3;i=1001': {
+        'aggregation_function': 'LTS',
+        'frequency': 1000,
+        'max_value': 100,
+        'min_value': 0,
+        'tag_name': 'Counter',
+    },
+    'ns=3;i=1003': {
+        'aggregation_function': 'AVG',
+        'frequency': 1000,
+        'max_value': 100,
+        'min_value': 0,
+        'tag_name': 'Random',
+    },
+    'ns=3;i=1004': {
+        'aggregation_function': 'MDN',
+        'frequency': 1000,
+        'max_value': 100,
+        'min_value': 0,
+        'tag_name': 'Sawtooth',
+    },
+}
+
+metadata = {
+    'metadata': {
+        'model_id': 'test_model',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+        'schema_name': 'test_schedule',
+    },
+}
+
+
+@fixture
+@patch('ingestor.managers.opc_manager.metrics')
+def raw_opc_manager(mock_metrics):
+    opc_manager = OpcManager(
+        name='TestConnector',
+        url='opc.tcp://localhost:4840',
+        data_manager=AsyncMock(),
+        subscription_period_ms=1000,
+        logger=MagicMock(),
+        server_uri='opc.tcp://localhost:4840',
+        notification_handler=MagicMock(),
+        metadata=metadata['metadata'],
+        metrics_controller=AsyncMock(),
+    )
+
+    opc_manager.emit_metric = AsyncMock()
+    opc_manager.send_notification_async = AsyncMock()
+    opc_manager.send_notification = MagicMock()
+
+    return opc_manager
+
+
+@fixture
+def opc_manager(raw_opc_manager):
+    raw_opc_manager.client = AsyncMock()
+    raw_opc_manager.cert_path = 'cert.pem'
+    raw_opc_manager.private_key_path = 'private_key.pem'
+    raw_opc_manager.server_cert_path = 'server_cert.pem'
+
+    return raw_opc_manager
+
+
+@fixture
+def opc_manager_subscribed(opc_manager):
+    opc_manager.subscriptions['sub1'] = AsyncMock()
+
+    return opc_manager
+
+
+def test___str__(opc_manager):
+    assert (
+        str(opc_manager)
+        == 'OpcManager(name=TestConnector, url=opc.tcp://localhost:4840, server_uri=opc.tcp://localhost:4840)\nnodes={}, subscriptions={}'
+    )
+
+
+@mark.asyncio
+async def test_shutdown_success(opc_manager):
+    opc_manager.disconnect = AsyncMock()
+
+    await opc_manager.shutdown()
+
+    opc_manager.disconnect.assert_called_once()
+
+
+@mark.asyncio
+async def test_shutdown_error(opc_manager):
+    opc_manager.disconnect = AsyncMock(side_effect=Exception('Test error'))
+
+    await opc_manager.shutdown()
+
+    opc_manager.logger.error.assert_called_once_with('Error during cleanup: Test error')
+
+
+@mark.asyncio
+async def test_set_security_success(opc_manager):
+    await opc_manager.set_security()
+
+    assert opc_manager.client.application_uri == opc_manager.server_uri
+
+    opc_manager.client.set_security.assert_called_once_with(
+        SecurityPolicyBasic256,
+        certificate=opc_manager.cert_path,
+        private_key=opc_manager.private_key_path,
+        server_certificate=opc_manager.server_cert_path,
+    )
+
+    assert opc_manager.client.secure_channel_timeout == 10000000
+    assert opc_manager.client.session_timeout == 10000000
+
+
+@mark.asyncio
+async def test_set_security_no_cert(opc_manager):
+    opc_manager.cert_path = None
+    opc_manager.private_key_path = None
+
+    try:
+        await opc_manager.set_security()
+    except ValueError as e:
+        assert str(e) == 'Certificate and private key paths must be provided for secure connection.'
+    else:
+        raise AssertionError('ValueError not raised')
+
+    assert opc_manager.client.set_security.call_count == 0
+
+
+@mark.asyncio
+@patch('ingestor.managers.opc_manager.metrics')
+@patch('ingestor.managers.opc_manager.Client')
+async def test_connect_no_security(client, mock_metrics, raw_opc_manager):
+    raw_opc_manager.set_security = AsyncMock()
+    client.return_value = AsyncMock()
+
+    await raw_opc_manager.connect()
+
+    client.assert_called_once_with(raw_opc_manager.url, timeout=10, watchdog_intervall=3600000)
+    raw_opc_manager.client.connect.assert_called_once()
+    raw_opc_manager.set_security.assert_not_called()
+    raw_opc_manager.emit_metric.assert_has_calls(
+        [
+            call(
+                metric_object=mock_metrics.OPC_CONNECTIONS_TOTAL,
+                tags={
+                    'pod_id': raw_opc_manager.pod_id,
+                    'server_name': raw_opc_manager.name,
+                },
+            ),
+            call(
+                metric_object=mock_metrics.OPC_CONNECTION_STATUS,
+                method='set',
+                value=1,
+                tags={
+                    'pod_id': raw_opc_manager.pod_id,
+                    'server_name': raw_opc_manager.name,
+                    'server_url': raw_opc_manager.url,
+                },
+            ),
+        ],
+        any_order=True,
+    )
+
+
+@mark.asyncio
+@patch('ingestor.managers.opc_manager.Client')
+async def test_connect_with_security(client, raw_opc_manager):
+    raw_opc_manager.cert_path = 'cert.pem'
+    raw_opc_manager.private_key_path = 'private_key.pem'
+    raw_opc_manager.server_cert_path = 'server_cert.pem'
+    raw_opc_manager.set_security = AsyncMock()
+    client.return_value = AsyncMock()
+
+    await raw_opc_manager.connect()
+
+    client.assert_called_once_with(raw_opc_manager.url, timeout=10, watchdog_intervall=3600000)
+    raw_opc_manager.client.connect.assert_called_once()
+    raw_opc_manager.set_security.assert_called_once()
+
+
+@mark.asyncio
+@patch('ingestor.managers.opc_manager.Client')
+@patch('ingestor.managers.opc_manager.metrics')
+async def test_connect_exception_handling_and_metrics(
+    mock_metrics_module, mock_opc_client_class, raw_opc_manager
+):
+    mock_client_instance = mock_opc_client_class.return_value
+    simulated_error_message = 'Erro de conexão simulado'
+    mock_client_instance.connect.side_effect = Exception(simulated_error_message)
+
+    raw_opc_manager.disconnect = AsyncMock()
+
+    opc_manager_instance = raw_opc_manager
+    opc_manager_instance.cert_path = None
+
+    with pytest.raises(Exception, match=simulated_error_message):
+        await opc_manager_instance.connect()
+
+    opc_manager_instance.disconnect.assert_called_once()
+
+
+@mark.asyncio
+async def test_create_subscription_no_client(raw_opc_manager):
+    try:
+        await raw_opc_manager.create_subscription('sub1')
+    except ValueError as e:
+        assert str(e) == 'Client not connected. Call connect first.'
+    else:
+        raise AssertionError('ValueError not raised')
+
+
+@mark.asyncio
+async def test_create_subscription_success_has_period(opc_manager):
+    await opc_manager.create_subscription('sub1')
+
+    opc_manager.client.create_subscription.assert_called_once_with(
+        opc_manager.subscription_period_ms, opc_manager
+    )
+    assert opc_manager.subscriptions['sub1'] is not None
+
+
+@mark.asyncio
+async def test_create_subscription_success_no_period(opc_manager):
+    await opc_manager.create_subscription('sub1')
+
+    opc_manager.client.create_subscription.assert_called_once_with(
+        opc_manager.subscription_period_ms, opc_manager
+    )
+    assert opc_manager.subscriptions['sub1'] is not None
+
+
+@patch('ingestor.managers.opc_manager.metrics')
+@mark.asyncio
+async def test_create_subscription_with_metrics(metrics, opc_manager):
+    await opc_manager.create_subscription('sub1')
+
+    opc_manager.emit_metric.assert_called_once_with(
+        metric_object=metrics.OPC_SUBSCRIPTIONS_CREATED,
+        method='inc',
+        value=1,
+        tags={
+            'pod_id': opc_manager.pod_id,
+            'server_name': opc_manager.name,
+            'slot_name': 'sub1',
+        },
+    )
+
+
+@patch('ingestor.managers.opc_manager.metrics')
+@mark.asyncio
+async def test_create_subscription_exception_during_client_call(
+    mock_metrics_module, raw_opc_manager
+):
+    opc_manager_instance = raw_opc_manager
+    opc_manager_instance.client = AsyncMock()
+
+    subscription_name = 'test_sub_client_error'
+    simulated_error_message = 'Falha ao criar subscrição no cliente OPC'
+
+    opc_manager_instance.client.create_subscription.side_effect = Exception(simulated_error_message)
+
+    with pytest.raises(Exception, match=simulated_error_message):
+        await opc_manager_instance.create_subscription(subscription_name)
+
+    opc_manager_instance.client.create_subscription.assert_called_once_with(
+        opc_manager_instance.subscription_period_ms, opc_manager_instance
+    )
+
+    opc_manager_instance.logger.error.assert_called_once_with(
+        f'Failed to create subscription {subscription_name} on {opc_manager_instance.name}: {simulated_error_message}'
+    )
+
+    mock_metrics_module.OPC_SUBSCRIPTIONS_CREATED.labels.assert_not_called()
+
+
+@patch('ingestor.managers.opc_manager.metrics')
+@mark.asyncio
+async def test_subscribe_no_subscription(metrics, opc_manager):
+    try:
+        await opc_manager.subscribe('sub1', tags, 1000)
+    except ValueError as e:
+        assert str(e) == 'Subscription not created. Call create_subscription first.'
+    else:
+        raise AssertionError('ValueError not raised')
+    metrics.OPC_TAGS_SUBSCRIBED.labels.assert_not_called()
+
+
+@mark.asyncio
+async def test_subscribe_success(opc_manager_subscribed):
+    opc_manager_subscribed.client.get_node = MagicMock()
+    opc_manager_subscribed.nodes = {'ns=3;i=1001': 'data'}
+
+    await opc_manager_subscribed.subscribe('sub1', tags, 1000)
+
+    assert opc_manager_subscribed.nodes == tags
+    opc_manager_subscribed.subscriptions['sub1'].subscribe_data_change.assert_called_once_with(
+        [opc_manager_subscribed.client.get_node(n) for n in tags]
+    )
+
+
+@mark.asyncio
+async def test_unsubscribe_no_subscription(opc_manager):
+    await opc_manager.unsubscribe('sub1')
+
+    opc_manager.logger.warning.assert_called_once_with(
+        "Subscription 'sub1' not found. Cannot unsubscribe."
+    )
+    assert opc_manager.subscriptions.get('sub1') is None
+
+
+@mark.asyncio
+async def test_unsubscribe_success(opc_manager_subscribed):
+    await opc_manager_subscribed.unsubscribe('sub1')
+
+    assert opc_manager_subscribed.subscriptions.get('sub1') is None
+
+
+@mark.asyncio
+async def test_disconnection_fallback_success(opc_manager):
+    opc_manager.client = AsyncMock()
+    opc_manager.client.disconnect.return_value = True
+    result = await opc_manager.disconnection_fallback()
+    assert result == []
+
+
+@mark.asyncio
+async def test_disconnection_fallback_fail(opc_manager):
+    opc_manager.client = AsyncMock()
+    opc_manager.client.disconnect.side_effect = Exception('Test error')
+    result = await opc_manager.disconnection_fallback()
+    assert result == [
+        {'attempt': 1, 'error': 'Test error', 'traceback': ANY},
+        {'attempt': 2, 'error': 'Test error', 'traceback': ANY},
+        {'attempt': 3, 'error': 'Test error', 'traceback': ANY},
+        {'attempt': 4, 'error': 'Test error', 'traceback': ANY},
+        {'attempt': 5, 'error': 'Test error', 'traceback': ANY},
+    ]
+    assert opc_manager.client.disconnect.call_count == 5
+
+
+@mark.asyncio
+async def test_disconnect_success(opc_manager_subscribed):
+    opc_manager_subscribed.client = MagicMock()
+
+    await opc_manager_subscribed.disconnect()
+
+    opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
+    assert opc_manager_subscribed.client is None
+
+
+@mark.asyncio
+async def test_disconnect_no_client(opc_manager_subscribed):
+    opc_manager_subscribed.client = None
+    assert await opc_manager_subscribed.disconnect() is None
+
+    opc_manager_subscribed.logger.warning.assert_has_calls(
+        [
+            call('Client already disconnected.'),
+        ]
+    )
+
+
+@mark.asyncio
+async def test_disconnect_error_unsubscribe(opc_manager_subscribed):
+    opc_manager_subscribed.client = MagicMock(disconnect=AsyncMock())
+    opc_manager_subscribed.subscriptions['sub1'] = MagicMock(
+        delete=AsyncMock(side_effect=Exception('Test error'))
+    )
+
+    await opc_manager_subscribed.disconnect()
+
+    opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
+    opc_manager_subscribed.client = None
+    opc_manager_subscribed.logger.error.assert_called_once_with(
+        'Failed to clean up subscription: Test error'
+    )
+
+
+@mark.asyncio
+async def test_disconnect_error(opc_manager_subscribed):
+    opc_manager_subscribed.client = MagicMock()
+    opc_manager_subscribed.client.disconnect = MagicMock(side_effect=Exception('Test error'))
+
+    await opc_manager_subscribed.disconnect()
+
+    opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
+    opc_manager_subscribed.client = None
+
+
+@patch('ingestor.managers.opc_manager.metrics')
+@mark.asyncio
+async def test_disconnect_metrics_on_successful_path(mock_metrics_module, raw_opc_manager):
+    mock_metrics_module.OPC_CONNECTION_STATUS.reset_mock()
+    mock_metrics_module.OPC_TAGS_SUBSCRIBED.reset_mock()
+
+    raw_opc_manager.client = MagicMock()
+    mock_sub1 = MagicMock()
+    mock_sub2 = MagicMock()
+    raw_opc_manager.subscriptions = {'sub1': mock_sub1, 'sub2': mock_sub2}
+
+    await raw_opc_manager.disconnect()
+
+    raw_opc_manager.emit_metric.assert_has_calls(
+        [
+            call(
+                metric_object=mock_metrics_module.OPC_CONNECTION_STATUS,
+                method='set',
+                value=0,
+                tags={
+                    'pod_id': raw_opc_manager.pod_id,
+                    'server_name': raw_opc_manager.name,
+                    'server_url': raw_opc_manager.url,
+                },
+            ),
+            call(
+                metric_object=mock_metrics_module.OPC_TAGS_SUBSCRIBED,
+                method='set',
+                value=0,
+                tags={
+                    'pod_id': raw_opc_manager.pod_id,
+                    'server_name': raw_opc_manager.name,
+                },
+            ),
+        ],
+        any_order=True,
+    )
+
+
+@patch('ingestor.managers.opc_manager.metrics')
+@mark.asyncio
+async def test_datachange_notification(metrics, opc_manager_subscribed):
+    data = MagicMock(
+        monitored_item=MagicMock(
+            Value=MagicMock(
+                Value=MagicMock(Value=42),
+                SourceTimestamp=datetime.strptime('2021-01-01T00:00:00', '%Y-%m-%dT%H:%M:%S'),
+            )
+        )
+    )
+    opc_manager_subscribed.nodes = {
+        'ns=3;i=1001': {
+            'tag_name': 'Counter',
+            'cycle_rule': {'cycle_increment': 1.0, 'cycle_count': 2},
+            'topics': ['topic1', 'topic2'],
+        }
+    }
+
+    await opc_manager_subscribed.datachange_notification('ns=3;i=1001', None, data)
+
+    opc_manager_subscribed.data_manager.publish.assert_any_call(
+        'topic1',
+        {
+            'tag': 'ns=3;i=1001',
+            'name': 'Counter',
+            'timestamp': '2021-01-01 00:00:00-0300',
+            'value': 42,
+        },
+    )
+    opc_manager_subscribed.data_manager.publish.assert_any_call(
+        'topic2',
+        {
+            'tag': 'ns=3;i=1001',
+            'name': 'Counter',
+            'timestamp': '2021-01-01 00:00:00-0300',
+            'value': 42,
+        },
+    )
+    assert opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == 0
+
+    opc_manager_subscribed.emit_metric.assert_called_once_with(
+        metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
+        method='set',
+        value=0,
+        tags={
+            'pod_id': opc_manager_subscribed.pod_id,
+            'server_name': opc_manager_subscribed.name,
+        },
+    )
+
+
+@mark.asyncio
+async def test_check_cycles_no_notification(opc_manager):
+    # Setup: node with cycle_count just below threshold
+    opc_manager.nodes = {
+        'ns=3;i=1001': {
+            'tag_name': 'Counter',
+            'cycle_rule': {'cycle_increment': 1.0, 'cycle_count': 3.0},
+        }
+    }
+
+    await opc_manager.check_cycles()
+
+    # After one increment, cycle_count = 4.0, still below threshold
+    assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(4.0)
+    opc_manager.send_notification_async.assert_not_called()
+
+
+@mark.asyncio
+async def test_check_cycles_triggers_notification(opc_manager):
+    # Setup: node with cycle_count just below threshold, increment will cross threshold
+    opc_manager.nodes = {
+        'ns=3;i=1001': {
+            'tag_name': 'Counter',
+            'cycle_rule': {'cycle_increment': 2.5, 'cycle_count': 3.0},
+        }
+    }
+    opc_manager.notification_handler.build_and_send_notification = MagicMock()
+
+    await opc_manager.check_cycles()
+
+    # After increment, cycle_count = 5.5, should trigger notification
+    assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(5.5)
+    opc_manager.send_notification_async.assert_called_once_with(
+        notification_id='TAG_ns=3;i=1001:Counter_LISTENNING_STOPPED',
+        message='5.5 cycles without receive from ns=3;i=1001:Counter',
+        block='opc_manager',
+        level=NotificationLevel.WARNING,
+        metadata=metadata['metadata'],
+    )
+
+
+@patch('ingestor.managers.opc_manager.metrics')
+@mark.asyncio
+async def test_check_opc_listenning_no_notification(metrics, opc_manager):
+    opc_manager.non_receive_count = 3
+
+    result = await opc_manager.check_opc_listenning()
+
+    assert opc_manager.non_receive_count == 4
+    opc_manager.send_notification_async.assert_not_called()
+    assert result is False
+
+    opc_manager.emit_metric.assert_called_once_with(
+        metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
+        method='set',
+        value=opc_manager.non_receive_count,
+        tags={
+            'pod_id': opc_manager.pod_id,
+            'server_name': opc_manager.name,
+        },
+    )
+
+
+@mark.asyncio
+async def test_check_opc_listenning_warning_notification(opc_manager):
+    opc_manager.non_receive_count = 4
+
+    result = await opc_manager.check_opc_listenning()
+
+    assert opc_manager.non_receive_count == 5
+    opc_manager.send_notification_async.assert_called_once_with(
+        notification_id=f'OPC_LISTENNING_STOPPED__{opc_manager.name}',
+        message=f'5 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}',
+        block='opc_manager',
+        level=NotificationLevel.ERROR,
+        metadata=metadata['metadata'],
+    )
+    assert result is False
+
+
+@patch('ingestor.managers.opc_manager.metrics')
+@mark.asyncio
+async def test_check_opc_listenning_error_notification_and_retry(metrics, opc_manager):
+    opc_manager.non_receive_count = 14
+
+    result = await opc_manager.check_opc_listenning()
+
+    assert opc_manager.non_receive_count == 15
+    # Should be called twice: once for 5, once for 15
+    assert opc_manager.send_notification_async.call_count == 2
+    calls = opc_manager.send_notification_async.call_args_list
+    # First call: 5 cycles warning
+    assert calls[0].kwargs == {
+        'notification_id': f'OPC_LISTENNING_STOPPED__{opc_manager.name}',
+        'message': f'15 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}',
+        'block': 'opc_manager',
+        'level': NotificationLevel.ERROR,
+        'metadata': metadata['metadata'],
+    }
+    # Second call: 15 cycles retry
+    assert calls[1].kwargs == {
+        'notification_id': f'OPC_CONNECTION_RETRY__{opc_manager.name}',
+        'message': f'Retrying to connect to server {opc_manager.name}',
+        'block': 'opc_manager',
+        'level': NotificationLevel.ERROR,
+        'metadata': metadata['metadata'],
+    }
+    assert result is True
+
+    opc_manager.emit_metric.assert_has_calls(
+        [
+            call(
+                metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
+                method='set',
+                value=opc_manager.non_receive_count,
+                tags={
+                    'pod_id': opc_manager.pod_id,
+                    'server_name': opc_manager.name,
+                },
+            ),
+        ]
+    )
+
+    opc_manager.emit_metric.assert_has_calls(
+        [
+            call(
+                metric_object=metrics.OPC_RECONNECTIONS_TOTAL,
+                tags={
+                    'pod_id': opc_manager.pod_id,
+                    'server_name': opc_manager.name,
+                },
+            ),
+        ]
+    )

tests/unit/managers/test_resource_manager.py

@@ -0,0 +1,178 @@
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from pytest import fixture, mark, raises
+
+from ingestor.managers.resource_manager import ResourceManager
+
+metadata = {
+    'metadata': {
+        'model_id': 'test_model',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+        'schema_name': 'test_schedule',
+    },
+}
+
+
+@patch('ingestor.managers.resource_manager.RedisRepository')
+def test___init__(redis_repository):
+    logger = MagicMock()
+    notification_handler = MagicMock()
+    metrics_controller = MagicMock()
+    resource_manager = ResourceManager(
+        host='localhost',
+        port=6379,
+        lease_ttl=10,
+        heartbeat_ttl=10,
+        metadata=metadata['metadata'],
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+    )
+    assert resource_manager.redis_repository == redis_repository.return_value
+    redis_repository.assert_called_once_with(
+        host='localhost',
+        port=6379,
+        username=None,
+        password=None,
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+    )
+    redis_repository.return_value.redis_client.ping.assert_called_once()
+
+
+@patch('ingestor.managers.resource_manager.RedisRepository')
+def test___init__connection_failure(redis_repository):
+    redis_repository.return_value.redis_client.ping.side_effect = Exception('Connection failed')
+    with raises(Exception, match='Connection failed'):
+        ResourceManager(
+            host='localhost',
+            port=6379,
+            lease_ttl=10,
+            heartbeat_ttl=10,
+            metadata=metadata['metadata'],
+            logger=MagicMock(),
+            notification_handler=MagicMock(),
+            metrics_controller=MagicMock(),
+        )
+        redis_repository.return_value.redis_client.ping.assert_called_once()
+        redis_repository.logger.error.assert_called_once_with(
+            'Failed to connect to Redis: Connection failed'
+        )
+        redis_repository.return_value.redis_client.ping.assert_called_once()
+
+
+@fixture
+@patch('ingestor.managers.resource_manager.RedisRepository')
+def resource_manager(redis_repository):
+    resource_manager = ResourceManager(
+        host='localhost',
+        port=6379,
+        lease_ttl=10,
+        heartbeat_ttl=10,
+        metadata=metadata['metadata'],
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=MagicMock(),
+    )
+
+    resource_manager.send_notification = MagicMock()
+    resource_manager.send_notification_async = AsyncMock()
+    resource_manager.emit_metric = AsyncMock()
+
+    resource_manager.redis_repository = AsyncMock()
+
+    return resource_manager
+
+
+@mark.asyncio
+async def test_get_tag_slot(resource_manager):
+    result = await resource_manager.get_tag_slot('id')
+    assert result == resource_manager.redis_repository.get.return_value
+
+    resource_manager.redis_repository.get.assert_called_once_with(
+        'slot:opc_tags:id', metadata=metadata['metadata']
+    )
+
+
+@mark.asyncio
+async def test_ingestor_heartbeat(resource_manager):
+    await resource_manager.ingestor_heartbeat()
+    resource_manager.redis_repository.set.assert_called_once_with(
+        'heartbeat:ingestor:localhost', 1, ttl=10, metadata=metadata['metadata']
+    )
+
+
+@mark.asyncio
+async def test_lease_tag(resource_manager):
+    output = await resource_manager.lease_tag('tag_id')
+    assert output is resource_manager.redis_repository.set.return_value
+    resource_manager.redis_repository.set.assert_called_once_with(
+        'lease:opc_tags:tag_id', 'localhost', ttl=10, nx=True, metadata=metadata['metadata']
+    )
+
+
+@mark.asyncio
+async def test_renew_tag_lease_success(resource_manager):
+    resource_manager.redis_repository.get.return_value = 'localhost'
+    resource_manager.redis_repository.expire.return_value = True
+    result = await resource_manager.renew_tag_lease('tag_id')
+    assert result is resource_manager.redis_repository.expire.return_value
+    resource_manager.redis_repository.get.assert_called_once_with(
+        'lease:opc_tags:tag_id', metadata=metadata['metadata']
+    )
+    resource_manager.redis_repository.expire.assert_called_once_with(
+        'lease:opc_tags:tag_id', 10, metadata=metadata['metadata']
+    )
+
+
+@mark.asyncio
+async def test_renew_tag_lease_failure(resource_manager):
+    resource_manager.redis_repository.get.return_value = 'other_pod_id'
+
+    result = await resource_manager.renew_tag_lease('tag_id')
+    assert result is False
+
+    resource_manager.redis_repository.get.assert_called_once_with(
+        'lease:opc_tags:tag_id', metadata=metadata['metadata']
+    )
+    resource_manager.redis_repository.expire.assert_not_called()
+
+
+@mark.asyncio
+async def test_drop_tag_lease(resource_manager):
+    await resource_manager.drop_tag_lease('tag_id')
+    resource_manager.redis_repository.delete.assert_called_once_with(
+        'lease:opc_tags:tag_id', metadata=metadata['metadata']
+    )
+
+
+@mark.asyncio
+async def test_get_all_ingestors(resource_manager):
+    resource_manager.redis_repository.keys.return_value = ['ingestor1', 'ingestor2']
+    result = await resource_manager.get_all_ingestors()
+    assert result == ['ingestor1', 'ingestor2']
+    resource_manager.redis_repository.keys.assert_called_once_with(
+        'heartbeat:ingestor:*', metadata=metadata['metadata']
+    )
+
+
+@mark.asyncio
+async def test_get_all_slots(resource_manager):
+    resource_manager.redis_repository.keys.return_value = ['slot1', 'slot2']
+    result = await resource_manager.get_all_slots()
+    assert result == ['slot1', 'slot2']
+    resource_manager.redis_repository.keys.assert_called_once_with(
+        'slot:opc_tags:*', metadata=metadata['metadata']
+    )
+
+
+@mark.asyncio
+async def test_get_all_leases(resource_manager):
+    resource_manager.redis_repository.keys.return_value = ['lease1', 'lease2']
+    result = await resource_manager.get_all_leases()
+    assert result == ['lease1', 'lease2']
+    resource_manager.redis_repository.keys.assert_called_once_with(
+        'lease:opc_tags:*', metadata=metadata['metadata']
+    )

tests/unit/test_app.py

@@ -0,0 +1,319 @@
+import asyncio
+import signal as signal_module  # To avoid conflict with mock names
+from threading import Event
+from unittest.mock import AsyncMock, MagicMock, call
+
+import pytest
+
+# Import the 'app' module to be tested
+from ingestor import app
+
+
+# Custom exception to catch os._exit calls
+class OsExitCalledError(Exception):
+    def __init__(self, code):
+        super().__init__(f'os._exit({code}) called')
+        self.code = code
+
+
+# Helper function for the os_exit mock's side_effect
+def raise_os_exit_with_code(exit_code):
+    raise OsExitCalledError(exit_code)
+
+
+@pytest.fixture
+def mock_app_env(monkeypatch):
+    """Fixture to mock dependencies of app.main and app.signal_handler."""
+    mocks = {
+        'start_http_server': MagicMock(),
+        'Ingestor': MagicMock(),
+        'metrics_APP_UP_labels_set': MagicMock(),
+        'metrics_APP_LOOP_COUNT_labels_inc': MagicMock(),
+        'metrics_APP_LOOP_DURATION_labels_observe': MagicMock(),
+        'metrics_APP_ERRORS_TOTAL_labels_inc': MagicMock(),
+        'os_exit': MagicMock(side_effect=raise_os_exit_with_code),
+        'time_time': MagicMock(),
+        'asyncio_sleep': AsyncMock(),
+        'signal_signal': MagicMock(),
+        'traceback_print_exc': MagicMock(),
+        'mock_exit_signal': MagicMock(spec=Event),
+    }
+
+    monkeypatch.setattr(app, 'start_http_server', mocks['start_http_server'])
+    monkeypatch.setattr(app, 'Ingestor', mocks['Ingestor'])
+    monkeypatch.setattr(asyncio, 'sleep', mocks['asyncio_sleep'])
+
+    monkeypatch.setattr(
+        app.metrics.APP_UP,
+        'labels',
+        MagicMock(return_value=MagicMock(set=mocks['metrics_APP_UP_labels_set'])),
+    )
+    monkeypatch.setattr(
+        app.metrics.APP_LOOP_COUNT,
+        'labels',
+        MagicMock(return_value=MagicMock(inc=mocks['metrics_APP_LOOP_COUNT_labels_inc'])),
+    )
+    monkeypatch.setattr(
+        app.metrics.APP_LOOP_DURATION,
+        'labels',
+        MagicMock(
+            return_value=MagicMock(observe=mocks['metrics_APP_LOOP_DURATION_labels_observe'])
+        ),
+    )
+    monkeypatch.setattr(
+        app.metrics.APP_ERRORS_TOTAL,
+        'labels',
+        MagicMock(return_value=MagicMock(inc=mocks['metrics_APP_ERRORS_TOTAL_labels_inc'])),
+    )
+
+    monkeypatch.setattr(app.os, '_exit', mocks['os_exit'])
+    monkeypatch.setattr(app, 'time', mocks['time_time'])
+    monkeypatch.setattr(app.signal, 'signal', mocks['signal_signal'])
+    monkeypatch.setattr(app.traceback, 'print_exc', mocks['traceback_print_exc'])
+
+    monkeypatch.setattr(app, 'exit_signal', mocks['mock_exit_signal'])
+    monkeypatch.setattr(app, 'POD_ID', 'test_pod')
+
+    mock_ingestor_instance = mocks['Ingestor'].return_value
+    mock_ingestor_instance.poll_interval = 0.01
+    mock_ingestor_instance.logger = MagicMock()
+
+    # Make async methods async mocks
+    mock_ingestor_instance.prepare_ingestor = AsyncMock()
+    mock_ingestor_instance.loop = AsyncMock()
+    mock_ingestor_instance.shutdown = AsyncMock()
+
+    return mocks
+
+
+@pytest.mark.asyncio
+async def test_main_successful_run_one_loop(mock_app_env, capsys):
+    """Test a successful run where the loop executes once and then exits gracefully."""
+    mock_ingestor_instance = mock_app_env['Ingestor'].return_value
+    mock_exit_signal = mock_app_env['mock_exit_signal']
+
+    mock_exit_signal.is_set.side_effect = [False, True]
+    mock_app_env['time_time'].side_effect = [10.0, 11.5]
+
+    with pytest.raises(OsExitCalledError) as excinfo:
+        await app.main()
+    assert excinfo.value.code == 0
+
+    mock_app_env['start_http_server'].assert_called_once_with(9090)
+    app.metrics.APP_UP.labels.assert_any_call(pod_id='test_pod')
+    set_calls = mock_app_env['metrics_APP_UP_labels_set'].call_args_list
+    assert call(1) in set_calls
+    assert call(0) in set_calls
+    assert set_calls.index(call(1)) < set_calls.index(call(0))
+
+    mock_app_env['Ingestor'].assert_called_once_with()
+    mock_ingestor_instance.prepare_ingestor.assert_called_once()
+    mock_ingestor_instance.logger.info.assert_any_call('Ingestor prepared. Starting main loop.')
+
+    mock_ingestor_instance.loop.assert_called_once()
+    app.metrics.APP_LOOP_COUNT.labels.assert_called_with(pod_id='test_pod')
+    mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].assert_called_once()
+
+    mock_app_env['asyncio_sleep'].assert_has_calls(
+        [call(mock_ingestor_instance.poll_interval), call(5)]
+    )
+
+    app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id='test_pod')
+    mock_app_env['metrics_APP_LOOP_DURATION_labels_observe'].assert_called_once_with(1.5)
+
+    mock_ingestor_instance.shutdown.assert_called_once()
+    mock_ingestor_instance.logger.info.assert_any_call('Main loop exit_signaled.')
+
+    captured = capsys.readouterr()
+    assert 'Prometheus server started on port 9090.' in captured.out
+
+
+@pytest.mark.asyncio
+async def test_main_prometheus_server_fails_to_start(mock_app_env, capsys):
+    """Test the scenario where starting the Prometheus server fails."""
+    mock_app_env['start_http_server'].side_effect = OSError('Port already in use')
+
+    with pytest.raises(OsExitCalledError) as excinfo:
+        await app.main()
+    assert excinfo.value.code == 1
+
+    # Check that .set(1) was not called. .set(0) definitely not called.
+    called_with_1 = False
+    for call_args in mock_app_env['metrics_APP_UP_labels_set'].call_args_list:
+        if call_args == call(1):
+            called_with_1 = True
+            break
+    assert not called_with_1, 'APP_UP.set(1) should not have been called if server start failed'
+
+    mock_app_env['Ingestor'].assert_not_called()
+    captured = capsys.readouterr()
+    assert 'Failed to start Prometheus server: Port already in use' in captured.out
+
+
+@pytest.mark.asyncio
+async def test_main_loop_exception_handling(mock_app_env, capsys):
+    """Test that an exception in ingestor.loop() is handled gracefully."""
+    mock_ingestor_instance = mock_app_env['Ingestor'].return_value
+    mock_exit_signal = mock_app_env['mock_exit_signal']
+
+    mock_exit_signal.is_set.side_effect = [False, True]
+    mock_ingestor_instance.loop.side_effect = Exception('Test loop exception')
+    mock_app_env['time_time'].side_effect = [10.0, 10.1]
+
+    with pytest.raises(OsExitCalledError) as excinfo:
+        await app.main()
+    assert excinfo.value.code == 0
+
+    mock_ingestor_instance.loop.assert_called_once()
+    mock_app_env['traceback_print_exc'].assert_called_once()
+
+    app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id='test_pod')
+    mock_app_env['metrics_APP_ERRORS_TOTAL_labels_inc'].assert_called_once()
+
+    mock_exit_signal.set.assert_called_once()
+
+    mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].assert_not_called()
+
+    app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id='test_pod')
+    mock_app_env['metrics_APP_LOOP_DURATION_labels_observe'].assert_called_once_with(
+        pytest.approx(0.1)
+    )
+
+    mock_ingestor_instance.shutdown.assert_called_once()
+    captured = capsys.readouterr()
+    assert 'Exception in main loop. Setting exit_signal flag.' in captured.out
+
+
+@pytest.mark.asyncio
+async def test_main_keyboard_interrupt_handling(mock_app_env, capsys):
+    """Test that KeyboardInterrupt in ingestor.loop() is handled."""
+    mock_ingestor_instance = mock_app_env['Ingestor'].return_value
+    mock_exit_signal = mock_app_env['mock_exit_signal']
+
+    mock_exit_signal.is_set.side_effect = [False, True]
+    mock_ingestor_instance.loop.side_effect = KeyboardInterrupt()
+    mock_app_env['time_time'].side_effect = [10.0, 10.1]
+
+    with pytest.raises(OsExitCalledError) as excinfo:
+        await app.main()
+    assert excinfo.value.code == 0
+
+    mock_ingestor_instance.loop.assert_called_once()
+    mock_exit_signal.set.assert_called_once()
+    mock_ingestor_instance.shutdown.assert_called_once()
+    captured = capsys.readouterr()
+    assert 'KeyboardInterrupt received. Setting exit_signal flag.' in captured.out
+    mock_app_env['metrics_APP_ERRORS_TOTAL_labels_inc'].assert_not_called()
+
+
+def test_signal_handler_sets_exit_signal(mock_app_env):
+    """Test that the signal_handler function calls exit_signal.set()."""
+    mock_exit_signal_set = mock_app_env['mock_exit_signal'].set
+
+    app.signal_handler(signal_module.SIGINT, None)
+    mock_exit_signal_set.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_main_multiple_loop_iterations(mock_app_env):
+    """Test the main loop runs for a few iterations."""
+    mock_ingestor_instance = mock_app_env['Ingestor'].return_value
+    mock_exit_signal = mock_app_env['mock_exit_signal']
+
+    mock_exit_signal.is_set.side_effect = [False, False, False, True]
+    mock_app_env['time_time'].side_effect = [10.0, 10.1, 10.2, 10.3, 10.4, 10.5]
+
+    with pytest.raises(OsExitCalledError) as excinfo:
+        await app.main()
+    assert excinfo.value.code == 0
+
+    assert mock_ingestor_instance.loop.call_count == 3
+
+    assert app.metrics.APP_LOOP_COUNT.labels.call_count == 3
+    app.metrics.APP_LOOP_COUNT.labels.assert_called_with(
+        pod_id='test_pod'
+    )  # Checks last call or any call
+    assert mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].call_count == 3
+
+    assert mock_app_env['asyncio_sleep'].call_count == 4
+
+    assert app.metrics.APP_LOOP_DURATION.labels.call_count == 3
+    app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id='test_pod')
+    duration_calls = mock_app_env['metrics_APP_LOOP_DURATION_labels_observe'].call_args_list
+    assert duration_calls[0] == call(pytest.approx(0.1, abs=1e-9))
+    assert duration_calls[1] == call(pytest.approx(0.1, abs=1e-9))
+    assert duration_calls[2] == call(pytest.approx(0.1, abs=1e-9))
+
+    mock_ingestor_instance.shutdown.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_main_pod_id_used_in_metrics(mock_app_env):
+    """Test that the POD_ID from app module is used in metric labels."""
+    mock_exit_signal = mock_app_env['mock_exit_signal']
+    mock_exit_signal.is_set.side_effect = [False, True]
+    mock_app_env['time_time'].side_effect = [10.0, 11.0]
+
+    with pytest.raises(OsExitCalledError):
+        await app.main()
+
+    app.metrics.APP_UP.labels.assert_any_call(pod_id='test_pod')
+    app.metrics.APP_LOOP_COUNT.labels.assert_any_call(pod_id='test_pod')
+    app.metrics.APP_LOOP_DURATION.labels.assert_any_call(pod_id='test_pod')
+    # APP_ERRORS_TOTAL would be checked similarly if it were called in this flow.
+
+    # Check the .set() / .inc() calls on the mocks returned by .labels()
+    mock_app_env['metrics_APP_UP_labels_set'].assert_any_call(1)
+    mock_app_env['metrics_APP_UP_labels_set'].assert_any_call(0)
+    mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_run_async_main(mock_app_env, capsys):
+    """Test the run_async_main function that sets up the event loop."""
+    mock_ingestor_instance = mock_app_env['Ingestor'].return_value
+    mock_exit_signal = mock_app_env['mock_exit_signal']
+
+    mock_exit_signal.is_set.side_effect = [False, True]
+    mock_app_env['time_time'].side_effect = [10.0, 11.0]
+
+    # Instead of calling run_async_main() which creates a new event loop,
+    # we test the main() function directly since that's what run_async_main() would call
+    with pytest.raises(OsExitCalledError) as excinfo:
+        await app.main()
+    assert excinfo.value.code == 0
+
+    # Verify the main function was called through the event loop
+    mock_app_env['start_http_server'].assert_called_once_with(9090)
+    mock_ingestor_instance.prepare_ingestor.assert_called_once()
+    mock_ingestor_instance.loop.assert_called_once()
+    mock_ingestor_instance.shutdown.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_main_prepare_ingestor_failure(mock_app_env, capsys):
+    """Test that prepare_ingestor failure is handled correctly."""
+    mock_ingestor_instance = mock_app_env['Ingestor'].return_value
+    mock_exit_signal = mock_app_env['mock_exit_signal']
+
+    mock_ingestor_instance.prepare_ingestor.side_effect = Exception('Preparation failed')
+    mock_exit_signal.is_set.side_effect = [False, True]
+
+    with pytest.raises(OsExitCalledError) as excinfo:
+        await app.main()
+    assert excinfo.value.code == 0
+
+    # Verify error metrics were incremented
+    app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id='test_pod')
+    mock_app_env['metrics_APP_ERRORS_TOTAL_labels_inc'].assert_called_once()
+
+    # Verify exit signal was set
+    mock_exit_signal.set.assert_called_once()
+
+    # Verify shutdown was called
+    mock_ingestor_instance.shutdown.assert_called_once()
+
+    # Verify error was logged
+    mock_ingestor_instance.logger.error.assert_called_once_with(
+        'Failed to prepare ingestor: Preparation failed'
+    )

tests/unit/test_ingestor.py

@@ -0,0 +1,340 @@
+from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
+
+from pytest import fixture, mark
+
+from ingestor.ingestor import Ingestor
+
+
+@patch('ingestor.ingestor.getenv')
+@patch('ingestor.ingestor.NotificationHandler')
+def test___init__(notification_handler, getenv):
+    getenv.side_effect = [
+        'localhost:9092,localhost:35',  # KAFKA_SERVERS
+        'true',  # EXPORT_TO_KAFKA
+        'localhost',  # REDIS_HOST
+        '63790',  # REDIS_PORT
+        'user',  # REDIS_USERNAME
+        'password',  # REDIS_PASSWORD
+        '100',  # LEASE_TTL
+        '200',  # HEARTBEAT_TTL
+        'localhost',  # HOSTNAME
+        '50',  # POLL_INTERVAL
+        'localhost:27017',  # MONGODB_URL
+        'sientia',  # MONGODB_USERNAME
+        'sientia',  # MONGODB_PASSWORD
+        'sientia',  # MONGODB_DATABASE
+    ]
+
+    ingestor = Ingestor()
+
+    getenv.assert_any_call('KAFKA_SERVERS', 'localhost:9092')
+    getenv.assert_any_call('REDIS_HOST', 'localhost')
+    getenv.assert_any_call('REDIS_PORT', '6379')
+    getenv.assert_any_call('REDIS_USERNAME', None)
+    getenv.assert_any_call('REDIS_PASSWORD', None)
+    getenv.assert_any_call('LEASE_TTL', '10')
+    getenv.assert_any_call('HEARTBEAT_TTL', '20')
+    getenv.assert_any_call('HOSTNAME', 'localhost')
+    getenv.assert_any_call('POLL_INTERVAL', '5')
+
+    assert ingestor.kafka_servers == ['localhost:9092', 'localhost:35']
+    assert ingestor.redis_host == 'localhost'
+    assert ingestor.redis_port == 63790
+    assert ingestor.redis_username == 'user'
+    assert ingestor.redis_password == 'password'
+    assert ingestor.lease_ttl == 100
+    assert ingestor.heartbeat_ttl == 200
+    assert ingestor.pod_id == 'localhost'
+    assert ingestor.poll_interval == 50
+    assert ingestor.metadata == {
+        'model_id': '-',
+        'model_name': '-',
+        'workflow_name': 'opc_ingestor',
+        'schema_name': 'opc_ingestor',
+        'pod_id': 'localhost',
+    }
+
+    notification_handler.assert_called_once_with(
+        connection_string='mongodb://sientia:sientia@localhost:27017',
+        database='sientia',
+        logger=ingestor.logger,
+        project_name='opc_ingestor',
+    )
+
+
+@fixture
+@patch('ingestor.ingestor.getenv')
+@patch('ingestor.ingestor.NotificationHandler')
+def ingestor(_notification_handler, _getenv):
+    ing = Ingestor()
+    ing.logger = MagicMock()
+
+    return ing
+
+
+@fixture
+def ingestor_manager_started(ingestor):
+    ingestor.ingestor_manager = AsyncMock(
+        initialize_opc_from_config=AsyncMock(),
+        shutdown=AsyncMock(),
+        update_opc_servers=AsyncMock(),
+        subscribe_to_tags=AsyncMock(),
+        unsubscribe_slot=AsyncMock(),
+    )
+    return ingestor
+
+
+@mark.asyncio
+async def test_shutdown(ingestor_manager_started):
+    await ingestor_manager_started.shutdown()
+
+    ingestor_manager_started.ingestor_manager.shutdown.assert_called_once()
+
+
+@mark.asyncio
+async def test_handle_acquired_tags_not_acquired(ingestor_manager_started):
+    await ingestor_manager_started.handle_acquired_tags([])
+
+    ingestor_manager_started.logger.warning.assert_called_once_with('No slots available')
+    ingestor_manager_started.ingestor_manager.update_opc_servers.assert_not_called()
+    ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_not_called()
+
+
+@mark.asyncio
+async def test_handle_acquired_tags_success(ingestor_manager_started):
+    await ingestor_manager_started.handle_acquired_tags(['tag1', 'tag2'])
+
+    ingestor_manager_started.logger.warning.assert_not_called()
+    ingestor_manager_started.ingestor_manager.update_opc_servers.assert_called_once()
+    ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_called_once_with(
+        ['tag1', 'tag2']
+    )
+
+
+@patch('ingestor.ingestor.IngestorManager')
+@mark.asyncio
+async def test_prepare_ingestor(ingestor_manager_mock, ingestor):
+    ingestor_manager = ingestor_manager_mock.return_value
+    ingestor_manager.get_slot_leases.return_value = True
+
+    ingestor_manager.declare_active = AsyncMock()
+    ingestor_manager.get_slot_leases = AsyncMock()
+    ingestor.handle_acquired_tags = AsyncMock()
+
+    await ingestor.prepare_ingestor()
+
+    ingestor_manager_mock.assert_called_once_with(
+        kafka_servers=','.join(ingestor.kafka_servers),
+        redis_data={
+            'host': ingestor.redis_host,
+            'port': ingestor.redis_port,
+            'username': ingestor.redis_username,
+            'password': ingestor.redis_password,
+        },
+        lease_ttl=ingestor.lease_ttl,
+        heartbeat_ttl=ingestor.heartbeat_ttl,
+        poll_interval=ingestor.poll_interval,
+        mongo_connection_string=ingestor.mongo_connection_string,
+        mongo_database=ingestor.mongo_database,
+        metadata=ingestor.metadata,
+        logger=ingestor.logger,
+        notification_handler=ingestor.notification_handler,
+        export_to_kafka=ingestor.export_to_kafka,
+        metrics_controller=ingestor.metrics_controller,
+    )
+    ingestor_manager.declare_active.assert_called_once()
+    ingestor_manager.get_slot_leases.assert_called_once()
+
+    ingestor.handle_acquired_tags.assert_called_once_with(
+        ingestor_manager.get_slot_leases.return_value
+    )
+
+
+def test_manage_slots_has_slots(ingestor_manager_started):
+    ingestor_manager_started.handle_acquired_tags = MagicMock()
+    ingestor_manager_started.ingestor_manager.managed_tags = True
+
+    ingestor_manager_started.manage_no_slots(5)
+
+    ingestor_manager_started.ingestor_manager.get_slot_leases.assert_not_called()
+    ingestor_manager_started.ingestor_manager.handle_acquired_tags.assert_not_called()
+
+
+def test_manage_no_slots_has_slots_none_available(ingestor_manager_started):
+    ingestor_manager_started.handle_acquired_tags = MagicMock()
+    ingestor_manager_started.ingestor_manager.managed_tags = True
+
+    ingestor_manager_started.manage_no_slots(0)
+
+    ingestor_manager_started.ingestor_manager.get_slot_leases.assert_not_called()
+    ingestor_manager_started.ingestor_manager.handle_acquired_tags.assert_not_called()
+
+
+def test_manage_slots_none_available(ingestor_manager_started):
+    ingestor_manager_started.handle_acquired_tags = MagicMock()
+    ingestor_manager_started.ingestor_manager.managed_tags = False
+
+    ingestor_manager_started.manage_no_slots(0)
+
+    ingestor_manager_started.ingestor_manager.get_slot_leases.assert_not_called()
+    ingestor_manager_started.ingestor_manager.handle_acquired_tags.assert_not_called()
+
+
+@mark.asyncio
+async def test_manage_slots_none_available_none_available(ingestor_manager_started):
+    ingestor_manager_started.handle_acquired_tags = MagicMock()
+    ingestor_manager_started.ingestor_manager.managed_tags = False
+
+    await ingestor_manager_started.manage_no_slots(2)
+
+    ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with(1)
+
+
+@mark.asyncio
+async def test_manage_leases_no_ingestor_manager(ingestor_manager_started):
+    ingestor_manager_started.ingestor_manager = None
+
+    assert await ingestor_manager_started.manage_leases(2, 2, 5) is None
+
+
+@mark.asyncio
+async def test_manage_leases_no_available_slots_no_extra_slots(ingestor_manager_started):
+    ingestor_manager_started.handle_acquired_tags = MagicMock()
+
+    await ingestor_manager_started.manage_leases(0, 0, 0)
+
+    ingestor_manager_started.ingestor_manager.get_slot_leases.assert_not_called()
+    ingestor_manager_started.ingestor_manager.handle_acquired_tags.assert_not_called()
+    ingestor_manager_started.ingestor_manager.drop_slot_leases.assert_not_called()
+
+
+@mark.asyncio
+async def test_manage_leases_available_slots_innactive_ingestors(ingestor_manager_started):
+    ingestor_manager_started.handle_acquired_tags = MagicMock()
+
+    await ingestor_manager_started.manage_leases(2, 2, 5)
+
+    ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with(2)
+
+    ingestor_manager_started.ingestor_manager.drop_slot_leases.assert_not_called()
+
+
+@mark.asyncio
+async def test_manage_leases_no_available_slots_extra_sltos(ingestor_manager_started):
+    ingestor_manager_started.handle_acquired_tags = MagicMock()
+    ingestor_manager_started.ingestor_manager.managed_tags = {
+        'tag1': 'server1',
+        'tag2': 'server2',
+        'tag3': 'server3',
+    }
+
+    await ingestor_manager_started.manage_leases(0, 0, 2)
+
+    ingestor_manager_started.ingestor_manager.get_slot_leases.assert_not_called()
+    ingestor_manager_started.handle_acquired_tags.assert_not_called()
+    ingestor_manager_started.ingestor_manager.drop_slot_leases.assert_called_once_with(
+        ['tag2', 'tag3']
+    )
+
+
+@mark.asyncio
+async def test_loop(ingestor_manager_started):
+    ingestor_manager_started.manage_no_slots = AsyncMock()
+    ingestor_manager_started.manage_leases = AsyncMock()
+    ingestor_manager_started.update_ingestor_manager = AsyncMock()
+    ingestor_manager_started.ingestor_manager.check_opc_servers_integrity = AsyncMock()
+    ingestor_manager_started.ingestor_manager.managed_tags = {
+        'slot1': 'server1',
+        'slot2': 'server2',
+        'slot3': 'server3',
+    }
+    ingestor_manager_started.ingestor_manager.get_active_ingestors = AsyncMock(
+        return_value=['ingestor1', 'ingestor2']
+    )
+    ingestor_manager_started.ingestor_manager.get_number_of_slots = AsyncMock(return_value=5)
+    ingestor_manager_started.ingestor_manager.get_number_of_leases = AsyncMock(return_value=1)
+
+    await ingestor_manager_started.loop()
+
+    ingestor_manager_started.ingestor_manager.declare_active.assert_called_once()
+    ingestor_manager_started.ingestor_manager.get_active_ingestors.assert_called_once()
+    ingestor_manager_started.ingestor_manager.get_number_of_slots.assert_called_once()
+
+    ingestor_manager_started.manage_no_slots.assert_called_once_with(
+        ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value
+    )
+    # Explanation: 5 - 2 = 3, 3 - 1 = 2
+    ingestor_manager_started.manage_leases.assert_called_once_with(4, 3, 2)
+    ingestor_manager_started.ingestor_manager.update_slot_config.assert_called_once()
+
+    ingestor_manager_started.ingestor_manager.check_opc_servers_integrity.assert_called_once()
+    ingestor_manager_started.update_ingestor_manager.assert_called_once()
+
+
+@mark.asyncio
+async def test_loop_no_managed(ingestor_manager_started):
+    ingestor_manager_started.manage_no_slots = AsyncMock()
+    ingestor_manager_started.manage_leases = AsyncMock()
+    ingestor_manager_started.update_ingestor_manager = AsyncMock()
+    ingestor_manager_started.ingestor_manager.check_opc_servers_integrity = AsyncMock()
+    ingestor_manager_started.ingestor_manager.managed_tags = {}
+    ingestor_manager_started.ingestor_manager.get_active_ingestors = AsyncMock(
+        return_value=['ingestor1', 'ingestor2']
+    )
+    ingestor_manager_started.ingestor_manager.get_number_of_slots = AsyncMock(return_value=5)
+
+    await ingestor_manager_started.loop()
+
+    ingestor_manager_started.ingestor_manager.declare_active.assert_called_once()
+    ingestor_manager_started.ingestor_manager.get_active_ingestors.assert_called_once()
+    ingestor_manager_started.ingestor_manager.get_number_of_slots.assert_called_once()
+
+    ingestor_manager_started.manage_no_slots.assert_called_once_with(
+        ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value
+    )
+    # Explanation: 5 - 2 = 3, 3 - 1 = 2
+    ingestor_manager_started.manage_leases.assert_called_once_with(ANY, 3, -1)
+    ingestor_manager_started.ingestor_manager.update_slot_config.assert_called_once()
+    ingestor_manager_started.logger.info.assert_any_call('No slots acquired in this loop')
+
+    ingestor_manager_started.ingestor_manager.check_opc_servers_integrity.assert_called_once()
+    ingestor_manager_started.update_ingestor_manager.assert_called_once()
+
+
+@mark.asyncio
+async def test_loop_no_ingestor_manager(ingestor_manager_started):
+    ingestor_manager_started.ingestor_manager = None
+
+    assert await ingestor_manager_started.loop() is None
+
+
+@mark.asyncio
+async def test_update_ingestor_manager(ingestor_manager_started):
+    ingestor_manager_started.ingestor_manager.managed_tags = {
+        'slot_to_create': 'new_config',
+        'slot_to_update': 'new_config',
+        'slot_to_do_nothing': 'old_config',
+    }
+
+    old_managed_tags = {
+        'slot_to_update': 'old_config',
+        'slot_to_delete': 'old_config',
+        'slot_to_do_nothing': 'old_config',
+    }
+
+    await ingestor_manager_started.update_ingestor_manager(old_managed_tags)
+
+    ingestor_manager_started.ingestor_manager.update_opc_servers.assert_called_once()
+    ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_has_calls(
+        [call({'slot_to_create': 'new_config'}), call({'slot_to_update': 'new_config'})]
+    )
+    ingestor_manager_started.ingestor_manager.unsubscribe_slot.assert_has_calls(
+        [call('slot_to_update'), call('slot_to_delete')]
+    )
+
+
+@mark.asyncio
+async def test_update_ingestor_manager_no_ingestor_manager(ingestor_manager_started):
+    ingestor_manager_started.ingestor_manager = None
+
+    await ingestor_manager_started.update_ingestor_manager({})

tests/unit/test_metrics.py

@@ -0,0 +1,215 @@
+# tests/unit/test_metrics.py
+
+from prometheus_client import Counter, Gauge, Histogram
+
+import ingestor.metrics as metrics
+
+# --- Test Functions for Each Metric (Corrected for v0.22.0 _name behavior) ---
+
+
+def test_ingestor_tag_written_count():
+    """Verify the definition of INGESTOR_TAG_WRITTEN_COUNT."""
+    assert metrics.TAG_WRITTEN_COUNT is not None
+    assert isinstance(metrics.TAG_WRITTEN_COUNT, Counter)
+    assert metrics.TAG_WRITTEN_COUNT._name == 'ingestor_tag_written_count'
+    assert set(metrics.TAG_WRITTEN_COUNT._labelnames) == {'pod_id', 'tag_name', 'collection_name'}
+
+
+def test_app_loop_count():
+    """Verify the definition of APP_LOOP_COUNT."""
+    assert metrics.APP_LOOP_COUNT is not None
+    assert isinstance(metrics.APP_LOOP_COUNT, Counter)
+    assert metrics.APP_LOOP_COUNT._name == 'app_main_loop'  # REMOVED _total
+    assert set(metrics.APP_LOOP_COUNT._labelnames) == {'pod_id'}
+
+
+def test_app_loop_duration():
+    """Verify the definition of APP_LOOP_DURATION."""
+    assert metrics.APP_LOOP_DURATION is not None
+    assert isinstance(metrics.APP_LOOP_DURATION, Histogram)
+    assert (
+        metrics.APP_LOOP_DURATION._name == 'app_main_loop_duration_seconds'
+    )  # Histograms don't have _total
+    assert set(metrics.APP_LOOP_DURATION._labelnames) == {'pod_id'}
+
+
+def test_app_errors_total():
+    """Verify the definition of APP_ERRORS_TOTAL."""
+    assert metrics.APP_ERRORS_TOTAL is not None
+    assert isinstance(metrics.APP_ERRORS_TOTAL, Counter)
+    assert metrics.APP_ERRORS_TOTAL._name == 'app_errors'  # REMOVED _total
+    assert set(metrics.APP_ERRORS_TOTAL._labelnames) == {'pod_id'}
+
+
+def test_app_up():
+    """Verify the definition of APP_UP."""
+    assert metrics.APP_UP is not None
+    assert isinstance(metrics.APP_UP, Gauge)
+    assert metrics.APP_UP._name == 'app_up'  # Gauges don't have _total
+    assert set(metrics.APP_UP._labelnames) == {'pod_id'}
+
+
+def test_active_ingestors():
+    """Verify the definition of ACTIVE_INGESTORS."""
+    assert metrics.ACTIVE_INGESTORS is not None
+    assert isinstance(metrics.ACTIVE_INGESTORS, Gauge)
+    assert metrics.ACTIVE_INGESTORS._name == 'ingestor_active_total'
+    assert set(metrics.ACTIVE_INGESTORS._labelnames) == {'pod_id'}
+
+
+def test_slots_total():
+    """Verify the definition of SLOTS_TOTAL."""
+    assert metrics.SLOTS_TOTAL is not None
+    assert isinstance(metrics.SLOTS_TOTAL, Gauge)
+    assert metrics.SLOTS_TOTAL._name == 'ingestor_slots_total'
+    assert set(metrics.SLOTS_TOTAL._labelnames) == {'pod_id'}
+
+
+def test_leases_total():
+    """Verify the definition of LEASES_TOTAL."""
+    assert metrics.LEASES_TOTAL is not None
+    assert isinstance(metrics.LEASES_TOTAL, Gauge)
+    assert metrics.LEASES_TOTAL._name == 'ingestor_leases_total'
+    assert set(metrics.LEASES_TOTAL._labelnames) == {'pod_id'}
+
+
+def test_slots_managed():
+    """Verify the definition of SLOTS_MANAGED."""
+    assert metrics.SLOTS_MANAGED is not None
+    assert isinstance(metrics.SLOTS_MANAGED, Gauge)
+    assert metrics.SLOTS_MANAGED._name == 'ingestor_slots_managed_current'
+    assert set(metrics.SLOTS_MANAGED._labelnames) == {'pod_id'}
+
+
+def test_slots_acquired():
+    """Verify the definition of SLOTS_ACQUIRED."""
+    assert metrics.SLOTS_ACQUIRED is not None
+    assert isinstance(metrics.SLOTS_ACQUIRED, Counter)
+    assert metrics.SLOTS_ACQUIRED._name == 'ingestor_slots_acquired'  # REMOVED _total
+    assert set(metrics.SLOTS_ACQUIRED._labelnames) == {'pod_id'}
+
+
+def test_slots_released():
+    """Verify the definition of SLOTS_RELEASED."""
+    assert metrics.SLOTS_RELEASED is not None
+    assert isinstance(metrics.SLOTS_RELEASED, Counter)
+    assert metrics.SLOTS_RELEASED._name == 'ingestor_slots_released'  # REMOVED _total
+    assert set(metrics.SLOTS_RELEASED._labelnames) == {'pod_id'}
+
+
+def test_opc_managers_active():
+    """Verify the definition of OPC_MANAGERS_ACTIVE."""
+    assert metrics.OPC_MANAGERS_ACTIVE is not None
+    assert isinstance(metrics.OPC_MANAGERS_ACTIVE, Gauge)
+    assert metrics.OPC_MANAGERS_ACTIVE._name == 'ingestor_opc_managers_active'
+    assert set(metrics.OPC_MANAGERS_ACTIVE._labelnames) == {'pod_id'}
+
+
+def test_opc_subscription_errors():
+    """Verify the definition of OPC_SUBSCRIPTION_ERRORS."""
+    assert metrics.OPC_SUBSCRIPTION_ERRORS is not None
+    assert isinstance(metrics.OPC_SUBSCRIPTION_ERRORS, Counter)
+    assert (
+        metrics.OPC_SUBSCRIPTION_ERRORS._name == 'ingestor_opc_subscription_errors'
+    )  # REMOVED _total
+    assert set(metrics.OPC_SUBSCRIPTION_ERRORS._labelnames) == {
+        'pod_id',
+        'server',
+        'slot',
+    }
+
+
+def test_opc_connections_total():
+    """Verify the definition of OPC_CONNECTIONS_TOTAL."""
+    assert metrics.OPC_CONNECTIONS_TOTAL is not None
+    assert isinstance(metrics.OPC_CONNECTIONS_TOTAL, Counter)
+    assert metrics.OPC_CONNECTIONS_TOTAL._name == 'opc_connections_initiated'  # REMOVED _total
+    assert set(metrics.OPC_CONNECTIONS_TOTAL._labelnames) == {'pod_id', 'server_name'}
+
+
+def test_opc_connections_failed():
+    """Verify the definition of OPC_CONNECTIONS_FAILED."""
+    assert metrics.OPC_CONNECTIONS_FAILED is not None
+    assert isinstance(metrics.OPC_CONNECTIONS_FAILED, Counter)
+    assert metrics.OPC_CONNECTIONS_FAILED._name == 'opc_connections_failed'  # REMOVED _total
+    assert set(metrics.OPC_CONNECTIONS_FAILED._labelnames) == {'pod_id', 'server_name'}
+
+
+def test_opc_connection_status():
+    """Verify the definition of OPC_CONNECTION_STATUS."""
+    assert metrics.OPC_CONNECTION_STATUS is not None
+    assert isinstance(metrics.OPC_CONNECTION_STATUS, Gauge)
+    assert metrics.OPC_CONNECTION_STATUS._name == 'opc_connection_status'
+    assert set(metrics.OPC_CONNECTION_STATUS._labelnames) == {
+        'pod_id',
+        'server_name',
+        'server_url',
+    }
+
+
+def test_opc_subscriptions_created():
+    """Verify the definition of OPC_SUBSCRIPTIONS_CREATED."""
+    assert metrics.OPC_SUBSCRIPTIONS_CREATED is not None
+    assert isinstance(metrics.OPC_SUBSCRIPTIONS_CREATED, Counter)
+    assert metrics.OPC_SUBSCRIPTIONS_CREATED._name == 'opc_subscriptions_created'  # REMOVED _total
+    assert set(metrics.OPC_SUBSCRIPTIONS_CREATED._labelnames) == {
+        'pod_id',
+        'server_name',
+        'slot_name',
+    }
+
+
+def test_opc_tags_subscribed():
+    """Verify the definition of OPC_TAGS_SUBSCRIBED."""
+    assert metrics.OPC_TAGS_SUBSCRIBED is not None
+    assert isinstance(metrics.OPC_TAGS_SUBSCRIBED, Gauge)
+    assert metrics.OPC_TAGS_SUBSCRIBED._name == 'opc_tags_subscribed_current'
+    assert set(metrics.OPC_TAGS_SUBSCRIBED._labelnames) == {'pod_id', 'server_name'}
+
+
+def test_opc_cycles_without_data():
+    """Verify the definition of OPC_CYCLES_WITHOUT_DATA."""
+    assert metrics.OPC_CYCLES_WITHOUT_DATA is not None
+    assert isinstance(metrics.OPC_CYCLES_WITHOUT_DATA, Gauge)
+    assert metrics.OPC_CYCLES_WITHOUT_DATA._name == 'opc_cycles_without_data'
+    assert set(metrics.OPC_CYCLES_WITHOUT_DATA._labelnames) == {'pod_id', 'server_name'}
+
+
+def test_opc_reconnections_total():
+    """Verify the definition of OPC_RECONNECTIONS_TOTAL."""
+    assert metrics.OPC_RECONNECTIONS_TOTAL is not None
+    assert isinstance(metrics.OPC_RECONNECTIONS_TOTAL, Counter)
+    assert metrics.OPC_RECONNECTIONS_TOTAL._name == 'opc_reconnections_tried'  # REMOVED _total
+    assert set(metrics.OPC_RECONNECTIONS_TOTAL._labelnames) == {'pod_id', 'server_name'}
+
+
+def test_kafka_messages_sent():
+    """Verify the definition of KAFKA_MESSAGES_SENT."""
+    assert metrics.KAFKA_MESSAGES_SENT is not None
+    assert isinstance(metrics.KAFKA_MESSAGES_SENT, Counter)
+    assert metrics.KAFKA_MESSAGES_SENT._name == 'kafka_messages_sent'  # REMOVED _total
+    assert set(metrics.KAFKA_MESSAGES_SENT._labelnames) == {'pod_id', 'topic'}
+
+
+def test_kafka_messages_errors():
+    """Verify the definition of KAFKA_MESSAGES_ERRORS."""
+    assert metrics.KAFKA_MESSAGES_ERRORS is not None
+    assert isinstance(metrics.KAFKA_MESSAGES_ERRORS, Counter)
+    assert metrics.KAFKA_MESSAGES_ERRORS._name == 'kafka_messages_errors'  # REMOVED _total
+    assert set(metrics.KAFKA_MESSAGES_ERRORS._labelnames) == {'pod_id', 'topic'}
+
+
+def test_kafka_connection_status():
+    """Verify the definition of KAFKA_CONNECTION_STATUS."""
+    assert metrics.KAFKA_CONNECTION_STATUS is not None
+    assert isinstance(metrics.KAFKA_CONNECTION_STATUS, Gauge)
+    assert metrics.KAFKA_CONNECTION_STATUS._name == 'kafka_connection_status'
+    assert set(metrics.KAFKA_CONNECTION_STATUS._labelnames) == {'pod_id'}
+
+
+def test_notifications_sent():
+    """Verify the definition of NOTIFICATIONS_SENT."""
+    assert metrics.NOTIFICATIONS_SENT is not None
+    assert isinstance(metrics.NOTIFICATIONS_SENT, Counter)
+    assert metrics.NOTIFICATIONS_SENT._name == 'notifications_sent'  # REMOVED _total
+    assert set(metrics.NOTIFICATIONS_SENT._labelnames) == {'pod_id', 'level', 'block'}

validate.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+# Model Manager Code Validation Script
+# This script runs all code quality checks before committing or deploying
+
+set -e  # Exit on any error
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Args
+FIX_MODE=false
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --fix)
+            FIX_MODE=true
+            shift
+            ;;
+        -h|--help)
+            echo "Usage: $0 [--fix]"
+            echo "  --fix   Apply Ruff auto-fixes (format and lint fixes)."
+            exit 0
+            ;;
+        *)
+            echo -e "${RED}Unknown option: $1${NC}"
+            echo "Usage: $0 [--fix]"
+            exit 2
+            ;;
+    esac
+done
+
+echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
+echo -e "${BLUE}║  Model Manager - Code Validation Suite                ║${NC}"
+echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
+echo ""
+
+# Check if virtual environment is activated
+if [[ -z "${VIRTUAL_ENV}" ]] && [[ -z "${CONDA_DEFAULT_ENV}" ]]; then
+    echo -e "${YELLOW}⚠️  Warning: No virtual environment detected${NC}"
+    echo -e "${YELLOW}   Consider activating your venv/conda environment${NC}"
+    echo ""
+fi
+
+# Function to run a validation step
+run_step() {
+    local step_name=$1
+    local step_command=$2
+    
+    echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+    echo -e "${BLUE}▶ ${step_name}${NC}"
+    echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+    
+    if eval "$step_command"; then
+        echo -e "${GREEN}✅ ${step_name} - PASSED${NC}"
+        echo ""
+        return 0
+    else
+        echo -e "${RED}❌ ${step_name} - FAILED${NC}"
+        echo ""
+        return 1
+    fi
+}
+
+# Track failures
+FAILED_STEPS=()
+
+# Step 1: Code Formatting Check (Ruff)
+# - default: check only
+# - --fix: write changes
+if ! run_step "1. Code Formatting (Ruff)" "if \$FIX_MODE; then ruff format ingestor/ tests/; else ruff format --check ingestor/ tests/; fi"; then
+    FAILED_STEPS+=("Code Formatting")
+fi
+
+# Step 2: Linting (Ruff)
+# - default: check only
+# - --fix: apply autofixes
+if ! run_step "2. Code Linting (Ruff)" "if \$FIX_MODE; then ruff check --fix ingestor/ tests/; else ruff check ingestor/ tests/; fi"; then
+    FAILED_STEPS+=("Linting")
+fi
+
+# Step 3: Type Checking (mypy)
+if ! run_step "3. Type Checking (mypy)" "mypy ingestor/"; then
+    FAILED_STEPS+=("Type Checking")
+fi
+
+# Step 4: Security Analysis (Bandit)
+if ! run_step "4. Security Analysis (Bandit)" "bandit -r ingestor/ -ll -q"; then
+    FAILED_STEPS+=("Security Analysis")
+fi
+
+# Step 5: Unit Tests (pytest)
+if ! run_step "5. Unit Tests (pytest)" "pytest tests/ --cov=ingestor --cov-report=term-missing --cov-report=xml --cov-report=html --cov-fail-under=80 -q"; then
+    FAILED_STEPS+=("Unit Tests")
+fi
+
+# Summary
+echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
+echo -e "${BLUE}║  Validation Summary                                    ║${NC}"
+echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
+echo ""
+
+if [ ${#FAILED_STEPS[@]} -eq 0 ]; then
+    echo -e "${GREEN}✅ All validation checks passed!${NC}"
+    echo -e "${GREEN}   Your code is ready for commit/deployment.${NC}"
+    echo ""
+    exit 0
+else
+    echo -e "${RED}❌ Validation failed for the following steps:${NC}"
+    for step in "${FAILED_STEPS[@]}"; do
+        echo -e "${RED}   • ${step}${NC}"
+    done
+    echo ""
+    echo -e "${YELLOW}💡 Tips:${NC}"
+    echo -e "${YELLOW}   • Run 'ruff format ingestor/ tests/' to auto-fix formatting${NC}"
+    echo -e "${YELLOW}   • Run 'ruff check --fix ingestor/ tests/' to auto-fix linting issues${NC}"
+    echo -e "${YELLOW}   • Review mypy errors and add type hints where needed${NC}"
+    echo -e "${YELLOW}   • Check bandit warnings for security issues${NC}"
+    echo -e "${YELLOW}   • Fix failing tests or improve test coverage${NC}"
+    echo ""
+    exit 1
+fi

values.yaml

@@ -0,0 +1,203 @@
+# Default values for sientia-module.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
+replicaCount: 1
+
+# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
+image:
+  repository: aignosi.azurecr.io/sientia-module
+  # This sets the pull policy for images.
+  pullPolicy: Always
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "0.5.0"
+
+# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+imagePullSecrets:
+- name: docker-hub-secret
+# This is to override the chart name.
+nameOverride: "sientia-opc-ingestor"
+fullnameOverride: "sientia-opc-ingestor"
+namespace: sientia
+
+# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Automatically mount a ServiceAccount's API credentials?
+  automount: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: "sientia-opc-ingestor"
+
+# This is for setting Kubernetes Annotations to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+podAnnotations: {}
+# This is for setting Kubernetes Labels to a Pod.
+# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+podLabels: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+livenessProbe:
+  exec:
+    command:
+      - sh
+      - -c
+      - pgrep -f "ingestor.app"
+  initialDelaySeconds: 20
+  periodSeconds: 30
+
+readinessProbe:
+  exec:
+    command:
+      - sh
+      - -c
+      - pgrep -f "ingestor.app"
+  initialDelaySeconds: 10
+  periodSeconds: 15
+
+
+# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+# Additional volumes on the output Deployment definition.
+volumes: []
+# - name: foo
+#   secret:
+#     secretName: mysecret
+#     optional: false
+
+# Additional volumeMounts on the output Deployment definition.
+volumeMounts: []
+# - name: foo
+#   mountPath: "/etc/foo"
+#   readOnly: true
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+services:
+  metrics:
+    enabled: true
+    type: ClusterIP
+    port: 9090
+    targetPort: 9090
+    name: metrics
+
+# Configuração do ServiceMonitor para o Prometheus Operator
+# ref: https://github.com/prometheus-operator/prometheus-operator
+serviceMonitor:
+  # Se true, um recurso ServiceMonitor será criado.
+  enabled: true
+  # O intervalo no qual as métricas devem ser coletadas (ex: 30s, 1m).
+  endpoints:
+  - port: metrics
+    path: /metrics
+    interval: 30s
+    relabelings: []
+
+  additionalLabels:
+    release: kube-prometheus-stack
+  
+env:
+  # Entrypoint variables 
+  - name: GITHUB_REPO_URL
+    value: "git@github.com:Aignosi/sientia-dataops-opc-ingestor.git"
+  - name: GITHUB_BRANCH
+    value: "feature/SIENTIAPDE-1325-adicionar-metricas-especificas-de-operacoes-externas"
+  - name: PYTHON_APP
+    value: "ingestor.app"
+
+  # Application variables
+  - name: KAFKA_SERVERS
+    value: "kafka.kafka.svc.cluster.local:9092"
+  - name: EXPORT_TO_KAFKA
+    value: "false"
+
+  - name: REDIS_HOST
+    value: "redis-master.redis.svc.cluster.local"
+  - name: REDIS_PORT
+    value: "6379"
+  - name: REDIS_USERNAME
+    valueFrom:
+      secretKeyRef:
+        name: redis
+        key: redis-username
+  - name: REDIS_PASSWORD
+    valueFrom:
+      secretKeyRef:
+        name: redis
+        key: redis-password
+  - name: LEASE_TTL
+    value: "20"
+  - name: HEARTBEAT_TTL
+    value: "30"
+  - name: POLL_INTERVAL
+    value: "10"
+  - name: LOG_LEVEL
+    value: "DEBUG"
+  - name: HTTP_METRICS_PORT
+    value: "9090"
+  - name: OPC_TIMEZONE
+    value: "UTC"
+
+  
+  - name: MONGODB_USERNAME
+    value: "root"
+  - name: MONGODB_PASSWORD
+    value: "wKZDbMNU1c"
+  - name: MONGODB_URL
+    value: "my-release-mongodb.mongodb.svc.cluster.local:27017"
+  - name: MONGODB_DATABASE
+    value: "sientia"
+
+
+ssh:
+  enabled: true
+  secretName: git-ssh-key-sientia-opc-ingestor
+  sshPath: /mnt/.ssh
+  knownHostsPath: /mnt/known_hosts
+
+# kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp
+
+# helm upgrade --install sientia-opc-ingestor sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.5.0
+
+# kubectl create secret generic git-ssh-key-sientia-opc-ingestor \
+#   --namespace sientia \
+#   --from-file=ssh-privatekey=git_key \
+#   --type=kubernetes.io/ssh-auth