commit 6282d0fc6ccee955e5af3cf5d135eace8afb1dc3 Author: gitea_admin Date: Wed Aug 5 13:53:41 2026 +0000 Code import - branch 1.4.2 diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..2249368 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + ingestor/app.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..54ea343 --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +# Redis +REDIS_HOST="localhost" +REDIS_PORT="6379" +REDIS_USERNAME="redis_user" +REDIS_PASSWORD="redis_password" + +# Pod +HOSTNAME="localhost" +POLL_INTERVAL="5" +LOG_LEVEL="DEBUG" + +# Mongo DB +MONGODB_URL="localhost:27018" +MONGODB_USERNAME="mongo_user" +MONGODB_PASSWORD="mongo_password" +MONGODB_DATABASE="sientia" + diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml new file mode 100644 index 0000000..61b07c5 --- /dev/null +++ b/.github/workflows/quality-gate.yml @@ -0,0 +1,19 @@ +name: Quality gate + +on: + pull_request: + branches: + - main + - 'release/**' + - 'feature/**' + types: [ opened, synchronize, reopened ] + +jobs: + quality-gate: + uses: Aignosi/github_workflow_templates/.github/workflows/python-quality-gate.yml@main + permissions: write-all + with: + project_name: 'ingestor' + repositories: 'sientia-dataops-library' + requirements_file: 'requirements.txt' + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..45d63bb --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,16 @@ +name: Create Release on Merge to Main + +on: + pull_request: + types: [closed] + branches: + - main + +jobs: + release: + if: github.event.pull_request.merged == true + uses: Aignosi/github_workflow_templates/.github/workflows/dataops-module-release.yml@main + permissions: write-all + with: + project_name: 'ingestor' + secrets: inherit \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..198783f --- /dev/null +++ b/.gitignore @@ -0,0 +1,184 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ +pytest.xml + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or pytest.xmlpackage, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# VSCode +.vscode/ + +git_log + +.git/ +.ruff_cache/ +.mypy_cache/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..0f2db31 --- /dev/null +++ b/README.md @@ -0,0 +1,407 @@ +# Sientia DataOps OPC Ingestor + +A high-performance, scalable OPC UA data ingestion system designed for industrial data collection and real-time streaming. The OPC Ingestor provides enterprise-grade data acquisition from OPC UA servers with automatic load balancing, fault tolerance, and comprehensive monitoring. + +## Features + +### Core Functionality +- **OPC UA Integration**: Native support for OPC UA servers with secure and unsecured connections +- **Automatic Load Balancing**: Slot-based architecture for horizontal scaling across multiple instances +- **Persistent Storage**: MongoDB integration for historical data persistence +- **Health Monitoring**: Comprehensive Prometheus metrics and health checks +- **Fault Tolerance**: Automatic failover, reconnection, and error recovery + +### Advanced Capabilities +- **Certificate-based Security**: Support for X.509 certificates and private keys +- **Dynamic Tag Management**: Runtime configuration updates without service interruption +- **Resource Coordination**: Redis-based slot leasing and instance coordination +- **Notification System**: Integrated alerting and notification management +- **Performance Optimization**: Configurable polling intervals and data collection frequencies + +## Architecture + +The OPC Ingestor uses a modular, manager-based architecture designed for scalability and fault tolerance: + +``` +┌──────────────────────────────────────────── +│ Main App │ │ IngestorManager │ │ OPC Manager │ +│ │◄──►│ │◄──►│ │ +│ - Signal Hand. │ │ - Slot Mgmt │ │ - Connections │ +│ - Metrics │ │ - Load Balancing │ │ - Subscriptions │ +│ - Lifecycle │ │ - Coordination │ │ - Data Handler │ +└─────────────────┘ └──────────────────┘ └─────────────────┘ + │ + ▼ + ┌──────────────────┐ ┌─────────────────┐ + │ ResourceManager │ │ DataManager │ + │ │ │ │ + │ - Redis Coord. │ │ - MongoDB Store │ + │ - Slot Leasing │ │ - Data Pipeline │ + │ - Heartbeats │ │ │ + └──────────────────┘ └─────────────────┘ +``` + +### Key Components + +- **Ingestor**: Main application orchestrator managing the overall lifecycle +- **IngestorManager**: Coordinates slot allocation, OPC server management, and load balancing +- **OPC Manager**: Handles individual OPC UA server connections and tag subscriptions +- **Data Manager**: Manages data persistence (MongoDB) +- **Resource Manager**: Coordinates resource allocation and instance coordination via Redis + +## 📋 Prerequisites + +- Python 3.11+ +- Redis server +- MongoDB server +- OPC UA servers for data collection + +**Note**: External dependencies (Redis, MongoDB) must be available either through: +- Port forwarding from a Kubernetes cluster +- External Docker Compose setup +- Cloud-managed services +- Local installations + +## ️ Installation + +### Local Development Setup + +1. **Clone the repository** + ```bash + git clone + cd sientia-dataops-opc-ingestor + ``` + +2. **Create virtual environment** + ```bash + python3.11 -m venv venv + source ./venv/bin/activate + ``` + +3. **Install dependencies** + ```bash + pip install -r requirements.txt + ``` + +4. **Create environment configuration file** + ```bash + cp .env.example .env + # Edit .env with your connection details + ``` + +5. **Configure external dependencies** + + You'll need to set up port forwarding or connections to external services. For example: + + ```bash + # Port forwarding from Kubernetes cluster + kubectl port-forward svc/redis-master 6379:6379 + kubectl port-forward svc/mongodb 27017:27017 + + # Or connect to external Docker Compose + # Ensure services are accessible on localhost with appropriate ports + ``` + +6. **Generate an OPC UA client certificate** (only if connecting to a secured OPC UA server) + + See [Certificate Generation](#-certificate-generation) below. + +## Usage + +### Running the Ingestor + +Use the provided script to run the application locally: + +```bash +# Make script executable (first time only) +chmod +x run_local.sh + +# Run the application +./run_local.sh +``` + +The script will: +- Activate the virtual environment +- Load environment variables from `.env` +- Start the ingestor application + +### Running Tests and Coverage + +Use the provided script to run tests with coverage: + +```bash +# Make script executable (first time only) +chmod +x run_coverage.sh + +# Run tests with coverage +./run_coverage.sh +``` + +The script will: +- Activate the virtual environment +- Run pytest with coverage reporting +- Generate HTML coverage report +- Open the coverage report in your browser + +### Manual Test Execution + +You can also run tests manually: + +```bash +# Activate virtual environment +source ./venv/bin/activate + +# Run all tests +pytest + +# Run with coverage +pytest --cov=ingestor --cov-report=html +``` + +### Populating Redis with OPC Configuration + +1. **Activate virtual environment** + ```bash + source ./venv/bin/activate + ``` + +2. **Run the Redis feeder** + ```bash + python simulator/redis-feeder.py + ``` + +The feeder creates sample OPC tag configurations in Redis that the ingestor can discover and manage. + +## 🔐 Certificate Generation + +OPC UA servers that require secure connections (e.g. KEPServer with `Basic256` policy) validate the ingestor via an X.509 client certificate. Use `scripts/generate-and-push-opc-cert.sh` to generate that certificate and push it to the Gitea repo the pod clones on boot. + +```bash +GITEA_PASSWORD='' ./scripts/generate-and-push-opc-cert.sh +``` + +Environment variables (all optional except `GITEA_PASSWORD`): + +| Variable | Description | Default | +|----------|-------------|---------| +| `GITEA_URL` | Gitea in-cluster URL | `https://git.sientia.ai` | +| `GITEA_USER` | Gitea user | `gitea_admin` | +| `GITEA_PASSWORD` | Gitea password (**required**) | — | +| `GITEA_REPO` | Repo the ingestor pod clones | `gitea_admin/sientia-dataops-opc-ingestor` | +| `BRANCH` | Branch to push the cert to | `main` | +| `APP_URI` | SAN URI of the cert — **must match** the `uri` field of the corresponding document in the `OPC_servers` Mongo collection, since the ingestor uses it as `application_uri` (KEPServer rejects with `BadCertificateUriInvalid` on mismatch) | `urn:sientia:opc-ingestor` | +| `CN` | Certificate common name | `sientia-opc-ingestor` | +| `DAYS` | Certificate validity (days) | `3650` | +| `FORCE` | Set to `1` to overwrite an existing cert already pushed to the repo | unset | + +What the script does: +1. Generates a self-signed RSA-2048 X.509 cert + key with `openssl`, with the SAN URI, `keyUsage`, and `extendedKeyUsage` extensions OPC UA / KEPServer require. +2. Clones the ingestor's Gitea repo, copies the cert/key (`.pem`) and a `.der` copy (for manual import) into `certs/`, commits, and pushes. +3. Prints the paths the pod will see after cloning (`/app/certs/opc-ingestor-cert.pem`, `/app/certs/opc-ingestor-key.pem`) and the next manual steps. + +After pushing, you still need to: +- Set `cert_path` / `private_key_path` (and `uri`) on the server's document in the `OPC_servers` collection so `OpcManager` picks them up (see [OPC Server Configuration](#opc-server-configuration)). +- Restart the ingestor deployment so it re-clones the repo: `kubectl -n sientia rollout restart deployment/opc-ingestor`. +- On first connection, the server rejects the new cert — trust it in the OPC UA server's console (e.g. KEPServer: *OPC UA Configuration Manager → Trusted Clients → Trust*) and restart its runtime. + +## 🧪 Testing + +### Unit Tests +```bash +# Install pytest +pip install pytest + +# Run tests +pytest + +# Run with coverage +pip install pytest-cov +pytest --cov=ingestor + +# Generate HTML coverage report +pytest --cov=ingestor --cov-report=html +``` + +### Functional Tests +```bash +# Run functional tests +pytest tests/functional/ +``` + +## 📊 Monitoring and Metrics + +The OPC Ingestor exposes comprehensive Prometheus metrics for monitoring: + +### Application Metrics +- `app_up`: Application health status +- `app_main_loop_total`: Main loop execution count +- `app_main_loop_duration_seconds`: Loop execution time +- `app_errors_total`: Error count + +### OPC Server Metrics +- `opc_connection_status`: Server connection status +- `opc_tags_subscribed_current`: Number of subscribed tags +- `opc_cycles_without_data`: Data reception health +- `opc_subscriptions_created_total`: Subscription count + +### Resource Management Metrics +- `ingestor_slots_managed_current`: Slots managed by this instance +- `ingestor_active_total`: Total active ingestor instances +- `redis_connection_status`: Redis connection health + +### Data Processing Metrics +- `ingestor_tag_written_count`: Data write operations +- `redis_operations_total`: Redis operation count + +## ⚙️ Configuration + +### Environment Variables + +| Variable | Description | Default | Required | +|----------|-------------|---------|----------| +| `REDIS_HOST` | Redis server hostname | `localhost` | Yes | +| `REDIS_PORT` | Redis server port | `6379` | Yes | +| `REDIS_USERNAME` | Redis username | `None` | No | +| `REDIS_PASSWORD` | Redis password | `None` | No | +| `LEASE_TTL` | Slot lease time-to-live (seconds) | `10` | No | +| `HEARTBEAT_TTL` | Heartbeat time-to-live (seconds) | `20` | No | +| `HOSTNAME` | Pod identifier | `localhost` | No | +| `POLL_INTERVAL` | Main loop polling interval (seconds) | `5` | No | +| `MONGODB_URL` | MongoDB server address | `localhost:27017` | Yes | +| `MONGODB_DATABASE` | MongoDB database name | `sientia` | No | +| `MONGODB_USERNAME` | MongoDB username | `sientia` | No | +| `MONGODB_PASSWORD` | MongoDB password | `sientia` | No | +| `HTTP_METRICS_PORT` | Prometheus metrics port | `9090` | No | + +### OPC Server Configuration + +OPC servers are configured through Redis with the following structure. `cert_path` / `private_key_path` are only required for secured connections (see [Certificate Generation](#-certificate-generation)): + +```json +{ + "slot:opc_tags:1": { + "server1": { + "name": "server1", + "server_id": "1", + "url": "opc.tcp://localhost:4841", + "server_uri": "http://opcua-server.simulator", + "cert_path": "/app/certs/opc-ingestor-cert.pem", + "private_key_path": "/app/certs/opc-ingestor-key.pem", + "tags": { + "ns=2;i=2": { + "aggr_func": "avg", + "data_range": [ + -100, + 100 + ], + "frequency": "15000", + "server_id": "1", + "tag_address": "ns=2;i=2", + "tag_name": "Counter", + "topics": [ + "raw_scouter-opcua-orchestrated-pipeline", + "raw_scouter-basic-sum-model" + ] + } + } + } +} +``` + +## 🔧 Development + +### Project Structure +``` +sientia-dataops-opc-ingestor/ +├── ingestor/ # Main application code +│ ├── managers/ # Component managers +│ │ ├── data_manager.py # Data persistence and export +│ │ ├── ingestor_manager.py # Main coordination +│ │ ├── opc_manager.py # OPC UA server management +│ │ └── resource_manager.py # Resource coordination +│ ├── app.py # Main application entry point +│ ├── ingestor.py # Core ingestor logic +│ └── metrics.py # Prometheus metrics definitions +├── simulator/ # OPC simulation and testing tools +├── scripts/ # Ops scripts (e.g. OPC UA cert generation) +├── tests/ # Test suite +└── requirements.txt # Python dependencies +``` + +### Adding New Features + +1. **Follow the manager pattern** for new components +2. **Add comprehensive docstrings** for all public methods +3. **Include Prometheus metrics** for monitoring +4. **Add unit tests** for new functionality +5. **Update this README** with new features and configuration + +## 🐛 Troubleshooting + +### Common Issues + +1. **OPC Connection Failures** + - Verify server URLs and network connectivity + - Check certificate paths and security settings + - Review server logs for authentication issues + +2. **Redis Connection Issues** + - Verify Redis server is running and accessible + - Check authentication credentials + - Ensure proper network configuration + +3. **Performance Issues** + - Monitor Prometheus metrics for bottlenecks + - Adjust polling intervals and lease TTLs + - Review OPC server performance and network latency + +### Debug Mode + +Enable debug logging by setting the log level in your environment: +```bash +export LOG_LEVEL=DEBUG +``` + +## Performance Tuning + +### Key Parameters + +- **`POLL_INTERVAL`**: Main loop frequency (lower = more responsive, higher = less CPU) +- **`LEASE_TTL`**: Slot lease duration (lower = faster failover, higher = more stable) +- **`HEARTBEAT_TTL`**: Instance health check frequency +- **Tag frequency**: OPC tag collection rate (Hz) + +### Scaling Considerations + +- **Horizontal Scaling**: Deploy multiple ingestor instances for high availability +- **Load Distribution**: Use Redis-based slot allocation for automatic load balancing +- **Resource Limits**: Monitor CPU, memory, and network usage +- **Database Performance**: Optimize MongoDB indexes + +## 🤝 Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes with comprehensive testing +4. Update documentation and docstrings +5. Submit a pull request + +### Code Quality Standards + +- Follow PEP 8 style guidelines +- Include comprehensive docstrings for all public methods +- Maintain test coverage above 80% +- Use type hints where appropriate +- Follow the established architectural patterns + +## 🆘 Support + +For support and questions: +- Check the troubleshooting section above +- Review the metrics and logs for error patterns +- Open an issue in the project repository +- Contact the development team + +--- + +**Note**: This OPC Ingestor is designed for production use in industrial environments. Ensure proper security configuration and network isolation for production deployments. diff --git a/encrypt.py b/encrypt.py new file mode 100644 index 0000000..ad990a6 --- /dev/null +++ b/encrypt.py @@ -0,0 +1,113 @@ +import os +import argparse +from pathspec import PathSpec +import yaml # type: ignore +from typing import Any + +''' +Usage: + python .\encrypt.py path_to_dir output_file --ignore ignore_file --chunk-size 100000 +''' + + +def load_ignore_patterns(ignore_file, include_library): + # Ensure the .gitignore file exists + if not os.path.exists(ignore_file): + raise FileNotFoundError(f"Ignore file not found at {ignore_file}") + + # Load and parse the .gitignore patterns + with open(ignore_file, 'r') as file: + patterns = file.readlines() + if not include_library: + patterns.append('**/deploy/library/') + + spec = PathSpec.from_lines('gitwildmatch', patterns) + return spec + + +def is_ignored(file_path, spec): + """Check if a file should be ignored based on the ignore patterns.""" + return spec.match_file(file_path) if spec else False + + +def encode_file_tree_to_yaml(directory, ignore_file, include_library): + """Encode the file tree into a single YAML file.""" + ignore_patterns = load_ignore_patterns( + ignore_file, include_library) if ignore_file else None + file_tree: dict[str, Any] = {} + + for root, dirs, files in os.walk(directory): + # Skip ignored directories + dirs[:] = [d for d in dirs if not is_ignored( + os.path.join(root, d), ignore_patterns)] + + for file in files: + file_path = os.path.join(root, file) + + # Skip ignored files + if is_ignored(file_path, ignore_patterns): + continue + + # Read file content + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + except Exception as e: + print(f"Error reading file {file_path}: {e}") + raise + + # Create nested dictionary structure + path_parts = os.path.relpath(file_path, directory).split(os.sep) + current_level = file_tree + + # all except the last part (the file name) + for part in path_parts[:-1]: + current_level = current_level.setdefault(part, {}) + + # Add the file and its content + current_level[path_parts[-1]] = content + return yaml.dump(file_tree, default_flow_style=False) + + +def chunk_and_write_file_tree_to_yaml(yaml_content, output_file, chunk_size=None): + """Chunk the YAML content and write it to the output file.""" + + chunks = [yaml_content] if chunk_size is None else [ + yaml_content[i:i + chunk_size] for i in range(0, len(yaml_content), chunk_size)] + + for i, chunk in enumerate(chunks): + chunk_file = f"{output_file}_{i}.yaml" + # Write the file tree to the output YAML file + with open(chunk_file, 'w', encoding='utf-8') as yaml_file: + yaml_file.write(chunk) + + +def main(): + parser = argparse.ArgumentParser( + description="Encrypts file tree to yaml file") + parser.add_argument("input_directory", help="Directory to encode") + parser.add_argument("output_yaml_file", help="Output YAML file") + parser.add_argument("--ignore", default=None, + help="Path to the ignore file") + parser.add_argument("--chunk-size", type=int, default=None, + help="Chunk size for the output YAML file") + parser.add_argument("--library", type=bool, default=False, + help="Incude the library in the output YAML file") + + # Parse arguments + args = parser.parse_args() + + # Example usage + directory_to_encode = args.input_directory + ignore_file_path = args.ignore + output_yaml_file = args.output_yaml_file + include_library = args.library + + content = encode_file_tree_to_yaml( + directory_to_encode, ignore_file_path, include_library) + chunk_and_write_file_tree_to_yaml( + content, output_yaml_file, args.chunk_size) + + +if __name__ == "__main__": + main() diff --git a/ingestor/__init__.py b/ingestor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ingestor/app.py b/ingestor/app.py new file mode 100644 index 0000000..36902f6 --- /dev/null +++ b/ingestor/app.py @@ -0,0 +1,150 @@ +import asyncio +import os +import signal +import traceback +from threading import Event +from time import time + +from prometheus_client import start_http_server + +import ingestor.metrics as metrics +from ingestor.ingestor import Ingestor + +exit_signal = Event() +POD_ID = os.getenv('HOSTNAME', 'localhost') + + +async def main(): # NOSONAR + """ + Main asynchronous function that orchestrates the OPC Ingestor application. + + This function performs the following operations: + 1. Starts the Prometheus metrics server for monitoring + 2. Initializes the Ingestor instance + 3. Prepares the ingestor (connects to services, acquires slot leases) + 4. Runs the main processing loop until shutdown is requested + 5. Handles graceful shutdown and cleanup + + The main loop continuously: + - Processes OPC data from subscribed tags + - Manages slot leases and resource allocation + - Monitors OPC server connections + - Records metrics for monitoring and observability + + Environment Variables: + HOSTNAME: Pod identifier for metrics labeling (default: "localhost") + HTTP_METRICS_PORT: Port for Prometheus metrics server (default: 9090) + + Raises: + Exception: If ingestor preparation fails, the application will exit + """ + start_prometheus_server() + ingestor = Ingestor() + try: + await ingestor.prepare_ingestor() + except Exception as e: + metrics.APP_ERRORS_TOTAL.labels(pod_id=POD_ID).inc() # Increment errors + ingestor.logger.error(f'Failed to prepare ingestor: {e}') + exit_signal.set() + ingestor.logger.info('Ingestor prepared. Starting main loop.') + + while not exit_signal.is_set(): # NOSONAR + start_time = time() # Start loop timer + try: + await ingestor.loop() + metrics.APP_LOOP_COUNT.labels(pod_id=POD_ID).inc() # Increment loop counter + + # Use asyncio.sleep instead of exit_signal.wait for better async compatibility + await asyncio.sleep(ingestor.poll_interval) # NOSONAR + + except KeyboardInterrupt: # Handle Ctrl+C gracefully + print('KeyboardInterrupt received. Setting exit_signal flag.') + exit_signal.set() + except Exception: + print('Exception in main loop. Setting exit_signal flag.') + traceback.print_exc() + metrics.APP_ERRORS_TOTAL.labels(pod_id=POD_ID).inc() # Increment errors + exit_signal.set() + finally: + # Record loop duration + duration = time() - start_time + metrics.APP_LOOP_DURATION.labels(pod_id=POD_ID).observe(duration) + + await ingestor.shutdown() + metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN + + ingestor.logger.info('Main loop exit_signaled.') + + # Give Prometheus a chance to scrape one last time before exiting (optional) + await asyncio.sleep(5) + + os._exit(0) + + +def signal_handler(_signum, _frame): + """ + Signal handler for graceful application shutdown. + + This function handles system signals (SIGINT, SIGTERM, SIGHUP) by setting + the exit_signal flag, which triggers the main loop to complete its current + iteration and then shut down gracefully. + + Args: + _signum: The signal number received + _frame: The current stack frame (unused) + """ + print(f'Received signal {_signum}. Setting exit_signal flag.') + exit_signal.set() + + +def start_prometheus_server(): + """ + Starts the Prometheus metrics HTTP server. + + This function initializes a Prometheus metrics server on the configured port + to expose application metrics for monitoring and alerting. The server provides + metrics about application health, performance, and operational status. + + Environment Variables: + HTTP_METRICS_PORT: Port number for the metrics server (default: 9090) + + Raises: + Exception: If the server fails to start, the application will exit + """ + try: + port = int(os.getenv('HTTP_METRICS_PORT', 9090)) + start_http_server(port) + print(f'Prometheus server started on port {port}.') + metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP + except Exception as e: + print(f'Failed to start Prometheus server: {e}') + os._exit(1) + + +def run_async_main(): + """ + Run the async main function with proper event loop setup. + + This function sets up the asyncio event loop and runs the main async function. + It handles KeyboardInterrupt gracefully and ensures proper cleanup of the event loop. + + The function is designed to work with both direct execution and containerized + environments, providing consistent behavior across different deployment scenarios. + """ + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(main()) + except KeyboardInterrupt: + print('KeyboardInterrupt received in main thread.') + exit_signal.set() + finally: + loop.close() + + +if __name__ == '__main__': + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGHUP, signal_handler) + + run_async_main() diff --git a/ingestor/ingestor.py b/ingestor/ingestor.py new file mode 100644 index 0000000..2244d61 --- /dev/null +++ b/ingestor/ingestor.py @@ -0,0 +1,444 @@ +from copy import deepcopy +from os import getenv +from typing import Any + +from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler +from sientia_do.observability.logger import get_logger +from sientia_do.observability.metrics_controller import MetricsController +from sientia_do.observability.sientia_monitoring import SientiaMonitoring + +import ingestor.metrics as metrics +from ingestor.managers.ingestor_manager import IngestorManager + + +class Ingestor(SientiaMonitoring): + """ + Main OPC Ingestor class that orchestrates data collection from OPC UA servers. + + The Ingestor is responsible for: + - Managing slot leases for load balancing across multiple instances + - Connecting to and monitoring OPC UA servers + - Subscribing to OPC tags and collecting real-time data + - Distributing data to MongoDB + - Providing health monitoring and metrics collection + + The ingestor uses a slot-based architecture where each slot represents + a collection of OPC tags that can be managed by a single ingestor instance. + This allows for horizontal scaling and load distribution. + + Environment Variables: + REDIS_HOST: Redis server hostname (default: "localhost") + REDIS_PORT: Redis server port (default: 6379) + REDIS_USERNAME: Redis username (optional) + REDIS_PASSWORD: Redis password (optional) + LEASE_TTL: Time-to-live for slot leases in seconds (default: 10) + HEARTBEAT_TTL: Time-to-live for heartbeats in seconds (default: 20) + HOSTNAME: Pod identifier (default: "localhost") + POLL_INTERVAL: Main loop polling interval in seconds (default: 5) + MONGODB_URL: MongoDB server address (default: "localhost:27017") + MONGODB_USERNAME: MongoDB username (default: "sientia") + MONGODB_PASSWORD: MongoDB password (default: "sientia") + MONGODB_DATABASE: MongoDB database name (default: "sientia") + + Attributes: + redis_host (str): Redis server hostname + redis_port (int): Redis server port + redis_username (str): Redis username (optional) + redis_password (str): Redis password (optional) + lease_ttl (int): Time-to-live for slot leases in seconds + heartbeat_ttl (int): Time-to-live for heartbeat signals in seconds + pod_id (str): Identifier for the current pod or host + poll_interval (int): Interval in seconds for polling operations + mongo_database (str): MongoDB database name + mongo_connection_string (str): Complete MongoDB connection string + logger: Logger instance for application logging + notification_handler: Handler for sending notifications + metadata (dict): Application metadata for notifications and tracking + ingestor_manager: Manager instance for coordinating operations + """ + + def __init__(self): + """ + Initializes the ingestor with configuration values retrieved from environment variables. + + Sets up all necessary connections and configurations for: + - Redis for slot management and coordination + - MongoDB for data persistence and notifications + - OPC UA server management + - Metrics collection and monitoring + """ + + self.redis_host = getenv('REDIS_HOST', 'localhost') + self.redis_port = int(getenv('REDIS_PORT', '6379')) + self.redis_username = getenv('REDIS_USERNAME', None) + self.redis_password = getenv('REDIS_PASSWORD', None) + self.lease_ttl = int(getenv('LEASE_TTL', '10')) + self.heartbeat_ttl = int(getenv('HEARTBEAT_TTL', '20')) + self.pod_id = getenv('HOSTNAME', 'localhost') + self.poll_interval = int(getenv('POLL_INTERVAL', '5')) + mongo_url = getenv('MONGODB_URL', 'localhost:27017') + mongo_username = getenv('MONGODB_USERNAME', 'sientia') + mongo_password = getenv('MONGODB_PASSWORD', 'sientia') + self.mongo_database = getenv('MONGODB_DATABASE', 'sientia') + self.mongo_connection_string = f'mongodb://{mongo_username}:{mongo_password}@{mongo_url}' + + self.logger = get_logger(__name__) + self.notification_handler = NotificationHandler( + connection_string=self.mongo_connection_string, + database=self.mongo_database, + logger=self.logger, + project_name='opc_ingestor', + ) + self.metrics_controller = MetricsController(logger=self.logger) + + SientiaMonitoring.__init__( + self, + logger=self.logger, + notification_handler=self.notification_handler, + metrics_controller=self.metrics_controller, + ) + + self.metadata = { + 'model_id': '-', + 'model_name': '-', + 'workflow_name': 'opc_ingestor', + 'schema_name': 'opc_ingestor', + 'pod_id': self.pod_id, + } + self.ingestor_manager: IngestorManager | None = None + + async def shutdown(self): + """ + Gracefully shuts down the ingestor and all its components. + + This method ensures proper cleanup of: + - OPC UA connections and subscriptions + - Resource managers and data connections + - Active slot leases and heartbeats + + Should be called before application termination to prevent resource leaks. + """ + if self.ingestor_manager: + await self.ingestor_manager.shutdown() + + async def handle_acquired_tags(self, acquired): + """ + Handles the acquired tags by subscribing to them if available. + + This method processes the tags that have been allocated to this ingestor + instance through the slot leasing system. It updates OPC server configurations + and establishes subscriptions to the allocated tags. + + Args: + acquired (list): A list of acquired tags to be processed. If the list + is empty or None, no action is taken other than logging + a warning. + + Behavior: + - If no tags are acquired, logs a warning about no slots being available + - If tags are acquired, updates OPC server configurations and subscribes + to the allocated tags for data collection + """ + + if not acquired: + self.logger.warning('No slots available') + + else: + # Subscribe to acquired slots + if self.ingestor_manager: + await self.ingestor_manager.update_opc_servers() + await self.ingestor_manager.subscribe_to_tags(acquired) + + async def prepare_ingestor(self): + """ + Prepares the ingestor by initializing all components and acquiring initial slot leases. + + This method performs the following steps: + 1. Initializes the IngestorManager with all necessary configuration parameters + 2. Declares the ingestor as active in the coordination system + 3. Acquires slot leases for tag management + 4. Processes the acquired tags and establishes OPC subscriptions + + The preparation phase is critical for establishing the ingestor's role in + the distributed system and ensuring it can begin processing OPC data. + + Raises: + Exception: If any error occurs during the initialization or lease acquisition process. + This will cause the application to exit as the ingestor cannot function + without proper initialization. + """ + + self.ingestor_manager = IngestorManager( + redis_data={ + 'host': self.redis_host, + 'port': self.redis_port, + 'username': self.redis_username, + 'password': self.redis_password, + }, + lease_ttl=self.lease_ttl, + heartbeat_ttl=self.heartbeat_ttl, + poll_interval=self.poll_interval, + mongo_connection_string=self.mongo_connection_string, + mongo_database=self.mongo_database, + metadata=self.metadata, + logger=self.logger, + notification_handler=self.notification_handler, + metrics_controller=self.metrics_controller, + ) + assert self.ingestor_manager is not None + + # Declare ingestor active + await self.ingestor_manager.declare_active() + + # Get slot lease + acquired = await self.ingestor_manager.get_slot_leases() + self.logger.info(f'Acquired slots: {acquired}') + + await self.handle_acquired_tags(acquired) + + await self.emit_metric( + metric_object=metrics.SLOTS_MANAGED, + method='set', + value=len(self.ingestor_manager.managed_tags), + tags={ + 'pod_id': self.pod_id, + }, + ) + + async def manage_no_slots(self, number_of_slots: int): + """ + Manages the scenario where there are no slots assigned to the ingestor. + + This method handles the case where an ingestor is active but has no + allocated slots. It attempts to acquire a slot lease if slots are + available in the system. + + Args: + number_of_slots (int): The number of available slots in the system. + + Behavior: + - Only attempts to acquire slots if the ingestor is currently active + (has no managed tags) and there are slots available + - Requests a single slot lease to begin processing + """ + + if self.ingestor_manager and not self.ingestor_manager.managed_tags and number_of_slots > 0: + # This ingestor is active and has no slots, so we need to try to + + # Get slot lease + await self.ingestor_manager.get_slot_leases(1) + + async def manage_leases(self, available_slots: int, lacking_ingestors: int, slot_diff: int): + """ + Manages the allocation and deallocation of slot leases for ingestors. + + This method implements the load balancing logic for distributing OPC tag + processing across multiple ingestor instances. It ensures optimal resource + utilization and fair distribution of work. + + Args: + available_slots (int): The number of slots currently available for allocation. + lacking_ingestors (int): The number of ingestors that are active and without slots. + slot_diff (int): The difference between the total slots and the required slots. + + Behavior: + - If there are available slots and lacking ingestors, attempts to acquire + slot leases for the available slots and processes the acquired tags. + - If there are no lacking ingestors but there are extra slots (slot_diff > 0), + releases the extra slot leases to ensure proper allocation. + + Logs: + - Logs the number of available slots when attempting to acquire leases. + - Logs the number of extra slots when releasing leases. + """ + if not self.ingestor_manager: + return + + if available_slots > 0 and lacking_ingestors > 0: + # Some ingestors are inactive, so there are "available_slots" slots available + self.logger.info(f'Slots available: {available_slots}') + + # Get slot lease + await self.ingestor_manager.get_slot_leases(available_slots) + + elif lacking_ingestors <= 0 and slot_diff > 0: + self.logger.info(f'Extra slots available: {slot_diff}') + # There's enough slots for all ingestors, but this ingestor has more than one slot + # So we need to drop the extra leases + + overleases = list(self.ingestor_manager.managed_tags.keys())[1:] + + await self.ingestor_manager.drop_slot_leases(overleases) + + for lease in overleases: + await self.ingestor_manager.unsubscribe_slot(lease) + self.ingestor_manager.managed_tags.pop(lease) + + await self.emit_metric( + metric_object=metrics.SLOTS_MANAGED, + method='set', + value=len(self.ingestor_manager.managed_tags), + tags={ + 'pod_id': self.pod_id, + }, + ) + + async def update_ingestor_manager(self, old_managed_tags: dict[str, Any]): + """ + Updates the ingestor manager with new managed tags and handles configuration changes. + + This method compares the current managed tags with the previous state and + performs necessary operations to maintain synchronization: + - Subscribes to newly allocated tags + - Resubscribes to tags with changed configurations + - Unsubscribes from deallocated tags + + Args: + old_managed_tags (Dict[str, Any]): The previous state of managed tags. + + Behavior: + - Compares current and previous tag configurations + - Establishes subscriptions for new tags + - Updates subscriptions for modified tags + - Removes subscriptions for deallocated tags + - Updates metrics to reflect current state + """ + + if not self.ingestor_manager: + return + + self.logger.debug(f'Current managed tags: {self.ingestor_manager.managed_tags}') + + await self.ingestor_manager.update_opc_servers() + new_managed_tags = deepcopy(self.ingestor_manager.managed_tags) + + self.logger.debug( + f'Comparing new managed tags {new_managed_tags} with old managed tags {old_managed_tags}' + ) + + keys = set(new_managed_tags) | set(old_managed_tags) + + changes = { + k: (new_managed_tags.get(k), old_managed_tags.get(k)) + for k in keys + if new_managed_tags.get(k) != old_managed_tags.get(k) + } + + self.logger.debug(f'Changes: {changes}') + + for slot, config in new_managed_tags.items(): + if slot not in old_managed_tags: + self.logger.info(f'Subscribing to new slot {slot}') + await self.ingestor_manager.subscribe_to_tags({slot: config}) + continue + + if config != old_managed_tags[slot]: + self.logger.info(f'Resubscribing to slot {slot}') + await self.ingestor_manager.unsubscribe_slot(slot) + await self.ingestor_manager.subscribe_to_tags({slot: config}) + + for slot in old_managed_tags.keys(): + if slot not in new_managed_tags: + self.logger.info(f'Unsubscribing from slot {slot}') + await self.ingestor_manager.unsubscribe_slot(slot) + + await self.emit_metric( + metric_object=metrics.SLOTS_MANAGED, + method='set', + value=len(self.ingestor_manager.managed_tags), + tags={ + 'pod_id': self.pod_id, + }, + ) + + async def loop(self): + """ + Executes the main processing loop for managing ingestors and slots. + + This method is the core of the ingestor's operation, performing the following + tasks in each iteration: + 1. Declares the ingestor as active to maintain its presence in the system + 2. Polls for slot updates and manages resource allocation + 3. Handles scenarios where no slots are available + 4. Manages slot leases based on system load and available resources + 5. Updates OPC server configurations and checks server integrity + 6. Synchronizes managed tags with the current system state + + The loop implements a sophisticated load balancing algorithm that: + - Distributes OPC tag processing across multiple ingestor instances + - Ensures optimal resource utilization + - Maintains system stability during scaling operations + - Provides real-time monitoring and metrics collection + + This method is intended to be called repeatedly to ensure the ingestor + manager operates correctly and maintains synchronization with the slots + and OPC servers. + """ + + if not self.ingestor_manager: + return + + await self.ingestor_manager.declare_active() + + self.logger.info('Polling for slot updates...') + # Get active ingestors + + current_managed_tags = deepcopy(self.ingestor_manager.managed_tags) + + ingestors = await self.ingestor_manager.get_active_ingestors() + number_of_ingestors = len(ingestors) + number_of_leases = await self.ingestor_manager.get_number_of_leases() + number_of_slots = await self.ingestor_manager.get_number_of_slots() + + # Update active ingestors gauge + await self.emit_metric( + metric_object=metrics.ACTIVE_INGESTORS, + method='set', + value=number_of_ingestors, + tags={ + 'pod_id': self.pod_id, + }, + ) + + # Handle no slots + self.logger.info('Managing no slots...') + await self.manage_no_slots(number_of_slots) + + available_slots = number_of_slots - number_of_leases + lacking_ingestors = number_of_slots - number_of_ingestors + slot_diff = len(self.ingestor_manager.managed_tags) - 1 + + self.logger.info('Managing leases...') + await self.manage_leases(available_slots, lacking_ingestors, slot_diff) + + # Update managed slots gauge + await self.emit_metric( + metric_object=metrics.SLOTS_MANAGED, + method='set', + value=len(self.ingestor_manager.managed_tags), + tags={ + 'pod_id': self.pod_id, + }, + ) + + self.logger.debug( + f'Active ingestors: {ingestors}, ' + f'Number of slots: {number_of_slots}, ' + f'Number of leases: {number_of_leases}, ' + f'Managed tags: {self.ingestor_manager.managed_tags}, ' + f'Managed servers: {self.ingestor_manager.opc_managers}' + ) + if not self.ingestor_manager.managed_tags: + # No slots acquired + self.logger.info('No slots acquired in this loop') + + # Update opc servers + self.logger.info('Updating slot config...') + await self.ingestor_manager.update_slot_config() + + # Check OPC cycles + self.logger.info('Checking OPC servers integrity...') + await self.ingestor_manager.check_opc_servers_integrity() + + self.logger.info('Updating managed tags...') + await self.update_ingestor_manager(current_managed_tags) diff --git a/ingestor/managers/__init__.py b/ingestor/managers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ingestor/managers/data_manager.py b/ingestor/managers/data_manager.py new file mode 100644 index 0000000..ef3b5ac --- /dev/null +++ b/ingestor/managers/data_manager.py @@ -0,0 +1,138 @@ +import os +import traceback + +from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler +from sientia_do.notifications.models import NotificationLevel +from sientia_do.observability.logger import Logger +from sientia_do.observability.metrics_controller import MetricsController +from sientia_do.observability.sientia_monitoring import SientiaMonitoring +from sientia_do.repository.mongodb_repository import MongoDBRepository +from sientia_do.temporal.constants import now + +import ingestor.metrics as metrics + + +class DataManager(SientiaMonitoring): + """ + Manages data persistence operations for the OPC Ingestor. + + The DataManager is responsible for: + - Storing OPC data in MongoDB for historical analysis and persistence + - Managing database connections and ensuring data integrity + - Providing data access interfaces for other components + + Args: + mongo_connection_string (str): MongoDB connection string + mongo_database (str): MongoDB database name + metadata (dict): Application metadata for notifications and tracking + logger (Logger): Logger instance for application logging + notification_handler (NotificationHandler): Handler for sending notifications + + Attributes: + pod_id (str): Pod identifier for metrics labeling + connection_string (str): MongoDB connection string + database (str): MongoDB database name + mongo_client (MongoClient): MongoDB client instance + metadata (dict): Application metadata + """ + + def __init__( + self, + mongo_connection_string: str, + mongo_database: str, + metadata: dict, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ) -> None: + """ + Initializes the DataManager instance with a MongoDB connection. + + Args: + mongo_connection_string (str): MongoDB connection string + mongo_database (str): MongoDB database name + metadata (dict): Application metadata + logger (Logger): Logger instance + notification_handler (NotificationHandler): Notification handler + """ + + self.pod_id = os.getenv('HOSTNAME', 'localhost') + + SientiaMonitoring.__init__( + self, + logger=logger, + metrics_controller=metrics_controller, + notification_handler=notification_handler, + ) + + logger.info( + f'Trying to initializing DataManager with MongoDB servers: {mongo_connection_string}' + ) + + self.connection_string = mongo_connection_string + self.database = mongo_database + + self.mongo_repository = MongoDBRepository( + connection_string=self.connection_string, + database_name=self.database, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + self.metadata = metadata + + logger.info(f'DataManager initialized with MongoDB servers: {self.connection_string}') + + def shutdown(self): + """ + Gracefully shuts down the DataManager and closes all connections. + """ + try: + self.mongo_repository.close() + except Exception as e: + self.logger.error(f'Error closing MongoDB client: {e}') + + def __del__(self): + self.shutdown() + + async def publish(self, topic: str, data: dict) -> None: + """ + Persists a message to MongoDB. + + Args: + topic (str): The name of the MongoDB collection to which the message will be written. + data (dict): The message data to be stored. + + Returns: + None + """ + + try: + await self.mongo_repository.insert( + collection_name=topic, + document={**data, 'inserted_at': now()}, + metadata=self.metadata, + ) + self.logger.debug(f'Message inserted into MongoDB collection {topic}: {data}') + + await self.emit_metric( + metric_object=metrics.TAG_WRITTEN_COUNT, + tags={ + 'pod_id': self.pod_id, + 'tag_name': data['name'], + 'collection_name': topic, + }, + ) + + except Exception as e: + trace = traceback.format_exc() + await self.send_notification_async( + metadata=self.metadata, + notification_id=f'MONGO_PRODUCER_ERROR_{topic}', + message=f'Error inserting message to MongoDB: {e}', + block='mongo_producer', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + self.logger.error(trace) diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py new file mode 100644 index 0000000..34a09ca --- /dev/null +++ b/ingestor/managers/ingestor_manager.py @@ -0,0 +1,683 @@ +import asyncio +import traceback +from copy import deepcopy + +from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler +from sientia_do.notifications.models import NotificationLevel +from sientia_do.observability.logger import Logger +from sientia_do.observability.metrics_controller import MetricsController +from sientia_do.observability.sientia_monitoring import SientiaMonitoring + +import ingestor.metrics as metrics +from ingestor.managers.data_manager import DataManager +from ingestor.managers.opc_manager import OpcManager +from ingestor.managers.resource_manager import ResourceManager + + +class IngestorManager(SientiaMonitoring): + """ + Central coordinator for managing OPC data ingestion operations. + + The IngestorManager orchestrates the interaction between different components: + - DataManager: Handles data persistence + - OPC Managers: Manage individual OPC UA server connections + - ResourceManager: Coordinates slot leasing and load balancing + + This class implements a slot-based architecture where: + - Each slot represents a collection of OPC tags from one or more servers + - Slots are distributed across multiple ingestor instances for load balancing + - Dynamic slot allocation ensures optimal resource utilization + + Key Responsibilities: + - Slot lease management and distribution + - OPC server connection lifecycle management + - Tag subscription coordination + - System health monitoring and integrity checks + - Load balancing across multiple ingestor instances + + Args: + redis_data (dict): Redis connection parameters (host, port, username, password) + lease_ttl (int): Time-to-live for slot leases in seconds + heartbeat_ttl (int): Time-to-live for heartbeat signals in seconds + poll_interval (int): Main loop polling interval in seconds + mongo_connection_string (str): MongoDB connection string + mongo_database (str): MongoDB database name + metadata (dict): Application metadata for notifications and tracking + logger (Logger): Logger instance for application logging + notification_handler (NotificationHandler): Handler for sending notifications + + Attributes: + data_manager (DataManager): Manages data persistence + opc_managers (dict): Dictionary of OPC managers keyed by server name + resource_manager (ResourceManager): Manages Redis-based resource coordination + number_of_slots (int): Total number of slots configured in the system + poll_interval (int): Main loop polling interval + managed_tags (dict): Currently managed tags organized by slot + opc_servers (dict): OPC server configurations + metadata (dict): Application metadata + """ + + def __init__( + self, + redis_data: dict, + lease_ttl: int, + heartbeat_ttl: int, + poll_interval: int, + mongo_connection_string: str, + mongo_database: str, + metadata: dict, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ): + redis_host: str = redis_data['host'] + redis_port: int = int(redis_data['port']) + redis_username: str | None = redis_data.get('username', None) + redis_password: str | None = redis_data.get('password', None) + + SientiaMonitoring.__init__( + self, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + self.data_manager = DataManager( + mongo_connection_string=mongo_connection_string, + mongo_database=mongo_database, + metadata=metadata, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + self.opc_managers: dict = {} + self.resource_manager = ResourceManager( + host=redis_host, + port=redis_port, + lease_ttl=lease_ttl, + heartbeat_ttl=heartbeat_ttl, + metadata=metadata, + logger=logger, + notification_handler=notification_handler, + username=redis_username, + password=redis_password, + metrics_controller=metrics_controller, + ) + self.number_of_slots = 0 + self.poll_interval = poll_interval + self.managed_tags: dict = {} + self.opc_servers: dict = {} + + self.metadata = metadata + + async def initialize_opc_from_config(self, server_config: dict) -> OpcManager | None: + """ + Initializes an OPC Manager instance using the provided server configuration. + + This method creates and configures an OPC Manager for a specific OPC UA server, + establishing the connection and preparing it for tag subscriptions. + + Args: + server_config (dict): A dictionary containing the OPC server configuration. + Expected keys include: + - 'name' (str): The name of the OPC server. + - 'url' (str): The URL of the OPC server. + - 'server_uri' (str): The URI of the OPC server. + - 'cert_path' (str, optional): Path to the client certificate file. + - 'private_key_path' (str, optional): Path to the private key file. + - 'server_cert_path' (str, optional): Path to the server certificate file. + + Returns: + OpcManager | None: An initialized OpcManager instance if successful, + otherwise None if an error occurs during initialization. + + Raises: + Exception: If OPC manager initialization fails, the error is logged and + a notification is sent, but the method returns None to allow + the system to continue operating with other servers. + """ + + try: + self.logger.info(f'Initializing OpcManager at {server_config["url"]}') + manager = OpcManager( + name=server_config['name'], + url=server_config['url'], + subscription_period_ms=server_config['subscription_period_ms'], + data_manager=self.data_manager, + logger=self.logger, + server_uri=server_config['server_uri'], + notification_handler=self.notification_handler, + metadata=self.metadata, + cert_path=server_config.get('cert_path'), + private_key_path=server_config.get('private_key_path'), + server_cert_path=server_config.get('server_cert_path'), + metrics_controller=self.metrics_controller, + ) + + manager.config = server_config + await manager.connect() + except Exception as e: + trace = traceback.format_exc() + self.send_notification( + metadata=self.metadata, + notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}', + message=f'Error initializing OPC manager: {e}', + block='opc_manager', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + self.logger.error(trace) + + return None + + return manager + + async def shutdown(self): + """ + Gracefully shuts down the IngestorManager and all its components. + + This method ensures proper cleanup of: + - All OPC manager instances and their connections + - Data manager connections and resources + - Active subscriptions and server connections + + The shutdown process is performed asynchronously to allow proper cleanup + of all managed resources before termination. + """ + for _server_name, server in self.opc_managers.items(): + await server.shutdown() + self.data_manager.shutdown() + + def __del__(self): + asyncio.run(self.shutdown()) + + async def remove_server(self, server: str): + """ + Removes an OPC server from the ingestor. + """ + if server in self.opc_managers: + await self.opc_managers[server].shutdown() + del self.opc_managers[server] + for slot, _config in self.managed_tags.items(): + self.managed_tags[slot].pop(server, None) + + async def update_opc_servers(self): + """ + Updates the OPC (OLE for Process Control) server connections managed by the ingestor. + This method ensures that the OPC servers defined in `self.managed_tags` are properly + initialized and updated. It performs the following tasks: + - Registers new OPC servers based on the configuration in `self.managed_tags`. + - Updates existing OPC server instances if their configuration has changed. + - Disconnects and removes OPC servers that are no longer present in `self.managed_tags`. + Steps: + 1. Iterates through the `self.managed_tags` dictionary to identify and register servers. + 2. Initializes new OPC server instances if they are not already managed. + 3. Reinitializes OPC server instances if their configuration has changed. + 4. Disconnects and removes OPC servers that are no longer registered. + Attributes: + self.managed_tags (dict): A nested dictionary containing slot and server configurations. + self.opc_managers (dict): A dictionary mapping server names to their + OPC manager instances. + self.data_manager: An object responsible for managing data operations. + self.logger: A logging object for recording warnings and other messages. + Raises: + Any exceptions raised during OPC server initialization or disconnection. + Logs: + - Warnings for servers that are no longer found in `self.managed_tags`. + """ + + registered_servers = [] + current_managed_tags = deepcopy(self.managed_tags) + for _slot, slot_config in current_managed_tags.items(): + for server, server_config in slot_config.items(): + registered_servers.append(server) + server_config = deepcopy(server_config) + server_config.pop('tags', None) + server_instance = self.opc_managers.get(server, None) + if server_instance is None: + self.logger.info(f'Initializing OPC manager for server {server}') + server_instance = await self.initialize_opc_from_config(server_config) + + elif server_instance.config != server_config: + self.logger.warning(f'Reinitializing OPC manager for server {server}') + await server_instance.shutdown() + del self.opc_managers[server] + server_instance = await self.initialize_opc_from_config(server_config) + else: + self.logger.debug( + f'OPC manager for server {server} is already initialized and up to date' + ) + + if server_instance is not None: + self.opc_managers[server] = server_instance + else: + self.logger.warning( + f'Failed to initialize OPC manager for server {server}, ' + f'removing server from managed tags.' + ) + + await self.remove_server(server) + + servers = list(self.opc_managers.keys()) + for server in servers: + if server not in registered_servers: + self.logger.warning( + f'Server {server} not found in managed tags. Desconnecting from server.' + ) + await self.remove_server(server) + + await self.emit_metric( + metric_object=metrics.OPC_MANAGERS_ACTIVE, + method='set', + value=len(self.opc_managers), + tags={ + 'pod_id': self.pod_id, + }, + ) + + async def check_opc_servers_integrity(self): + """ + Checks the integrity of the OPC servers and updates the OPC servers if necessary. + + This method performs health checks on all managed OPC servers by: + - Checking cycle counts for data reception + - Monitoring connection health and data flow + - Triggering reconnection for lost servers + - Updating metrics for active OPC managers + + Side Effects: + - Updates cycle monitoring for all nodes + - Removes lost servers from managed tags + - Updates OPC manager metrics + """ + to_disconnect: list[str] = [] + for server, opc_manager in self.opc_managers.items(): + await opc_manager.check_cycles() + + is_lost = await opc_manager.check_opc_listenning() + if is_lost: + self.logger.warning(f'OPC server {server} is lost. Server will be disconnected.') + + to_disconnect.append(server) + + for server in to_disconnect: + await self.remove_server(server) + + await self.emit_metric( + metric_object=metrics.OPC_MANAGERS_ACTIVE, + method='set', + value=len(self.opc_managers), + tags={ + 'pod_id': self.pod_id, + }, + ) + + async def declare_active(self): + """ + Declares the ingestor as active by sending a heartbeat signal to the resource manager. + + This method ensures that the ingestor is marked as active by invoking the + `ingestor_heartbeat` method of the associated resource manager. The heartbeat + mechanism enables load balancers and monitoring systems to track active instances. + + Side Effects: + - Updates Redis with current instance heartbeat + - Enables load balancing and health monitoring + """ + + await self.resource_manager.ingestor_heartbeat() + + async def get_active_ingestors(self) -> list[str]: + """ + Retrieve a list of active ingestors. + + This method fetches all ingestors from the resource manager and returns them. + If no ingestors are found, an empty list is returned. + + Returns: + List[str]: A list of active ingestor names, or an empty list if none are found. + + The method queries Redis for all active ingestor heartbeats and extracts + the pod identifiers for load balancing and coordination purposes. + """ + + ingestors = await self.resource_manager.get_all_ingestors() + return ingestors if ingestors else [] + + async def get_number_of_leases(self) -> int: + """ + Retrieves the number of leases managed by the resource manager. + + This method fetches all available leases from the resource manager, + calculates their count, and updates the `number_of_slots` attribute. + + Returns: + int: The total number of leases. Returns 0 if no leases are available. + + Side Effects: + - Updates internal slot count tracking + - Updates Prometheus metrics for total leases + """ + + leases = await self.resource_manager.get_all_leases() + self.number_of_slots = len(leases) if leases else 0 + await self.emit_metric( + metric_object=metrics.LEASES_TOTAL, + method='set', + value=self.number_of_slots, + tags={ + 'pod_id': self.pod_id, + }, + ) + return self.number_of_slots + + async def get_number_of_slots(self) -> int: + """ + Retrieves the number of slots managed by the resource manager. + + This method fetches all available slots from the resource manager, + calculates their count, and updates the `number_of_slots` attribute. + + Returns: + int: The total number of slots. Returns 0 if no slots are available. + + Side Effects: + - Updates internal slot count tracking + - Updates Prometheus metrics for total slots + """ + + slots = await self.resource_manager.get_all_slots() + self.number_of_slots = len(slots) if slots else 0 + await self.emit_metric( + metric_object=metrics.SLOTS_TOTAL, + method='set', + value=self.number_of_slots, + tags={ + 'pod_id': self.pod_id, + }, + ) + return self.number_of_slots + + async def get_slot_leases(self, max_slots: int = 1) -> dict: + """ + Acquires a specified number of resource slots by leasing them from the resource manager. + + This method implements the slot acquisition logic for load balancing: + - Iterates through available slots and attempts to lease them + - Logs the leasing of each slot + - Updates the `managed_tags` attribute with the acquired slots + - Stops leasing once the specified `max_slots` are acquired + + Args: + max_slots (int): The maximum number of slots to lease. Defaults to 1. + + Returns: + Dict: A dictionary where the keys are the slot identifiers (as strings) + and the values are the leased slot details. + + Behavior: + - Attempts to lease slots sequentially starting from slot 1 + - Skips slots that cannot be retrieved after leasing + - Logs warnings if unable to acquire the requested number of slots + - Updates metrics for acquired slots and managed slots count + + Notes: + - If a slot is leased but its details cannot be retrieved + (i.e., `get_tag_slot` returns None), that slot is skipped. + """ + + acquired = {} + for i in range(1, self.number_of_slots + 1): + if await self.resource_manager.lease_tag(str(i)): + self.logger.info(f'Leased slot {i}') + slots = await self.resource_manager.get_tag_slot(str(i)) + if slots is None: + continue + acquired[str(i)] = slots + await self.emit_metric( + metric_object=metrics.SLOTS_ACQUIRED, + method='inc', + value=1, + tags={ + 'pod_id': self.pod_id, + }, + ) + + if len(acquired) >= max_slots: + self.managed_tags.update(acquired) + await self.emit_metric( + metric_object=metrics.SLOTS_MANAGED, + method='set', + value=len(self.managed_tags), + tags={ + 'pod_id': self.pod_id, + }, + ) + return acquired + + self.logger.warning( + f'Unable to acquire {max_slots} slots. Only {acquired} slots were leased.' + ) + self.managed_tags.update(acquired) + await self.emit_metric( + metric_object=metrics.SLOTS_MANAGED, + method='set', + value=len(self.managed_tags), + tags={ + 'pod_id': self.pod_id, + }, + ) + return acquired + + async def unsubscribe_slot(self, slot: str): + """ + Unsubscribes a specific slot from all associated OPC servers. + + This method removes all subscriptions for a given slot across all + OPC servers that were managing it. It ensures clean cleanup of + resources when slots are released or reconfigured. + + Args: + slot (str): The name of the slot to unsubscribe. + + Raises: + KeyError: If the specified slot does not exist in the managed tags. + + Side Effects: + - Removes subscriptions from all OPC servers for the specified slot + - Cleans up subscription resources on the OPC servers + """ + + for server in self.managed_tags[slot].keys(): + if server in self.opc_managers: + await self.opc_managers[server].unsubscribe(slot) + + async def update_slot_config(self): + """ + Updates the configuration of managed slots by renewing their leases, + fetching the latest configurations, and handling any changes or removals. + + This method performs the following steps: + 1. Renews the lease for each managed slot using the resource manager. + 2. Fetches the latest configuration for each slot. + 3. Logs and removes slots whose configurations are no longer available. + 4. Updates the configuration of slots if changes are detected. + 5. Unsubscribes and re-subscribes to slots with updated configurations. + 6. Removes slots from the managed tags if they are no longer valid. + 7. Updates the OPC servers after processing all slots. + + Side Effects: + - Modifies the `managed_tags` dictionary to reflect the latest slot configurations. + - Updates OPC server subscriptions based on the current state of managed slots. + + Raises: + None explicitly, but relies on the behavior of `resource_manager` and + other dependencies for error handling. + + Logging: + - Logs warnings for removed slots. + - Logs informational messages for updated slot configurations. + """ + + removed_slots: list[str] = [] + for slot, _slot_config in self.managed_tags.items(): + await self.resource_manager.renew_tag_lease(slot) + update = await self.resource_manager.get_tag_slot(slot) + if update is None: + removed_slots.append(slot) + continue + + self.managed_tags[slot] = update + + for slot in removed_slots: + self.managed_tags.pop(slot, None) + + async def drop_slot_leases(self, ids: list[str]) -> None: + """ + Releases the leases associated with the specified slot IDs. + + This method iterates through a list of slot IDs and calls the + `drop_tag_lease` method of the `resource_manager` to release + the lease for each ID. It's used during load balancing and + graceful shutdown scenarios. + + Args: + ids (List[str]): A list of slot IDs for which the leases + should be released. + + Returns: + None + + Side Effects: + - Releases Redis-based leases for specified slots + - Updates metrics for released slots count + """ + for lease_id in ids: + await self.resource_manager.drop_tag_lease(lease_id) + await self.emit_metric( + metric_object=metrics.SLOTS_RELEASED, + method='inc', + value=1, + tags={ + 'pod_id': self.pod_id, + }, + ) + + async def manage_server(self, slot: str, server: str, server_config: dict, tags: dict) -> int: + """ + Manages the subscription of tags to a specified OPC server and slot. + + This method ensures that the specified server and slot have an active subscription + for the provided tags. If the server or slot is not properly configured, or if + subscription fails, appropriate error handling is performed. + + Args: + slot (str): The slot identifier for the subscription. + server (str): The name of the OPC server. + server_config (dict): Configuration dictionary for the server, which includes + the tags to be subscribed under the key 'tags'. + tags (dict): A dictionary of tags to be subscribed. + + Returns: + int: Status code indicating the result of the operation: + - 0: Subscription was successful. + - 1: Server not found in `opc_managers`. + - 2: Subscription creation or tag subscription failed. + + Logs: + - Logs informational messages about the subscription process. + - Logs errors if the server is not found, subscription creation fails, or + tag subscription fails. + - Logs a warning if a subscription is removed due to failure. + + Raises: + Exception: Any unexpected exceptions during subscription creation or tag + subscription are logged but not propagated. + + Side Effects: + - Creates or updates OPC subscriptions + - Manages tag subscriptions on OPC servers + - Updates error metrics and notifications + """ + + self.logger.info(f'Subscribing to tags from {slot}:{server}') + tags_to_sub = server_config.get('tags') + if server not in self.opc_managers: + self.logger.error(f'Server {server} not found in opc_managers.') + return 1 + if slot not in self.opc_managers[server].subscriptions: + try: + await self.opc_managers[server].create_subscription(slot) + except Exception as e: + self.logger.error(f'Failed to create subscription for slot {slot}: {e}') + return 2 + try: + self.logger.info(tags_to_sub) + await self.opc_managers[server].subscribe( + slot, deepcopy(tags_to_sub), self.poll_interval + ) + self.logger.info(tags_to_sub) + except Exception as e: + await self.emit_metric( + metric_object=metrics.OPC_SUBSCRIPTION_ERRORS, + tags={ + 'pod_id': self.pod_id, + 'server': server, + 'slot': slot, + }, + ) + trace = traceback.format_exc() + await self.send_notification_async( + metadata=self.metadata, + notification_id=f'OPC_SUBSCRIPTION_ERROR_{slot}:{server}', + message=f'Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}', + block='opc_manager', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + self.logger.error(trace) + + self.logger.warning(f'Removing subscription from server {server} for slot {slot}') + await self.opc_managers[server].unsubscribe(slot) + return 2 + return 0 + + async def subscribe_to_tags(self, tags: dict) -> None: + """ + Subscribes to a set of tags and manages their configurations. + + This method processes a dictionary of tags, iterating through each slot and server + configuration. It attempts to manage the server configurations and removes any + servers that return a specific response code. + + Args: + tags (Dict): A dictionary containing tag configurations. The structure is + expected to be {slot: {server: server_config}}. + + Side Effects: + - Logs the provided tags for debugging purposes. + - Updates the `managed_tags` attribute by removing servers that meet the + removal criteria. + - Establishes OPC subscriptions for all configured tags. + + Removal Criteria: + - If the `manage_server` method returns a response code of 2 for a given + slot and server, that server is removed from the `managed_tags` attribute. + + The method ensures that only successfully configured servers remain in the + managed tags, maintaining system stability and preventing subscription errors. + """ + + to_remove = [] + self.logger.info(tags) + for slot, slot_config in tags.items(): + for server, server_config in slot_config.items(): + response = await self.manage_server(slot, server, server_config, tags) + if response == 2: + to_remove.append([slot, server]) + + for slot, server in to_remove: + if server in self.opc_managers: + await self.opc_managers[server].shutdown() + del self.opc_managers[server] + self.managed_tags[slot].pop(server, None) diff --git a/ingestor/managers/opc_manager.py b/ingestor/managers/opc_manager.py new file mode 100644 index 0000000..3505757 --- /dev/null +++ b/ingestor/managers/opc_manager.py @@ -0,0 +1,591 @@ +import asyncio +import json +import traceback +from pathlib import Path + +from asyncua import Client +from asyncua.crypto.security_policies import SecurityPolicyBasic256 +from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler +from sientia_do.notifications.models import NotificationLevel +from sientia_do.observability.logger import Logger +from sientia_do.observability.metrics_controller import MetricsController +from sientia_do.observability.sientia_monitoring import SientiaMonitoring +from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, OPC_TIMEZONE + +import ingestor.metrics as metrics +from ingestor.managers.data_manager import DataManager + + +class OpcManager(SientiaMonitoring): + """ + Manages OPC UA server connections and tag subscriptions. + + The OpcManager is responsible for: + - Establishing and maintaining secure connections to OPC UA servers + - Managing tag subscriptions and data collection + - Handling server reconnection and error recovery + - Processing OPC data and forwarding it to the data manager + - Monitoring connection health and performance metrics + + The manager supports both secure and unsecured connections, with optional + certificate-based authentication for enhanced security. + + Args: + name (str): Unique identifier for the OPC server + url (str): OPC UA server endpoint URL + data_manager (DataManager): Manager for data persistence and export + logger (Logger): Logger instance for application logging + server_uri (str): OPC UA server application URI + notification_handler (NotificationHandler): Handler for sending notifications + metadata (dict): Application metadata for notifications and tracking + cert_path (str, optional): Path to client certificate file for secure connections + private_key_path (str, optional): Path to client private key file + server_cert_path (str, optional): Path to server certificate file for validation + + Attributes: + url (str): OPC UA server endpoint URL + name (str): Unique identifier for the OPC server + server_uri (str): OPC UA server application URI + data_queue (dict): Queue for buffering OPC data before processing + non_receive_count (int): Counter for cycles without data reception + client (Client): OPC UA client instance + cert_path (str): Path to client certificate file + private_key_path (str): Path to client private key file + server_cert_path (str): Path to server certificate file + nodes (dict): Dictionary of OPC node references + subscriptions (dict): Active OPC subscriptions + data_manager (DataManager): Manager for data persistence and export + metadata (dict): Application metadata + """ + + def __init__( + self, + name: str, + url: str, + subscription_period_ms: int, + data_manager: DataManager, + logger: Logger, + server_uri: str, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + metadata: dict, + cert_path: str | None = None, + private_key_path: str | None = None, + server_cert_path: str | None = None, + ): + self.url = url + self.name = name + self.server_uri = server_uri + self.data_queue: dict = {} + self.non_receive_count = 0 + self.client: Client | None = None + self.subscription_period_ms = subscription_period_ms + self.cert_path = cert_path + self.private_key_path = private_key_path + self.server_cert_path = server_cert_path + self.nodes: dict = {} + self.subscriptions: dict = {} + self.data_manager = data_manager + self.metadata = metadata + + SientiaMonitoring.__init__( + self, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + metrics.OPC_CONNECTION_STATUS.labels( + pod_id=self.pod_id, server_name=self.name, server_url=self.url + ).set(0) + metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(0) + + def __str__(self): + """ + String representation of the OPC Manager. + + Returns: + str: Human-readable representation showing server details and current state. + """ + return ( + f'OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n' + f'nodes={self.nodes}, subscriptions={self.subscriptions}' + ) + + def __del__(self): + asyncio.run(self.shutdown()) + + async def shutdown(self): + """ + Comprehensive cleanup method for graceful shutdown. + + This method ensures proper cleanup of all OPC UA resources: + - Closes active subscriptions + - Disconnects from the OPC server + - Releases allocated resources + + Should be called before the application terminates to prevent resource leaks + and ensure clean disconnection from OPC servers. + """ + try: + await self.disconnect() + except Exception as e: + self.logger.error(f'Error during cleanup: {e}') + + async def set_security(self): + """ + Configures the security settings for the OPC UA client. + + This method sets up the security policy, certificates, and timeouts + required for establishing a secure connection with the OPC UA server. + It implements Basic256 security policy with certificate-based authentication. + + Raises: + ValueError: If either the certificate path or private key path is not provided. + + Security Settings: + - Security Policy: Basic256 + - Secure Channel Timeout: 10,000,000 ms + - Session Timeout: 10,000,000 ms + + The method configures: + - Client application URI + - Certificate-based authentication + - Server certificate validation (if provided) + - Connection timeouts for stability + """ + + if not all([self.cert_path, self.private_key_path]): + raise ValueError( + 'Certificate and private key paths must be provided for secure connection.' + ) + cert = str(Path(self.cert_path)) if self.cert_path else None + private_key = str(Path(self.private_key_path)) if self.private_key_path else None + server_cert = str(Path(self.server_cert_path)) if self.server_cert_path else None + assert cert is not None and private_key is not None + + if self.client: + self.client.application_uri = self.server_uri + self.logger.info('Setting security...') + await self.client.set_security( + SecurityPolicyBasic256, + certificate=cert, + private_key=private_key, + server_certificate=server_cert, + ) + self.client.secure_channel_timeout = 10000000 + self.client.session_timeout = 10000000 + + async def connect(self): + """ + Establishes a connection to the OPC server. + + This method initializes the OPC client using the provided URL and + sets up security if a certificate path is specified. It then + attempts to connect to the server and logs the connection status. + + The connection process includes: + 1. Client initialization with server URL + 2. Security configuration (if certificates are provided) + 3. Connection establishment + 4. Metrics recording for monitoring + + Raises: + Exception: If the connection to the OPC server fails. + + Metrics: + - OPC_CONNECTIONS_TOTAL: Incremented on connection attempt + - OPC_CONNECTION_STATUS: Set to 1 on successful connection + """ + + await self.emit_metric( + metric_object=metrics.OPC_CONNECTIONS_TOTAL, + tags={ + 'pod_id': self.pod_id, + 'server_name': self.name, + }, + ) + try: + self.client = Client(self.url, timeout=10, watchdog_intervall=3600000) + assert self.client is not None # Informa ao mypy que client não é None + + self.client.name = self.pod_id + self.client.description = self.pod_id + pod_uri = self.pod_id.replace('-', ':') + self.client.application_uri = pod_uri + self.client.product_uri = pod_uri + + if self.cert_path: + await self.set_security() + self.logger.info(f'Starting connection to {self.name}...') + await self.client.connect() + await self.emit_metric( + metric_object=metrics.OPC_CONNECTION_STATUS, + method='set', + value=1, + tags={ + 'pod_id': self.pod_id, + 'server_name': self.name, + 'server_url': self.url, + }, + ) + self.logger.info(f'Connection to {self.name} successful.') + except Exception: + await self.disconnect() + + raise + + async def create_subscription(self, name: str): + """ + Creates a subscription with the specified monitoring period. + + This method establishes a subscription to monitor data changes or events + from the OPC UA server. If the client is not connected, an exception is raised. + + Args: + name (str): The name identifier for the subscription + period (int, optional): The monitoring period in milliseconds. Defaults to 500 ms. + + Raises: + ValueError: If the client is not connected. + + Side Effects: + - Sets the `self.period` attribute to the specified or default period. + - Creates a subscription and assigns it to `self.subscriptions[name]`. + - Logs the creation of the subscription. + - Increments subscription creation metrics. + """ + + if not self.client: + raise ValueError('Client not connected. Call connect first.') + try: + self.subscriptions[name] = await self.client.create_subscription( + self.subscription_period_ms, self + ) + self.logger.info(f'Subscription {name} created on {self.name}.') + await self.emit_metric( + metric_object=metrics.OPC_SUBSCRIPTIONS_CREATED, + method='inc', + value=1, + tags={ + 'pod_id': self.pod_id, + 'server_name': self.name, + 'slot_name': name, + }, + ) + except Exception as e: + self.logger.error(f'Failed to create subscription {name} on {self.name}: {e}') + raise + + async def subscribe(self, subscription: str, nodes: dict, collect_period: int): + """ + Subscribes to a set of OPC UA nodes for data change notifications. + + This method adds the specified nodes to the subscription and configures + their data collection rules based on the provided collection period and + node-specific frequency. + + Args: + subscription (str): The name of the subscription to use + nodes (dict): A dictionary where keys are node identifiers and values are + configurations for each node. Each configuration must include a 'frequency' + key indicating the frequency of data collection in Hz. + collect_period (int): The data collection period in seconds. + + Raises: + ValueError: If the subscription has not been created by calling + `create_subscription` prior to this method. + + Side Effects: + - Updates internal node tracking and cycle rules + - Establishes data change monitoring for specified nodes + - Updates metrics for subscribed tags count + """ + + if not self.subscriptions.get(subscription): + raise ValueError('Subscription not created. Call create_subscription first.') + + self.logger.info(f'Subscribing to {subscription} on {self.name}...') + self.logger.info(f'Subscribing to nodes: {nodes}') + assert self.client is not None # Informa ao mypy que client não é None + addr_nodes = [self.client.get_node(n) for n in nodes] + self.logger.debug(f'Addr nodes: {addr_nodes}') + self.nodes.update(nodes) + self.logger.debug(f'Nodes: {self.nodes}') + + self.collect_period = collect_period + + for node, config in self.nodes.items(): + self.nodes[node]['cycle_rule'] = { + 'cycle_increment': collect_period * 1000 / float(config['frequency']), + 'cycle_count': 0, + } + + await self.subscriptions[subscription].subscribe_data_change(addr_nodes) + + await self.emit_metric( + metric_object=metrics.OPC_TAGS_SUBSCRIBED, + method='set', + value=len(self.nodes), + tags={ + 'pod_id': self.pod_id, + 'server_name': self.name, + }, + ) + + async def unsubscribe(self, subscription: str): + """ + Unsubscribes from a given subscription. + + This method removes the specified subscription and cleans up associated + resources. It handles cases where the subscription doesn't exist gracefully. + + Args: + subscription (str): The name of the subscription to unsubscribe from. + + Logs: + - A warning if the specified subscription does not exist. + - An info message upon successful unsubscription. + + Behavior: + - If the subscription exists, it is deleted and removed from the + subscriptions dictionary. + - If the subscription does not exist, no action is taken. + """ + + if not self.subscriptions.get(subscription): + self.logger.warning(f"Subscription '{subscription}' not found. Cannot unsubscribe.") + return + await self.subscriptions[subscription].delete() + del self.subscriptions[subscription] + self.logger.info(f'Unsubscribed from {subscription}.') + + async def disconnection_fallback(self) -> list: + """ + Tries 5 times to disconnect from the OPC UA server, with a delay of 100ms x try. + """ + + assert self.client is not None + error_stack = [] + for i in range(5): + try: + self.logger.info(f'Disconnecting from OPC UA server, attempt {i + 1} of 5') + await self.client.disconnect() + return [] + except Exception as e: + self.logger.error( + f'Failed to disconnect from OPC UA serve in attempt {i + 1} of 5: {e}' + ) + error_stack.append( + { + 'attempt': i + 1, + 'error': str(e), + 'traceback': traceback.format_exc(), + } + ) + await asyncio.sleep(0.1 * i) + return error_stack + + async def disconnect(self): + """ + Disconnects from the OPC UA server. + + This method handles the disconnection process by deleting all subscriptions + and disconnecting the client from the OPC UA server. It logs the disconnection + process and handles any exceptions that may occur during cleanup. + + Raises: + Exception: If an error occurs while deleting the subscription or disconnecting + from the OPC UA server, it logs the error details. + + Side Effects: + - Deletes all active subscriptions + - Disconnects the OPC client + - Updates connection status metrics + - Clears internal client reference + """ + + self.logger.warning('Disconnecting from OPC server') + if self.client is None: + self.logger.warning('Client already disconnected.') + return + try: + for sub in self.subscriptions: + await self.subscriptions[sub].delete() + self.logger.warning('Deleted all subscriptions.') + except Exception as sub_error: + self.logger.error(f'Failed to clean up subscription: {sub_error}') + + errors = await self.disconnection_fallback() + + if errors: + await self.send_notification_async( + metadata=self.metadata, + notification_id=f'OPC_DISCONNECTION_ERROR_{self.name}', + message=f'Failed to disconnect from OPC UA server {self.name} after 5 attempts', + block='opc_manager', + level=NotificationLevel.ERROR, + attachment_content=json.dumps(errors, indent=4), + ) + else: + self.logger.warning('Disconnected from OPC UA server.') + + del self.client + self.client = None + await self.emit_metric( + metric_object=metrics.OPC_CONNECTION_STATUS, + method='set', + value=0, + tags={ + 'pod_id': self.pod_id, + 'server_name': self.name, + 'server_url': self.url, + }, + ) + await self.emit_metric( + metric_object=metrics.OPC_TAGS_SUBSCRIBED, + method='set', + value=0, + tags={ + 'pod_id': self.pod_id, + 'server_name': self.name, + }, + ) + + async def datachange_notification(self, node, _val, data): + """ + Handles data change notifications for monitored OPC UA nodes. + + This method is triggered when a monitored node's value changes. It processes + the notification, updates internal state, and publishes the data to the + appropriate topics. + + Args: + node (NodeId): The OPC UA node that triggered the data change notification. + _val (Any): The new value of the node (unused in this implementation). + data (DataChangeNotification): The data change notification object containing + details about the change. + + Behavior: + - Extracts the value and source timestamp from the monitored item. + - Resets the cycle count for the node's cycle rule. + - Resets the non-receive count. + - Constructs a data dictionary containing the tag, tag name, timestamp, and value. + - Publishes the data to all topics associated with the node. + """ + + # get data value + monitored_item = data.monitored_item + value = monitored_item.Value.Value.Value + # source_timestamp + source_timestamp = monitored_item.Value.SourceTimestamp.replace(tzinfo=OPC_TIMEZONE) + tag = str(node) + + self.logger.debug( + f'Data change notification received for tag:' + f'{tag} after {self.nodes[tag]["cycle_rule"]["cycle_count"]} cycles' + ) + + data = { + 'tag': tag, + 'name': self.nodes[str(node)]['tag_name'], + 'timestamp': source_timestamp.strftime(DATETIME_FORMAT_WITH_TZ), + 'value': value, + } + + for topic in self.nodes[tag]['topics']: + await self.data_manager.publish(topic, data) + + self.nodes[tag]['cycle_rule']['cycle_count'] = 0 + self.non_receive_count = 0 + await self.emit_metric( + metric_object=metrics.OPC_CYCLES_WITHOUT_DATA, + method='set', + value=0, + tags={ + 'pod_id': self.pod_id, + 'server_name': self.name, + }, + ) + + async def check_cycles(self): + """ + Checks the cycle counts for all monitored nodes and sends + notifications if thresholds are exceeded. + + This method iterates through all monitored nodes and updates their cycle counts based on + configured increments. If a node's cycle count exceeds a threshold (5 cycles), it triggers + a warning notification. + + Side Effects: + - Updates cycle counts for all monitored nodes + - Sends warning notifications for nodes exceeding cycle thresholds + """ + for node, config in self.nodes.items(): + self.nodes[node]['cycle_rule']['cycle_count'] += config['cycle_rule']['cycle_increment'] + if self.nodes[node]['cycle_rule']['cycle_count'] >= 5: + name = config['tag_name'] + cycles = self.nodes[node]['cycle_rule']['cycle_count'] + await self.send_notification_async( + metadata=self.metadata, + notification_id=f'TAG_{node}:{name}_LISTENNING_STOPPED', + message=f'{cycles} cycles without receive from {node}:{name}', + block='opc_manager', + level=NotificationLevel.WARNING, + ) + + async def check_opc_listenning(self) -> bool: + """ + Checks the OPC connection and triggers notifications if the connection is lost. + + This method monitors the data reception health by tracking cycles without + data. It sends notifications at different thresholds and can trigger + reconnection attempts. + + Returns: + bool: True if the connection is lost and reconnection should be attempted, + False otherwise. + + Side Effects: + - Increments non-receive count + - Updates metrics for cycles without data + - Sends warning notifications at 5 cycles + - Sends error notifications and triggers reconnection at 15 cycles + """ + + self.non_receive_count += 1 + await self.emit_metric( + metric_object=metrics.OPC_CYCLES_WITHOUT_DATA, + method='set', + value=self.non_receive_count, + tags={ + 'pod_id': self.pod_id, + 'server_name': self.name, + }, + ) + if self.non_receive_count >= 5: + await self.send_notification_async( + metadata=self.metadata, + notification_id=f'OPC_LISTENNING_STOPPED__{self.name}', + message=f'{self.non_receive_count} cycles without ' + f'receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}', + block='opc_manager', + level=NotificationLevel.ERROR, + ) + if self.non_receive_count >= 15: + await self.emit_metric( + metric_object=metrics.OPC_RECONNECTIONS_TOTAL, + tags={ + 'pod_id': self.pod_id, + 'server_name': self.name, + }, + ) + await self.send_notification_async( + metadata=self.metadata, + notification_id=f'OPC_CONNECTION_RETRY__{self.name}', + message=f'Retrying to connect to server {self.name}', + block='opc_manager', + level=NotificationLevel.ERROR, + ) + return True + return False diff --git a/ingestor/managers/resource_manager.py b/ingestor/managers/resource_manager.py new file mode 100644 index 0000000..50cbc17 --- /dev/null +++ b/ingestor/managers/resource_manager.py @@ -0,0 +1,315 @@ +from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler +from sientia_do.observability.logger import Logger +from sientia_do.observability.metrics_controller import MetricsController +from sientia_do.observability.sientia_monitoring import SientiaMonitoring +from sientia_do.repository.redis_repository import RedisRepository + + +class ResourceManager(SientiaMonitoring): + """ + Manages Redis-based resource coordination and slot leasing for the OPC Ingestor. + + The ResourceManager is responsible for: + - Coordinating slot allocation across multiple ingestor instances + - Managing lease lifecycles and heartbeats for load balancing + - Providing distributed locking and resource management + - Monitoring Redis operations and connection health + + The manager implements a sophisticated slot leasing system that enables: + - Dynamic load distribution across multiple ingestor instances + - Automatic failover and recovery from instance failures + - Fair resource allocation based on system capacity + - Real-time monitoring of system health and performance + + Args: + host (str): Redis server hostname + port (int): Redis server port + lease_ttl (int): Time-to-live for slot leases in seconds + heartbeat_ttl (int): Time-to-live for heartbeat signals in seconds + metadata (dict): Application metadata for notifications and tracking + logger (Logger): Logger instance for application logging + notification_handler (NotificationHandler): Handler for sending notifications + username (str, optional): Redis username for authentication + password (str, optional): Redis password for authentication + + Attributes: + redis (Redis): Redis client instance + lease_ttl (int): Time-to-live for slot leases + heartbeat_ttl (int): Time-to-live for heartbeat signals + metadata (dict): Application metadata + """ + + def __init__( + self, + host: str, + port: int, + lease_ttl: int, + heartbeat_ttl: int, + metadata: dict, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + username: str | None = None, + password: str | None = None, + ) -> None: + """ + Initializes the ResourceManager with Redis connection and configuration. + + This constructor establishes a connection to Redis and verifies connectivity + by performing a ping operation. It sets up the connection with optional + authentication and records the connection status in metrics. + + Args: + host (str): Redis server hostname + port (int): Redis server port + lease_ttl (int): Time-to-live for slot leases in seconds + heartbeat_ttl (int): Time-to-live for heartbeat signals in seconds + metadata (dict): Application metadata + logger (Logger): Logger instance + notification_handler (NotificationHandler): Notification handler + username (str, optional): Redis username for authentication + password (str, optional): Redis password for authentication + + Raises: + Exception: If Redis connection fails, the error is logged and metrics + are updated before re-raising the exception. + + Metrics: + - REDIS_CONNECTION_STATUS: Set to 1 on successful connection, 0 on failure + """ + SientiaMonitoring.__init__( + self, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + try: + self.redis_repository = RedisRepository( + host=host, + port=port, + username=username, + password=password, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + self.redis_repository.redis_client.ping() + except Exception as e: + self.logger.error(f'Failed to connect to Redis: {e}') + raise + + self.lease_ttl = lease_ttl + self.heartbeat_ttl = heartbeat_ttl + self.metadata = metadata + + async def get_tag_slot(self, tag_id: str) -> dict | None: + """ + Retrieve the tag slot information for a given ID. + + This method constructs the Redis key for a tag slot and retrieves + the associated configuration information. + + Args: + id (str): The unique identifier of the tag slot to retrieve. + + Returns: + dict: A dictionary containing the tag slot information associated + with the given ID, or None if not found. + + The method constructs the key using the pattern "slot:opc_tags:{id}" + and delegates to the get() method for the actual Redis operation. + """ + + self.info(f'Getting tag slot for tag_id: {tag_id}', metadata=self.metadata) + + slot = await self.redis_repository.get(f'slot:opc_tags:{tag_id}', metadata=self.metadata) + self.info(f'Tag slot for tag_id: {tag_id} is: {slot}', metadata=self.metadata) + return slot + + async def ingestor_heartbeat(self) -> None: + """ + Sends a heartbeat signal to Redis to indicate that the ingestor is active. + + This method sets a key in Redis with a specific format that includes the + ingestor's pod ID. The key is set with a value of 1 and an expiration + time defined by `self.heartbeat_ttl`. This allows monitoring systems to + track the activity and health of the ingestor. + + The heartbeat mechanism enables: + - Load balancers to identify active ingestor instances + - Health monitoring systems to detect failed instances + - Automatic failover and recovery mechanisms + + Metrics: + - REDIS_OPERATIONS_TOTAL: Incremented with operation="set" + - REDIS_OPERATIONS_DURATION: Records timing for heartbeat operations + """ + + await self.redis_repository.set( + f'heartbeat:ingestor:{self.pod_id}', 1, ttl=self.heartbeat_ttl, metadata=self.metadata + ) + + async def lease_tag(self, tag_id: str) -> bool: + """ + Attempts to lease a tag by setting a key in Redis with a specified TTL. + + This method uses the Redis `SET` command with the `NX` option to ensure that + the key is only set if it does not already exist. The key is set with an + expiration time defined by `lease_ttl`. This implements a distributed + locking mechanism for tag allocation. + + Args: + tag_id (str): The unique identifier of the tag to be leased. + + Returns: + bool: True if the lease was successfully acquired, False if the tag + is already leased by another ingestor. + + The leasing mechanism ensures: + - Only one ingestor can process a specific tag at a time + - Automatic lease expiration prevents deadlocks + - Fair distribution of tags across available ingestor instances + + Metrics: + - REDIS_OPERATIONS_TOTAL: Incremented with operation="set_nx" + - REDIS_OPERATIONS_DURATION: Records timing for lease operations + """ + + return await self.redis_repository.set( + f'lease:opc_tags:{tag_id}', + self.pod_id, + ttl=self.lease_ttl, + nx=True, + metadata=self.metadata, + ) + + async def renew_tag_lease(self, tag_id: str) -> bool: + """ + Renews the lease for a specific OPC tag if the current pod holds the lease. + + This method checks if the current pod (identified by `self.pod_id`) holds + the lease for the given OPC tag. If so, it extends the lease by resetting + its expiration time in Redis to the configured lease TTL. + + Args: + tag_id (str): The identifier of the OPC tag whose lease is to be renewed. + + Returns: + bool: True if the lease was successfully renewed, False if the current + pod doesn't hold the lease or renewal failed. + + Lease renewal is essential for: + - Maintaining continuous tag processing without interruptions + - Preventing lease expiration during long-running operations + - Ensuring system stability and reliability + + Metrics: + - REDIS_OPERATIONS_TOTAL: Incremented with operation="get" and "expire" + - REDIS_OPERATIONS_DURATION: Records timing for renewal operations + """ + + current = await self.redis_repository.get( + f'lease:opc_tags:{tag_id}', metadata=self.metadata + ) + if current == self.pod_id: + await self.redis_repository.expire( + f'lease:opc_tags:{tag_id}', self.lease_ttl, metadata=self.metadata + ) + return True + return False + + async def drop_tag_lease(self, tag_id: str) -> None: + """ + Drops the lease for a specific OPC tag. + + This method removes the lease for the given OPC tag by deleting the + corresponding key in Redis. This is typically called when an ingestor + is shutting down or when it needs to release a tag for reallocation. + + Args: + tag_id (str): The identifier of the OPC tag whose lease is to be dropped. + + Lease dropping enables: + - Graceful shutdown of ingestor instances + - Dynamic reallocation of tags for load balancing + - Recovery from failed or unresponsive ingestor instances + + Metrics: + - REDIS_OPERATIONS_TOTAL: Incremented with operation="delete" + - REDIS_OPERATIONS_DURATION: Records timing for lease dropping operations + """ + + await self.redis_repository.delete(f'lease:opc_tags:{tag_id}', metadata=self.metadata) + + async def get_all_ingestors(self) -> list[str]: + """ + Retrieves all active ingestors from Redis. + + This method fetches all keys in Redis that match the pattern for ingestor + heartbeats and returns a list of active ingestor identifiers. The method + uses the pattern "heartbeat:ingestor:*" to find all active instances. + + Returns: + List[str]: A list of active ingestor identifiers, extracted from + the Redis keys by removing the "heartbeat:ingestor:" prefix. + + This information is used for: + - Load balancing calculations + - System health monitoring + - Resource allocation decisions + + Metrics: + - REDIS_OPERATIONS_TOTAL: Incremented with operation="keys" + - REDIS_OPERATIONS_DURATION: Records timing for ingestor discovery + """ + + return await self.redis_repository.keys('heartbeat:ingestor:*', metadata=self.metadata) + + async def get_all_slots(self) -> list[str]: + """ + Retrieves all available slots from Redis. + + This method fetches all keys in Redis that match the pattern for OPC tag + slots and returns a list of slot identifiers. The method uses the pattern + "slot:opc_tags:*" to find all configured slots. + + Returns: + List[str]: A list of slot identifiers, extracted from the Redis keys + by removing the "slot:opc_tags:" prefix. + + Slot information is used for: + - Resource allocation planning + - Load balancing across ingestor instances + - System capacity monitoring + + Metrics: + - REDIS_OPERATIONS_TOTAL: Incremented with operation="keys" + - REDIS_OPERATIONS_DURATION: Records timing for slot discovery + """ + + return await self.redis_repository.keys('slot:opc_tags:*', metadata=self.metadata) + + async def get_all_leases(self) -> list[str]: + """ + Retrieves all active leases from Redis. + + This method fetches all keys in Redis that match the pattern for OPC tag + leases and returns a list of lease identifiers. The method uses the pattern + "lease:opc_tags:*" to find all active leases. + + Returns: + List[str]: A list of lease identifiers, extracted from the Redis keys + by removing the "lease:opc_tags:" prefix. + + Lease information is used for: + - Current resource utilization monitoring + - Load balancing calculations + - System health and performance analysis + + Metrics: + - REDIS_OPERATIONS_TOTAL: Incremented with operation="keys" + - REDIS_OPERATIONS_DURATION: Records timing for lease discovery + """ + + return await self.redis_repository.keys('lease:opc_tags:*', metadata=self.metadata) diff --git a/ingestor/metrics.py b/ingestor/metrics.py new file mode 100644 index 0000000..cb18652 --- /dev/null +++ b/ingestor/metrics.py @@ -0,0 +1,141 @@ +""" +Prometheus metrics configuration for the OPC Ingestor application. + +This module defines all the metrics used for monitoring and observability +of the OPC Ingestor system. It includes metrics for: + +- Application health and performance +- OPC server connections and subscriptions +- Data processing and storage operations +- Resource management and load balancing +- Error tracking and notification systems + +All metrics follow Prometheus naming conventions and include appropriate +labels for multi-dimensional analysis and alerting. +""" + +from prometheus_client import Counter, Gauge, Histogram + +# Metric label definitions for consistent labeling across all metrics +POD_ID_LABEL = ['pod_id'] +SERVER_LABELS = ['pod_id', 'server_name', 'server_url'] +REDIS_LABELS = ['pod_id', 'operation'] +NOTIFICATION_LABELS = ['pod_id', 'level', 'block'] + +MAIN_LABELS = ['pod_id'] + +# --- Reliability Metrics --- +TAG_WRITTEN_COUNT = Counter( + 'ingestor_tag_written_count', + 'Number of writing process to the collection', + [*MAIN_LABELS, 'tag_name', 'collection_name'], +) + +# --- General Application Metrics --- +APP_LOOP_COUNT = Counter( + 'app_main_loop_total', + 'Total number of times the application main loop has run', + POD_ID_LABEL, +) +APP_LOOP_DURATION = Histogram( + 'app_main_loop_duration_seconds', + 'Duration of the application main loop in seconds', + POD_ID_LABEL, +) +APP_ERRORS_TOTAL = Counter( + 'app_errors_total', + 'Total number of unhandled errors in the main loop', + POD_ID_LABEL, +) +APP_UP = Gauge( + 'app_up', + 'Indicates if the application is running (1) or shutting down (0)', + POD_ID_LABEL, +) + +# --- Ingestor Manager Metrics --- +ACTIVE_INGESTORS = Gauge( + 'ingestor_active_total', + 'Number of active ingestors reported by Redis', + POD_ID_LABEL, +) +SLOTS_TOTAL = Gauge( + 'ingestor_slots_total', + 'Total number of slots configured in Redis', + POD_ID_LABEL, +) +LEASES_TOTAL = Gauge( + 'ingestor_leases_total', + 'Total number of leases (allocated slots) in Redis', + POD_ID_LABEL, +) +SLOTS_MANAGED = Gauge( + 'ingestor_slots_managed_current', + 'Number of slots currently managed by this ingestor instance', + POD_ID_LABEL, +) +SLOTS_ACQUIRED = Counter( + 'ingestor_slots_acquired_total', + 'Total number of slots acquired by this instance', + POD_ID_LABEL, +) +SLOTS_RELEASED = Counter( + 'ingestor_slots_released_total', + 'Total number of slots released by this instance', + POD_ID_LABEL, +) + +# --- OPC Manager Metrics --- +OPC_MANAGERS_ACTIVE = Gauge( + 'ingestor_opc_managers_active', + 'Number of active OPC Managers in this instance', + POD_ID_LABEL, +) +OPC_SUBSCRIPTION_ERRORS = Counter( + 'ingestor_opc_subscription_errors_total', + 'Errors when trying to subscribe to OPC tags', + ['pod_id', 'server', 'slot'], +) + +OPC_CONNECTIONS_TOTAL = Counter( + 'opc_connections_initiated_total', + 'Total connection attempts to OPC servers', + ['pod_id', 'server_name'], +) +OPC_CONNECTIONS_FAILED = Counter( + 'opc_connections_failed_total', + 'Total failed connection attempts to OPC servers', + ['pod_id', 'server_name'], +) +OPC_CONNECTION_STATUS = Gauge( + 'opc_connection_status', + 'Connection status with the OPC server (1=connected, 0=disconnected)', + SERVER_LABELS, +) +OPC_SUBSCRIPTIONS_CREATED = Counter( + 'opc_subscriptions_created_total', + 'Total OPC subscriptions created', + ['pod_id', 'server_name', 'slot_name'], +) +OPC_TAGS_SUBSCRIBED = Gauge( + 'opc_tags_subscribed_current', + 'Current number of OPC tags subscribed on a server', + ['pod_id', 'server_name'], +) +OPC_CYCLES_WITHOUT_DATA = Gauge( + 'opc_cycles_without_data', + 'Current number of cycles without receiving data from a server', + ['pod_id', 'server_name'], +) +OPC_RECONNECTIONS_TOTAL = Counter( + 'opc_reconnections_tried_total', + 'Reconnection attempts to an OPC server after a loss', + ['pod_id', 'server_name'], +) + +# --- Notification Metrics --- +NOTIFICATIONS_SENT = Counter( + 'notifications_sent_total', + 'Total number of notifications sent', + NOTIFICATION_LABELS, +) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e4841ad --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,155 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "ingestor" +version = "0.0.0" +description = "Sientia DataOps Ingestor - OPC Tag Ingestor" +readme = "README.md" +requires-python = ">=3.11" +authors = [ + {name = "Aignosi", email = "dev@aignosi.com"} +] + +[tool.ruff] +line-length = 100 +target-version = "py311" +exclude = [ + ".git", + ".venv", + "venv", + "__pycache__", + "*.pyc", + ".pytest_cache", + "htmlcov", +] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "UP", # pyupgrade + "N", # pep8-naming + "YTT", # flake8-2020 + "S", # flake8-bandit + "BLE", # flake8-blind-except + "A", # flake8-builtins + "C90", # mccabe complexity +] + +ignore = [ + "BLE001", # ignore blind except, we need to send notifications with any error + "E501", # line too long (handled by formatter) + "S101", # use of assert (needed for tests) + "S105", # possible hardcoded password (false positives) + "S106", # possible hardcoded password (false positives) + "N802", # function name should be lowercase (temporal decorators) + "N806", # variable in function should be lowercase +] + +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = [ + "S101", # assert allowed in tests + "S105", # hardcoded passwords ok in tests + "S106", # hardcoded passwords ok in tests +] + +[tool.ruff.lint.mccabe] +max-complexity = 15 + +[tool.ruff.format] +quote-style = "single" +indent-style = "space" +line-ending = "auto" + +[tool.mypy] +python_version = "3.11" +warn_return_any = false +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = false +warn_no_return = true +strict_equality = true +ignore_missing_imports = true + +# Ignore missing imports for external packages +[[tool.mypy.overrides]] +module = "temporalio.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sientia_do.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mlflow.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "prometheus_client.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sientia.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pandas.*" +ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "-v", + "--strict-markers" +] +markers = [ + "asyncio: marks tests as async", + "integration: marks tests as integration tests", + "unit: marks tests as unit tests", +] + +[tool.coverage.run] +source = ["model_manager"] +omit = [ + "*/tests/*", + "*/venv/*", + "*/__pycache__/*", + "*/site-packages/*", +] +branch = true + +[tool.coverage.report] +precision = 2 +show_missing = true +skip_covered = false +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "def __str__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", + "class .*\\bProtocol\\):", + "@(abc\\.)?abstractmethod", +] + +[tool.coverage.html] +directory = "htmlcov" + +[tool.bandit] +exclude_dirs = ["tests", "venv", ".venv"] +skips = ["B101", "B601"] # Skip assert and shell injection in controlled environments \ No newline at end of file diff --git a/redis-ui.ipynb b/redis-ui.ipynb new file mode 100644 index 0000000..e69de29 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..56ab376 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,19 @@ +# Development and Testing Dependencies +# These packages are only needed for development, testing, and code quality checks +# Install with: pip install -r requirements-dev.txt + +# Code Quality & Linting +ruff>=0.1.0 # Fast Python linter and formatter (replaces flake8, black, isort) +mypy>=1.7.0 # Static type checker +bandit>=1.7.5 # Security vulnerability scanner +pandas-stubs>=2.0.0 # Type stubs for pandas +types-requests>=2.31.0 # Type stubs for requests + +# Testing +pytest>=7.4.0 # Testing framework +pytest-cov>=4.1.0 # Coverage plugin for pytest +pytest-asyncio>=0.21.0 # Async test support (already in main requirements) + +# Development Tools +ipython>=8.12.0 # Enhanced Python shell +ipdb>=0.13.13 # IPython debugger \ No newline at end of file diff --git a/requirements-local.txt b/requirements-local.txt new file mode 100644 index 0000000..190e40b --- /dev/null +++ b/requirements-local.txt @@ -0,0 +1,5 @@ +asyncua==1.1.5 +redis +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.1 +prometheus_client +pymongo \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4db22cf --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +asyncua +redis +sientia_do==1.12.1 +prometheus_client +pymongo \ No newline at end of file diff --git a/run_coverage.sh b/run_coverage.sh new file mode 100755 index 0000000..3e0626a --- /dev/null +++ b/run_coverage.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Exit on any error +set -e + +echo "Activating virtual environment..." +source ./venv/bin/activate + +pytest --cov=ingestor --cov-report=html + +xdg-open htmlcov/index.html \ No newline at end of file diff --git a/run_local.sh b/run_local.sh new file mode 100755 index 0000000..de3ad55 --- /dev/null +++ b/run_local.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Exit on any error +set -e + +echo "Activating virtual environment..." +source ./venv/bin/activate + +echo "Loading environment variables from .env..." +if [ -f .env ]; then + export $(cat .env | grep -v '^#' | xargs) + echo "Environment variables loaded from .env" +else + echo "Warning: .env file not found. Continuing without environment variables." +fi + +echo "Starting ingestor application..." +python -m ingestor.app diff --git a/scripts/generate-and-push-opc-cert.sh b/scripts/generate-and-push-opc-cert.sh new file mode 100755 index 0000000..713965e --- /dev/null +++ b/scripts/generate-and-push-opc-cert.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Gera o certificado X.509 de cliente OPC UA do ingestor e sobe pro Gitea +# in-cluster (repo que o pod clona no boot — nada muda no values/deployment, +# só rollout restart depois do push). +# +# Uso: +# GITEA_PASSWORD='' ./scripts/generate-and-push-opc-cert.sh +# +# Vars (defaults pro cluster dev/gcp): +# GITEA_URL default https://git.sientia.ai +# GITEA_USER default gitea_admin +# GITEA_PASSWORD obrigatória +# GITEA_REPO default gitea_admin/sientia-dataops-opc-ingestor +# BRANCH default main +# APP_URI default urn:sientia:opc-ingestor (SAN URI do cert — TEM que +# ser igual ao campo `uri` do documento na collection +# OPC_servers: o ingestor usa esse valor como application_uri +# e o KEPServer rejeita com BadCertificateUriInvalid se divergir) +# DAYS default 3650 +# FORCE=1 sobrescreve cert já existente no repo +set -euo pipefail + +GITEA_URL=${GITEA_URL:-https://git.sientia.ai} +GITEA_USER=${GITEA_USER:-gitea_admin} +GITEA_PASSWORD=${GITEA_PASSWORD:?export GITEA_PASSWORD com a senha do Gitea} +GITEA_REPO=${GITEA_REPO:-gitea_admin/sientia-dataops-opc-ingestor} +BRANCH=${BRANCH:-main} +APP_URI=${APP_URI:-urn:sientia:opc-ingestor} +CN=${CN:-sientia-opc-ingestor} +DAYS=${DAYS:-3650} + +CERT_DIR_IN_REPO="certs" +CERT_NAME="opc-ingestor-cert.pem" +KEY_NAME="opc-ingestor-key.pem" +DER_NAME="opc-ingestor-cert.der" + +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT + +echo ">> Gerando certificado (CN=${CN}, URI=${APP_URI}, ${DAYS} dias)..." +# Extensões exigidas por OPC UA (KEPServer valida keyUsage/EKU e o SAN URI). +openssl req -x509 -newkey rsa:2048 -sha256 -nodes -days "$DAYS" \ + -keyout "$workdir/$KEY_NAME" -out "$workdir/$CERT_NAME" \ + -subj "/CN=${CN}/O=Aignosi/OU=Sientia" \ + -addext "subjectAltName=URI:${APP_URI},DNS:${CN}" \ + -addext "keyUsage=critical,digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment" \ + -addext "extendedKeyUsage=critical,clientAuth,serverAuth" \ + -addext "basicConstraints=critical,CA:FALSE" + +# Cópia DER — só referência p/ import manual no KEPServer (o trust normal é +# aceitar o cert rejeitado na 1ª conexão). +openssl x509 -in "$workdir/$CERT_NAME" -outform der -out "$workdir/$DER_NAME" + +echo ">> Clonando ${GITEA_REPO}@${BRANCH} do Gitea..." +encoded_pass=$(python3 - <&1 | sed "s#${encoded_pass}#***#g" + +if [[ -f "$workdir/repo/$CERT_DIR_IN_REPO/$CERT_NAME" && "${FORCE:-0}" != "1" ]]; then + echo "ERRO: $CERT_DIR_IN_REPO/$CERT_NAME já existe no repo. Re-rodar com FORCE=1 sobrescreve" >&2 + echo " (o cert antigo deixa de valer — retrust no KEPServer necessário)." >&2 + exit 1 +fi + +mkdir -p "$workdir/repo/$CERT_DIR_IN_REPO" +cp "$workdir/$CERT_NAME" "$workdir/$KEY_NAME" "$workdir/$DER_NAME" "$workdir/repo/$CERT_DIR_IN_REPO/" + +cd "$workdir/repo" +git add "$CERT_DIR_IN_REPO" +git -c user.name="opc-cert-script" -c user.email="platform@aignosi.com.br" \ + commit -m "Add OPC UA client certificate for KEPServer connection (URI ${APP_URI})" +git push origin "$BRANCH" 2>&1 | sed "s#${encoded_pass}#***#g" + +fingerprint=$(openssl x509 -in "$CERT_DIR_IN_REPO/$CERT_NAME" -noout -fingerprint -sha1) + +cat < ${CERT_DIR_IN_REPO}/{${CERT_NAME},${KEY_NAME},${DER_NAME}} +${fingerprint} + +Paths dentro do pod (o entrypoint clona o repo em /app): + cert_path: /app/${CERT_DIR_IN_REPO}/${CERT_NAME} + private_key_path: /app/${CERT_DIR_IN_REPO}/${KEY_NAME} + +Próximos passos: + 1. Documento na collection OPC_servers com "uri": "${APP_URI}" (ver README/PR). + 2. kubectl -n sientia rollout restart deployment/opc-ingestor # re-clona o repo + 3. Pod tenta conectar -> no KEPServer: OPC UA Configuration Manager -> + Trusted Clients -> cert '${CN}' rejeitado -> Trust -> restart runtime. +EOF diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..081bdfa --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,11 @@ +sonar.projectKey=Aignosi_sientia-dataops-opc-ingestor_cffccd89-d77c-477d-90e8-b28e64545808 +sonar.projectName=sientia-dataops-opc-ingestor +sonar.sources=ingestor +sonar.tests=tests +sonar.qualitygate.wait=true +sonar.qualitygate.timeout=300 +sonar.python.coverage.reportPaths=coverage.xml +sonar.python.xunit.reportPath=pytest.xml +sonar.python.version=3.11 +sonar.projectVersion=1.2.0 +sonar.coverage.exclusions=ingestor/app.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py new file mode 100644 index 0000000..5d8aab1 --- /dev/null +++ b/tests/functional/conftest.py @@ -0,0 +1,47 @@ +# import subprocess +# from time import sleep +# from typing import Generator +# import uuid +# from kafka import KafkaConsumer +# import pytest +# from redis import Redis + + +# @pytest.fixture(scope="session", autouse=True) +# def docker_compose(): +# """Sobe os containers antes dos testes e derruba depois.""" +# print("\n🚀 Subindo Docker Compose...") +# subprocess.run(["docker", "compose", "up", "-d"], check=True) + +# print("⏳ Aguardando containers ficarem prontos...") +# sleep(15) # ajuste conforme necessário + +# yield # os testes rodam aqui + +# print("\n🧹 Derrubando Docker Compose...") +# subprocess.run(["docker", "compose", "down"], check=True) + + +# @pytest.fixture() +# def redis_client(): +# redis = Redis(host="localhost", port=6379, decode_responses=True) +# redis.flushdb() + +# yield redis + +# # Limpa o banco de dados após os testes +# redis.flushdb() +# redis.close() + + +# def kafka_searcher(topic) -> Generator[KafkaConsumer, None, None]: +# consumer = KafkaConsumer( +# topic, +# bootstrap_servers="localhost:9092", +# group_id=f"test-group-{uuid.uuid4()}", +# auto_offset_reset="earliest", # Começa a consumir apenas mensagens novas +# enable_auto_commit=True, +# ) + +# yield consumer +# consumer.close() diff --git a/tests/functional/test_single_node.py b/tests/functional/test_single_node.py new file mode 100644 index 0000000..5264e16 --- /dev/null +++ b/tests/functional/test_single_node.py @@ -0,0 +1,118 @@ +# import json +# import subprocess +# from time import sleep + +# from tests.functional.conftest import kafka_searcher + + +# new_data = { +# "slot:opc_tags:1": { +# "server1": { +# "name": "server1", +# "url": "opc.tcp://simulator:4840", +# "server_uri": "http://opcua-server.simulator", +# "tags": { +# 'ns=2;i=2': { +# 'tag_name': 'Counter', +# 'frequency': 1000, +# 'topics': [], +# }, +# 'ns=2;i=3': { +# 'tag_name': 'Rollout', +# 'frequency': 1000, +# "topics": [], +# }, +# 'ns=2;i=4': { +# 'tag_name': 'Square', +# 'frequency': 1000, +# "topics": [], +# }, +# } +# } +# }, +# "slot:opc_tags:2": { +# "server2": { +# "name": "server2", +# "url": "opc.tcp://simulator:4840", +# "server_uri": "http://opcua-server.simulator", +# "tags": { +# 'ns=2;i=2': { +# 'tag_name': 'Counter', +# 'frequency': 1000, +# 'topics': [], +# }, +# 'ns=2;i=3': { +# 'tag_name': 'Rollout', +# 'frequency': 1000, +# "topics": [], +# }, +# 'ns=2;i=4': { +# 'tag_name': 'Square', +# 'frequency': 1000, +# "topics": [], +# }, +# } +# } +# } +# } + + +# def test_simple(redis_client): +# new_data['slot:opc_tags:1']['server1']['tags']['ns=2;i=2']['topics'] = [ +# 'test_topic_1'] +# redis_client.set("slot:opc_tags:1", +# json.dumps(new_data['slot:opc_tags:1'])) + +# sleep(20) # Espera o Ingestor processar os dados + +# # Check if lease is in Redis +# assert redis_client.get("lease:opc_tags:1") == 'ingestor' +# assert redis_client.get("heartbeat:ingestor:ingestor") == '1' + +# # Check if data is in Kafka + +# kafka = next(kafka_searcher('test_topic_1')) +# sleep(1) +# messages = kafka.poll(timeout_ms=10000) + +# assert messages, "Expected messages in Kafka, but got none." + + +# def test_simple_double_slot(redis_client): + +# new_data['slot:opc_tags:1']['server1']['tags']['ns=2;i=2']['topics'] = [ +# 'test_topic_double_slot1'] +# redis_client.set("slot:opc_tags:1", +# json.dumps(new_data['slot:opc_tags:1'])) + +# sleep(20) # Espera o Ingestor processar os dados + +# assert redis_client.get("lease:opc_tags:1") == 'ingestor' +# assert redis_client.get("heartbeat:ingestor:ingestor") == '1' + +# # Check if data is in Kafka +# kafka1 = next(kafka_searcher('test_topic_double_slot1')) +# messages = kafka1.poll(timeout_ms=10000) + +# assert messages, "Expected messages in test_topic_double_slot1, but got none." + +# new_data['slot:opc_tags:2']['server2']['tags']['ns=2;i=2']['topics'] = [ +# 'test_topic_double_slot2'] +# redis_client.set("slot:opc_tags:2", +# json.dumps(new_data['slot:opc_tags:2'])) + +# sleep(20) # Espera o Ingestor processar os dados + +# # Check if lease is in Redis +# assert redis_client.get("lease:opc_tags:2") == 'ingestor' +# assert redis_client.get("lease:opc_tags:1") == 'ingestor' +# assert redis_client.get("heartbeat:ingestor:ingestor") == '1' + +# # Check if data is in Kafka +# kafka2 = next(kafka_searcher('test_topic_double_slot2')) +# messages = kafka2.poll(timeout_ms=10000) + +# assert messages, "Expected messages in test_topic_double_slot2, but got none." + +# messages = kafka1.poll(timeout_ms=10000) +# assert messages, "Expected messages in test_topic_double_slot1, but got none." diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/managers/__init__.py b/tests/unit/managers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/managers/test_data_manager.py b/tests/unit/managers/test_data_manager.py new file mode 100644 index 0000000..4c6b2b6 --- /dev/null +++ b/tests/unit/managers/test_data_manager.py @@ -0,0 +1,108 @@ +from unittest.mock import ANY, AsyncMock, MagicMock, patch + +from pytest import fixture, mark +from sientia_do.notifications.models import NotificationLevel + +from ingestor.managers.data_manager import DataManager + +metadata = { + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', + 'pod_id': 'localhost', + }, +} + + +@fixture +@patch('ingestor.managers.data_manager.MongoDBRepository') +def data_manager(mongodb_repository): + data_manager = DataManager( + mongo_connection_string='mongodb://localhost:27017', + mongo_database='sientia', + logger=MagicMock(), + notification_handler=MagicMock(), + metadata=metadata['metadata'], + metrics_controller=MagicMock(), + ) + + data_manager.send_notification = MagicMock() + data_manager.send_notification_async = AsyncMock() + data_manager.emit_metric = AsyncMock() + return data_manager + + +@patch('ingestor.managers.data_manager.MongoDBRepository') +def test___init___success(mongodb_repository): + logger_mock = MagicMock() + + data_manager = DataManager( + metadata=metadata['metadata'], + mongo_connection_string='mongodb://localhost:27017', + mongo_database='sientia', + logger=logger_mock, + notification_handler=MagicMock(), + metrics_controller=MagicMock(), + ) + + logger_mock.info.assert_any_call( + 'DataManager initialized with MongoDB servers: mongodb://localhost:27017' + ) + logger_mock.error.assert_not_called() + assert data_manager.connection_string == 'mongodb://localhost:27017' + + +def test_shutdown(data_manager): + close_mock = MagicMock() + data_manager.mongo_repository.close = close_mock + + data_manager.shutdown() + close_mock.assert_called_once() + + +def test_shutdown_exception_mongo(data_manager): + data_manager.mongo_repository.close = MagicMock(side_effect=Exception('Test error')) + + data_manager.shutdown() + + data_manager.logger.error.assert_called_once_with('Error closing MongoDB client: Test error') + + +def test___del__(data_manager): + data_manager.shutdown = MagicMock() + data_manager.__del__() + data_manager.shutdown.assert_called_once() + + +@mark.asyncio +async def test_publish(data_manager): + topic = 'test_topic' + data = {'key': 'value', 'name': 'test_tag'} + + data_manager.mongo_repository.insert = AsyncMock() + + await data_manager.publish(topic, data) + + data_manager.mongo_repository.insert.assert_called_once_with( + collection_name=topic, + document={**data, 'inserted_at': ANY}, + metadata=metadata['metadata'], + ) + + +@mark.asyncio +async def test_publish_error_mongo(data_manager): + data_manager.mongo_repository.insert = AsyncMock(side_effect=Exception('Test error')) + + await data_manager.publish('test_topic', {'key': 'value'}) + + data_manager.send_notification_async.assert_called_once_with( + notification_id='MONGO_PRODUCER_ERROR_test_topic', + message='Error inserting message to MongoDB: Test error', + block='mongo_producer', + level=NotificationLevel.ERROR, + attachment_content=ANY, + metadata=metadata['metadata'], + ) diff --git a/tests/unit/managers/test_ingestor_manager.py b/tests/unit/managers/test_ingestor_manager.py new file mode 100644 index 0000000..9dcdd21 --- /dev/null +++ b/tests/unit/managers/test_ingestor_manager.py @@ -0,0 +1,648 @@ +from unittest.mock import AsyncMock, MagicMock, call, patch + +from pytest import fixture, mark +from sientia_do.notifications.models import NotificationLevel + +from ingestor.managers.ingestor_manager import IngestorManager + +metadata = { + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', + }, +} + + +@fixture +@patch('ingestor.managers.ingestor_manager.DataManager') +@patch('ingestor.managers.ingestor_manager.ResourceManager') +def ingestor_manager(data_manager_mock, resource_manager_mock): + ingestor = IngestorManager( + redis_data={'host': 'localhost', 'port': 6379}, + lease_ttl=60, + heartbeat_ttl=60, + poll_interval=5, + mongo_connection_string='mongodb://localhost:27017', + mongo_database='sientia', + logger=MagicMock(), + notification_handler=MagicMock(), + metadata=metadata['metadata'], + metrics_controller=MagicMock(), + ) + + ingestor.send_notification = MagicMock() + ingestor.send_notification_async = AsyncMock() + ingestor.emit_metric = AsyncMock() + + return ingestor + + +@patch('ingestor.managers.ingestor_manager.OpcManager') +@patch('ingestor.managers.ingestor_manager.DataManager') +@patch('ingestor.managers.ingestor_manager.ResourceManager') +@patch('ingestor.managers.ingestor_manager.NotificationHandler') +def test___init__( + notification_handler_mock, resource_manager_mock, data_manager_mock, opc_manager_mock +): + ingestor = IngestorManager( + redis_data={'host': 'localhost', 'port': 6379}, + lease_ttl=60, + heartbeat_ttl=60, + poll_interval=5, + mongo_connection_string='mongodb://localhost:27017', + mongo_database='sientia', + logger=MagicMock(), + notification_handler=MagicMock(), + metadata=metadata['metadata'], + metrics_controller=MagicMock(), + ) + + opc_manager_mock.assert_not_called() + data_manager_mock.assert_called_once_with( + mongo_connection_string='mongodb://localhost:27017', + mongo_database='sientia', + metadata=metadata['metadata'], + logger=ingestor.logger, + notification_handler=ingestor.notification_handler, + metrics_controller=ingestor.metrics_controller, + ) + resource_manager_mock.assert_called_once_with( + host='localhost', + port=6379, + lease_ttl=60, + heartbeat_ttl=60, + metadata=metadata['metadata'], + logger=ingestor.logger, + notification_handler=ingestor.notification_handler, + username=None, + password=None, + metrics_controller=ingestor.metrics_controller, + ) + assert ingestor.poll_interval == 5 + assert ingestor.managed_tags == {} + assert ingestor.opc_servers == {} + assert ingestor.opc_managers == {} + assert ingestor.data_manager == data_manager_mock.return_value + assert ingestor.resource_manager == resource_manager_mock.return_value + + +@mark.asyncio +@patch('ingestor.managers.ingestor_manager.OpcManager') +async def test_initialize_opc_from_config(opc_manager, ingestor_manager): + server_config = { + 'name': 'server1', + 'url': 'opc.tcp://localhost:4840', + 'subscription_period_ms': 1000, + 'server_uri': 'http://opcua-server.simulator', + 'cert_path': '/path/to/cert', + 'private_key_path': '/path/to/private_key', + 'server_cert_path': '/path/to/server_cert', + 'pod_id': 'test_pod', + } + + opc_manager.return_value = MagicMock(connect=AsyncMock()) + result = await ingestor_manager.initialize_opc_from_config(server_config) + + opc_manager.assert_called_once_with( + name=server_config['name'], + url=server_config['url'], + data_manager=ingestor_manager.data_manager, + logger=ingestor_manager.logger, + subscription_period_ms=server_config['subscription_period_ms'], + server_uri=server_config['server_uri'], + notification_handler=ingestor_manager.notification_handler, + cert_path=server_config['cert_path'], + private_key_path=server_config['private_key_path'], + server_cert_path=server_config['server_cert_path'], + metadata=metadata['metadata'], + metrics_controller=ingestor_manager.metrics_controller, + ) + + assert result == opc_manager.return_value + result.connect.assert_called_once() + + +@mark.asyncio +@patch('ingestor.managers.ingestor_manager.OpcManager') +@patch('ingestor.managers.ingestor_manager.traceback') +async def test_initialize_opc_from_config_exception(traceback_mock, opc_manager, ingestor_manager): + server_config = { + 'name': 'server1', + 'url': 'opc.tcp://localhost:4840', + 'subscription_period_ms': 1000, + 'server_uri': 'http://opcua-server.simulator', + 'cert_path': '/path/to/cert', + 'private_key_path': '/path/to/private_key', + 'server_cert_path': '/path/to/server_cert', + } + + ingestor_manager.logger.error = MagicMock() + opc_manager.side_effect = Exception('Initialization error') + + result = await ingestor_manager.initialize_opc_from_config(server_config) + + assert result is None + + traceback_mock.format_exc.assert_called_once() + ingestor_manager.send_notification.assert_called_once_with( + metadata=metadata['metadata'], + notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}', + message='Error initializing OPC manager: Initialization error', + block='opc_manager', + level=NotificationLevel.ERROR, + attachment_content=traceback_mock.format_exc.return_value, + ) + + +@mark.asyncio +async def test_shutdown(ingestor_manager): + ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()} + + ingestor_manager.data_manager.shutdown = MagicMock() + + await ingestor_manager.shutdown() + + ingestor_manager.opc_managers['server1'].shutdown.assert_called_once() + ingestor_manager.opc_managers['server2'].shutdown.assert_called_once() + + ingestor_manager.data_manager.shutdown.assert_called_once() + + +@patch('ingestor.managers.ingestor_manager.asyncio') +def test___del__(asyncio_mock, ingestor_manager): + ingestor_manager.shutdown = MagicMock() + ingestor_manager.__del__() + asyncio_mock.run.assert_called_once_with(ingestor_manager.shutdown.return_value) + + +@mark.asyncio +async def test_remove_server(ingestor_manager): + server1 = AsyncMock() + server2 = AsyncMock() + ingestor_manager.opc_managers = {'server1': server1, 'server2': server2} + ingestor_manager.managed_tags = { + 'slot1': {'server1': {'config': 'config1'}, 'server2': {'config': 'config2'}} + } + await ingestor_manager.remove_server('server1') + server1.shutdown.assert_called_once() + server2.shutdown.assert_not_called() + assert 'server1' not in ingestor_manager.opc_managers + assert 'server2' in ingestor_manager.opc_managers + assert ingestor_manager.managed_tags == {'slot1': {'server2': {'config': 'config2'}}} + + +@mark.asyncio +async def test_remove_server_not_found(ingestor_manager): + server1 = AsyncMock() + server2 = AsyncMock() + ingestor_manager.opc_managers = {'server1': server1, 'server2': server2} + ingestor_manager.managed_tags = { + 'slot1': {'server1': {'config': 'config1'}, 'server2': {'config': 'config2'}} + } + await ingestor_manager.remove_server('server3') + server1.shutdown.assert_not_called() + server2.shutdown.assert_not_called() + assert 'server1' in ingestor_manager.opc_managers + assert 'server2' in ingestor_manager.opc_managers + assert ingestor_manager.managed_tags == { + 'slot1': {'server1': {'config': 'config1'}, 'server2': {'config': 'config2'}} + } + + +@mark.asyncio +@patch('ingestor.managers.ingestor_manager.OpcManager') +@patch('ingestor.managers.ingestor_manager.metrics') +async def test_update_opc_servers(metrics, opc_manager, ingestor_manager): + manager1 = MagicMock(config={'config': 'config1'}) + manager2 = MagicMock(config={'config': 'config2'}) + manager3 = MagicMock(config={'config': 'config3'}) + + async def mock_initialize_from_config(config): + if config == {'config': 'config1'}: + return manager1 + elif config == {'config': 'config2'}: + return manager2 + elif config == {'config': 'config3'}: + return manager3 + else: + return None + + ingestor_manager.initialize_opc_from_config = AsyncMock(side_effect=mock_initialize_from_config) + ingestor_manager.remove_server = AsyncMock() + + ingestor_manager.managed_tags = { + 'slot1': { + 'server1': {'config': 'config1'}, + 'server2': {'config': 'config2'}, + 'server5': {'config': 'config5'}, + }, + 'slot2': { + 'server3': {'config': 'config3'}, + 'server1': {'config': 'config1'}, + }, + } + + mock = AsyncMock(config={'config': 'old_config2'}) + ingestor_manager.opc_managers['server3'] = AsyncMock(config={'config': 'config3'}) + ingestor_manager.opc_managers['server2'] = mock + ingestor_manager.opc_managers['server4'] = AsyncMock() + + await ingestor_manager.update_opc_servers() + + ingestor_manager.initialize_opc_from_config.assert_any_call({'config': 'config1'}) + ingestor_manager.initialize_opc_from_config.assert_any_call({'config': 'config2'}) + ingestor_manager.initialize_opc_from_config.assert_any_call({'config': 'config5'}) + + assert ingestor_manager.initialize_opc_from_config.call_count == 3 + + assert ingestor_manager.opc_managers['server1'].config == {'config': 'config1'} + assert ingestor_manager.opc_managers['server2'].config == {'config': 'config2'} + assert ingestor_manager.opc_managers['server3'].config == {'config': 'config3'} + + ingestor_manager.remove_server.assert_any_call('server4') + ingestor_manager.remove_server.assert_any_call('server5') + assert ingestor_manager.remove_server.call_count == 2 + + assert ingestor_manager.opc_managers['server2'] != mock + + ingestor_manager.emit_metric.assert_called_once_with( + metric_object=metrics.OPC_MANAGERS_ACTIVE, + method='set', + value=len(ingestor_manager.opc_managers), + tags={'pod_id': ingestor_manager.pod_id}, + ) + + +@mark.asyncio +@patch('ingestor.managers.ingestor_manager.metrics') +async def test_check_opc_servers_integrity_all_healthy(metrics, ingestor_manager): + # Setup mock OPC managers + opc_manager1 = MagicMock() + opc_manager1.check_cycles = AsyncMock(return_value=None) + opc_manager1.check_opc_listenning = AsyncMock(return_value=False) + opc_manager1.config = {'config': 'config1'} + + opc_manager2 = MagicMock() + opc_manager2.check_cycles = AsyncMock(return_value=None) + opc_manager2.check_opc_listenning = AsyncMock(return_value=False) + opc_manager2.config = {'config': 'config2'} + + ingestor_manager.opc_managers = {'server1': opc_manager1, 'server2': opc_manager2} + + # Mock the initialize_opc_from_config method + ingestor_manager.initialize_opc_from_config = MagicMock() + + # Call the method + await ingestor_manager.check_opc_servers_integrity() + + # Verify that check_cycles and check_opc_listenning were called for each server + opc_manager1.check_cycles.assert_called_once() + opc_manager1.check_opc_listenning.assert_called_once() + opc_manager2.check_cycles.assert_called_once() + opc_manager2.check_opc_listenning.assert_called_once() + + # Verify that no reinitialization was needed + ingestor_manager.initialize_opc_from_config.assert_not_called() + + ingestor_manager.emit_metric.assert_called_once_with( + metric_object=metrics.OPC_MANAGERS_ACTIVE, + method='set', + value=len(ingestor_manager.opc_managers), + tags={'pod_id': ingestor_manager.pod_id}, + ) + + +@mark.asyncio +async def test_check_opc_servers_integrity_server_lost(ingestor_manager): + # Setup mock OPC manager that will be lost + opc_manager = AsyncMock() + opc_manager.check_cycles.return_value = None + opc_manager.check_opc_listenning.return_value = True # Server is lost + opc_manager.config = {'config': 'config1'} + + ingestor_manager.opc_managers = {'server1': opc_manager} + ingestor_manager.managed_tags = {'slot1': MagicMock()} + + # Call the method + await ingestor_manager.check_opc_servers_integrity() + + assert 'server1' not in ingestor_manager.opc_managers + ingestor_manager.managed_tags['slot1'].pop.assert_called_once_with('server1', None) + + +def test_check_opc_servers_integrity_server_lost_with_tags(ingestor_manager): + # Setup mock OPC manager that will be lost + opc_manager = MagicMock() + opc_manager.check_cycles.return_value = None + opc_manager.check_opc_listenning.return_value = True # Server is lost + opc_manager.config = {'config': 'config1'} + + ingestor_manager.opc_managers = {'server1': opc_manager} + + # Setup managed tags + ingestor_manager.managed_tags = { + 'slot1': {'server1': {'config': 'config1', 'tags': {'tag1': 'value1'}}} + } + + # Mock the initialize_opc_from_config method to return a new manager + new_manager = MagicMock() + ingestor_manager.initialize_opc_from_config = MagicMock(return_value=new_manager) + + # Call the method + ingestor_manager.check_opc_servers_integrity() + + +@mark.asyncio +async def test_declare_active(ingestor_manager): + ingestor_manager.resource_manager.ingestor_heartbeat = AsyncMock() + await ingestor_manager.declare_active() + ingestor_manager.resource_manager.ingestor_heartbeat.assert_called_once() + + +@mark.asyncio +async def test_get_active_ingestors(ingestor_manager): + ingestor_manager.resource_manager.get_all_ingestors = AsyncMock() + await ingestor_manager.get_active_ingestors() + ingestor_manager.resource_manager.get_all_ingestors.assert_called_once() + + +@mark.asyncio +async def test_get_active_ingestors_empty(ingestor_manager): + ingestor_manager.resource_manager.get_all_ingestors = AsyncMock(return_value=None) + result = await ingestor_manager.get_active_ingestors() + assert result == [] + ingestor_manager.resource_manager.get_all_ingestors.assert_called_once() + + +@patch('ingestor.managers.ingestor_manager.metrics') +@mark.asyncio +async def test_get_number_of_leases_success(metrics, ingestor_manager): + ingestor_manager.resource_manager.get_all_leases = AsyncMock(return_value=['lease1', 'lease2']) + result = await ingestor_manager.get_number_of_leases() + assert result == 2 + + ingestor_manager.emit_metric.assert_called_once_with( + metric_object=metrics.LEASES_TOTAL, + method='set', + value=2, + tags={'pod_id': ingestor_manager.pod_id}, + ) + ingestor_manager.resource_manager.get_all_leases.assert_called_once() + + +@patch('ingestor.managers.ingestor_manager.metrics') +@mark.asyncio +async def test_get_number_of_slots_success(metrics, ingestor_manager): + ingestor_manager.resource_manager.get_all_slots = AsyncMock(return_value=['slot1', 'slot2']) + result = await ingestor_manager.get_number_of_slots() + assert result == 2 + ingestor_manager.resource_manager.get_all_slots.assert_called_once() + ingestor_manager.emit_metric.assert_called_once_with( + metric_object=metrics.SLOTS_TOTAL, + method='set', + value=2, + tags={'pod_id': ingestor_manager.pod_id}, + ) + + +@patch('ingestor.managers.ingestor_manager.metrics') +@mark.asyncio +async def test_get_number_of_slots_empty(metrics, ingestor_manager): + ingestor_manager.resource_manager.get_all_slots = AsyncMock(return_value=None) + result = await ingestor_manager.get_number_of_slots() + assert result == 0 + ingestor_manager.resource_manager.get_all_slots.assert_called_once() + ingestor_manager.emit_metric.assert_called_once_with( + metric_object=metrics.SLOTS_TOTAL, + method='set', + value=0, + tags={'pod_id': ingestor_manager.pod_id}, + ) + + +@mark.asyncio +async def test_get_slot_leases_1_success(ingestor_manager): + ingestor_manager.resource_manager.lease_tag = AsyncMock(return_value=True) + ingestor_manager.resource_manager.get_tag_slot = AsyncMock(return_value={'tags': ['tag1']}) + + ingestor_manager.number_of_slots = 1 + result = await ingestor_manager.get_slot_leases() + + assert result == {'1': {'tags': ['tag1']}} + + +@mark.asyncio +async def test_get_slot_leases_2_success(ingestor_manager): + ingestor_manager.resource_manager.lease_tag = AsyncMock(side_effect=[True, True]) + ingestor_manager.resource_manager.get_tag_slot = AsyncMock( + side_effect=[{'tags': ['tag1']}, {'tags': ['tag2']}] + ) + + ingestor_manager.number_of_slots = 2 + result = await ingestor_manager.get_slot_leases(max_slots=2) + + assert result == {'1': {'tags': ['tag1']}, '2': {'tags': ['tag2']}} + + +@mark.asyncio +async def test_get_slot_leases_2_1_none(ingestor_manager): + ingestor_manager.resource_manager.lease_tag = AsyncMock(side_effect=[True, True]) + ingestor_manager.resource_manager.get_tag_slot = AsyncMock( + side_effect=[None, {'tags': ['tag1']}] + ) + + ingestor_manager.number_of_slots = 1 + result = await ingestor_manager.get_slot_leases(max_slots=1) + + assert result == {} + + +@mark.asyncio +async def test_get_slot_leases_1_failure(ingestor_manager): + ingestor_manager.resource_manager.lease_tag = AsyncMock(return_value=False) + ingestor_manager.resource_manager.get_tag_slot = AsyncMock(return_value={'tags': ['tag1']}) + + result = await ingestor_manager.get_slot_leases() + ingestor_manager.resource_manager.get_tag_slot.assert_not_called() + + assert result == {} + + +@mark.asyncio +async def test_unsubscribe_slot(ingestor_manager): + ingestor_manager.managed_tags = { + 'slot1': {'server1': {'tags': 'config1'}, 'server2': {'tags': 'config2'}}, + 'slot2': {'server3': {'tags': 'config3'}, 'server1': {'tags': 'config1'}}, + } + ingestor_manager.opc_managers = { + 'server1': AsyncMock(), + 'server2': AsyncMock(), + 'server3': AsyncMock(), + } + await ingestor_manager.unsubscribe_slot('slot1') + + ingestor_manager.opc_managers['server1'].unsubscribe.assert_called_once_with('slot1') + ingestor_manager.opc_managers['server2'].unsubscribe.assert_called_once_with('slot1') + ingestor_manager.opc_managers['server3'].unsubscribe.assert_not_called() + + +@mark.asyncio +async def test_update_slot_config(ingestor_manager): + ingestor_manager.managed_tags = { + 'slot1': {'config': 'old_config'}, + 'slot2': {'config': 'new_config'}, + 'slot3': {'config': 'old_config'}, + } + + ingestor_manager.resource_manager.get_tag_slot = AsyncMock( + side_effect=[{'config': 'updated_config'}, {'config': 'new_config'}, None] + ) + ingestor_manager.resource_manager.renew_tag_lease = AsyncMock() + + await ingestor_manager.update_slot_config() + + assert ingestor_manager.managed_tags['slot1'] == {'config': 'updated_config'} + assert ingestor_manager.managed_tags['slot2'] == {'config': 'new_config'} + assert 'slot3' not in ingestor_manager.managed_tags + + ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot1') + ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot3') + ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot2') + assert ingestor_manager.resource_manager.renew_tag_lease.call_count == 3 + + +@patch('ingestor.managers.ingestor_manager.metrics') +@mark.asyncio +async def test_drop_slot_leases(metrics, ingestor_manager): + ingestor_manager.resource_manager.drop_tag_lease = AsyncMock() + await ingestor_manager.drop_slot_leases(['1', '2']) + + ingestor_manager.resource_manager.drop_tag_lease.assert_any_call('1') + ingestor_manager.resource_manager.drop_tag_lease.assert_any_call('2') + + ingestor_manager.emit_metric.assert_has_calls( + [ + call( + metric_object=metrics.SLOTS_RELEASED, + method='inc', + value=1, + tags={'pod_id': ingestor_manager.pod_id}, + ), + call( + metric_object=metrics.SLOTS_RELEASED, + method='inc', + value=1, + tags={'pod_id': ingestor_manager.pod_id}, + ), + ] + ) + + +@mark.asyncio +async def test_manage_server_no_server(ingestor_manager): + ingestor_manager.opc_managers = {'server1': MagicMock(), 'server2': MagicMock()} + server_config = {'tags': 'config1'} + + result = await ingestor_manager.manage_server('slot1', 'server3', server_config, server_config) + + assert result == 1 + ingestor_manager.opc_managers['server1'].create_subscription.assert_not_called() + ingestor_manager.opc_managers['server1'].subscribe.assert_not_called() + + +@mark.asyncio +async def test_manage_server_create_subscription_failure(ingestor_manager): + ingestor_manager.opc_managers = {'server1': MagicMock(), 'server2': MagicMock()} + ingestor_manager.subscriptions = {'server1': MagicMock()} + server_config = {'tags': 'config1'} + + ingestor_manager.opc_managers['server1'].create_subscription.side_effect = Exception( + 'Subscription error' + ) + + result = await ingestor_manager.manage_server('slot1', 'server1', server_config, server_config) + + assert result == 2 + ingestor_manager.opc_managers['server1'].create_subscription.assert_called_once_with('slot1') + ingestor_manager.opc_managers['server1'].subscribe.assert_not_called() + + +@mark.asyncio +async def test_manage_server(ingestor_manager): + ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()} + ingestor_manager.subscriptions = {'server1': AsyncMock()} + server_config = {'tags': 'config1'} + + result = await ingestor_manager.manage_server('slot1', 'server1', server_config, server_config) + + assert result == 0 + ingestor_manager.opc_managers['server1'].create_subscription.assert_called_once_with('slot1') + ingestor_manager.opc_managers['server1'].subscribe.assert_called_once_with( + 'slot1', 'config1', ingestor_manager.poll_interval + ) + + +@patch('ingestor.managers.ingestor_manager.traceback') +@mark.asyncio +async def test_manage_server_subscribe_failure(traceback_mock, ingestor_manager): + ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()} + ingestor_manager.subscriptions = {'server1': AsyncMock()} + server_config = {'tags': 'config1'} + + ingestor_manager.opc_managers['server1'].subscribe.side_effect = Exception('Subscription error') + + result = await ingestor_manager.manage_server('slot1', 'server1', server_config, server_config) + + assert result == 2 + ingestor_manager.opc_managers['server1'].create_subscription.assert_called_once_with('slot1') + ingestor_manager.opc_managers['server1'].subscribe.assert_called_once_with( + 'slot1', 'config1', ingestor_manager.poll_interval + ) + ingestor_manager.opc_managers['server1'].unsubscribe.assert_called_once_with('slot1') + + traceback_mock.format_exc.assert_called_once() + + ingestor_manager.send_notification_async.assert_called_once_with( + metadata=metadata['metadata'], + notification_id='OPC_SUBSCRIPTION_ERROR_slot1:server1', + message="Failed to subscribe to tags from slot1:server1\n{'tags': 'config1'}: Subscription error", + block='opc_manager', + level=NotificationLevel.ERROR, + attachment_content=traceback_mock.format_exc.return_value, + ) + + ingestor_manager.logger.warning.assert_any_call( + 'Removing subscription from server server1 for slot slot1' + ) + + +@mark.asyncio +async def test_subscribe_to_tags(ingestor_manager): + ingestor_manager.manage_server = AsyncMock(side_effect=[0, 1, 2]) + ingestor_manager.managed_tags = {'slot1': MagicMock(), 'slot2': MagicMock()} + + ingestor_manager.opc_managers = { + 'server1': AsyncMock(), + 'server2': AsyncMock(), + 'server3': AsyncMock(), + } + ingestor_manager.subscriptions = {'server1': AsyncMock()} + tags = { + 'slot1': { + 'server1': {'tags': 'config1'}, + 'server2': {'tags': 'config2'}, + 'server3': {'tags': 'config3'}, + } + } + + await ingestor_manager.subscribe_to_tags(tags) + + ingestor_manager.manage_server.assert_any_call('slot1', 'server1', {'tags': 'config1'}, tags) + ingestor_manager.manage_server.assert_any_call('slot1', 'server2', {'tags': 'config2'}, tags) + ingestor_manager.manage_server.assert_any_call('slot1', 'server3', {'tags': 'config3'}, tags) + + assert ingestor_manager.manage_server.call_count == 3 + + ingestor_manager.managed_tags['slot1'].pop.assert_called_once_with('server3', None) diff --git a/tests/unit/managers/test_opc_manager.py b/tests/unit/managers/test_opc_manager.py new file mode 100644 index 0000000..407c166 --- /dev/null +++ b/tests/unit/managers/test_opc_manager.py @@ -0,0 +1,627 @@ +import json +from datetime import datetime +from unittest.mock import ANY, AsyncMock, MagicMock, call, patch + +import pytest +from asyncua.crypto.security_policies import SecurityPolicyBasic256 +from pytest import fixture, mark +from sientia_do.notifications.models import NotificationLevel + +from ingestor.managers.opc_manager import OpcManager + +tags = { + 'ns=3;i=1001': { + 'aggregation_function': 'LTS', + 'frequency': 1000, + 'max_value': 100, + 'min_value': 0, + 'tag_name': 'Counter', + }, + 'ns=3;i=1003': { + 'aggregation_function': 'AVG', + 'frequency': 1000, + 'max_value': 100, + 'min_value': 0, + 'tag_name': 'Random', + }, + 'ns=3;i=1004': { + 'aggregation_function': 'MDN', + 'frequency': 1000, + 'max_value': 100, + 'min_value': 0, + 'tag_name': 'Sawtooth', + }, +} + +metadata = { + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', + }, +} + + +@fixture +@patch('ingestor.managers.opc_manager.metrics') +def raw_opc_manager(mock_metrics): + opc_manager = OpcManager( + name='TestConnector', + url='opc.tcp://localhost:4840', + data_manager=AsyncMock(), + subscription_period_ms=1000, + logger=MagicMock(), + server_uri='opc.tcp://localhost:4840', + notification_handler=MagicMock(), + metadata=metadata['metadata'], + metrics_controller=AsyncMock(), + ) + + opc_manager.emit_metric = AsyncMock() + opc_manager.send_notification_async = AsyncMock() + opc_manager.send_notification = MagicMock() + + return opc_manager + + +@fixture +def opc_manager(raw_opc_manager): + raw_opc_manager.client = AsyncMock() + raw_opc_manager.cert_path = 'cert.pem' + raw_opc_manager.private_key_path = 'private_key.pem' + raw_opc_manager.server_cert_path = 'server_cert.pem' + + return raw_opc_manager + + +@fixture +def opc_manager_subscribed(opc_manager): + opc_manager.subscriptions['sub1'] = AsyncMock() + + return opc_manager + + +def test___str__(opc_manager): + assert ( + str(opc_manager) + == 'OpcManager(name=TestConnector, url=opc.tcp://localhost:4840, server_uri=opc.tcp://localhost:4840)\nnodes={}, subscriptions={}' + ) + + +@mark.asyncio +async def test_shutdown_success(opc_manager): + opc_manager.disconnect = AsyncMock() + + await opc_manager.shutdown() + + opc_manager.disconnect.assert_called_once() + + +@mark.asyncio +async def test_shutdown_error(opc_manager): + opc_manager.disconnect = AsyncMock(side_effect=Exception('Test error')) + + await opc_manager.shutdown() + + opc_manager.logger.error.assert_called_once_with('Error during cleanup: Test error') + + +@mark.asyncio +async def test_set_security_success(opc_manager): + await opc_manager.set_security() + + assert opc_manager.client.application_uri == opc_manager.server_uri + + opc_manager.client.set_security.assert_called_once_with( + SecurityPolicyBasic256, + certificate=opc_manager.cert_path, + private_key=opc_manager.private_key_path, + server_certificate=opc_manager.server_cert_path, + ) + + assert opc_manager.client.secure_channel_timeout == 10000000 + assert opc_manager.client.session_timeout == 10000000 + + +@mark.asyncio +async def test_set_security_no_cert(opc_manager): + opc_manager.cert_path = None + opc_manager.private_key_path = None + + try: + await opc_manager.set_security() + except ValueError as e: + assert str(e) == 'Certificate and private key paths must be provided for secure connection.' + else: + raise AssertionError('ValueError not raised') + + assert opc_manager.client.set_security.call_count == 0 + + +@mark.asyncio +@patch('ingestor.managers.opc_manager.metrics') +@patch('ingestor.managers.opc_manager.Client') +async def test_connect_no_security(client, mock_metrics, raw_opc_manager): + raw_opc_manager.set_security = AsyncMock() + client.return_value = AsyncMock() + + await raw_opc_manager.connect() + + client.assert_called_once_with(raw_opc_manager.url, timeout=10, watchdog_intervall=3600000) + raw_opc_manager.client.connect.assert_called_once() + raw_opc_manager.set_security.assert_not_called() + raw_opc_manager.emit_metric.assert_has_calls( + [ + call( + metric_object=mock_metrics.OPC_CONNECTIONS_TOTAL, + tags={ + 'pod_id': raw_opc_manager.pod_id, + 'server_name': raw_opc_manager.name, + }, + ), + call( + metric_object=mock_metrics.OPC_CONNECTION_STATUS, + method='set', + value=1, + tags={ + 'pod_id': raw_opc_manager.pod_id, + 'server_name': raw_opc_manager.name, + 'server_url': raw_opc_manager.url, + }, + ), + ], + any_order=True, + ) + + +@mark.asyncio +@patch('ingestor.managers.opc_manager.Client') +async def test_connect_with_security(client, raw_opc_manager): + raw_opc_manager.cert_path = 'cert.pem' + raw_opc_manager.private_key_path = 'private_key.pem' + raw_opc_manager.server_cert_path = 'server_cert.pem' + raw_opc_manager.set_security = AsyncMock() + client.return_value = AsyncMock() + + await raw_opc_manager.connect() + + client.assert_called_once_with(raw_opc_manager.url, timeout=10, watchdog_intervall=3600000) + raw_opc_manager.client.connect.assert_called_once() + raw_opc_manager.set_security.assert_called_once() + + +@mark.asyncio +@patch('ingestor.managers.opc_manager.Client') +@patch('ingestor.managers.opc_manager.metrics') +async def test_connect_exception_handling_and_metrics( + mock_metrics_module, mock_opc_client_class, raw_opc_manager +): + mock_client_instance = mock_opc_client_class.return_value + simulated_error_message = 'Erro de conexão simulado' + mock_client_instance.connect.side_effect = Exception(simulated_error_message) + + raw_opc_manager.disconnect = AsyncMock() + + opc_manager_instance = raw_opc_manager + opc_manager_instance.cert_path = None + + with pytest.raises(Exception, match=simulated_error_message): + await opc_manager_instance.connect() + + opc_manager_instance.disconnect.assert_called_once() + + +@mark.asyncio +async def test_create_subscription_no_client(raw_opc_manager): + try: + await raw_opc_manager.create_subscription('sub1') + except ValueError as e: + assert str(e) == 'Client not connected. Call connect first.' + else: + raise AssertionError('ValueError not raised') + + +@mark.asyncio +async def test_create_subscription_success_has_period(opc_manager): + await opc_manager.create_subscription('sub1') + + opc_manager.client.create_subscription.assert_called_once_with( + opc_manager.subscription_period_ms, opc_manager + ) + assert opc_manager.subscriptions['sub1'] is not None + + +@mark.asyncio +async def test_create_subscription_success_no_period(opc_manager): + await opc_manager.create_subscription('sub1') + + opc_manager.client.create_subscription.assert_called_once_with( + opc_manager.subscription_period_ms, opc_manager + ) + assert opc_manager.subscriptions['sub1'] is not None + + +@patch('ingestor.managers.opc_manager.metrics') +@mark.asyncio +async def test_create_subscription_with_metrics(metrics, opc_manager): + await opc_manager.create_subscription('sub1') + + opc_manager.emit_metric.assert_called_once_with( + metric_object=metrics.OPC_SUBSCRIPTIONS_CREATED, + method='inc', + value=1, + tags={ + 'pod_id': opc_manager.pod_id, + 'server_name': opc_manager.name, + 'slot_name': 'sub1', + }, + ) + + +@patch('ingestor.managers.opc_manager.metrics') +@mark.asyncio +async def test_create_subscription_exception_during_client_call( + mock_metrics_module, raw_opc_manager +): + opc_manager_instance = raw_opc_manager + opc_manager_instance.client = AsyncMock() + + subscription_name = 'test_sub_client_error' + simulated_error_message = 'Falha ao criar subscrição no cliente OPC' + + opc_manager_instance.client.create_subscription.side_effect = Exception(simulated_error_message) + + with pytest.raises(Exception, match=simulated_error_message): + await opc_manager_instance.create_subscription(subscription_name) + + opc_manager_instance.client.create_subscription.assert_called_once_with( + opc_manager_instance.subscription_period_ms, opc_manager_instance + ) + + opc_manager_instance.logger.error.assert_called_once_with( + f'Failed to create subscription {subscription_name} on {opc_manager_instance.name}: {simulated_error_message}' + ) + + mock_metrics_module.OPC_SUBSCRIPTIONS_CREATED.labels.assert_not_called() + + +@patch('ingestor.managers.opc_manager.metrics') +@mark.asyncio +async def test_subscribe_no_subscription(metrics, opc_manager): + try: + await opc_manager.subscribe('sub1', tags, 1000) + except ValueError as e: + assert str(e) == 'Subscription not created. Call create_subscription first.' + else: + raise AssertionError('ValueError not raised') + metrics.OPC_TAGS_SUBSCRIBED.labels.assert_not_called() + + +@mark.asyncio +async def test_subscribe_success(opc_manager_subscribed): + opc_manager_subscribed.client.get_node = MagicMock() + opc_manager_subscribed.nodes = {'ns=3;i=1001': 'data'} + + await opc_manager_subscribed.subscribe('sub1', tags, 1000) + + assert opc_manager_subscribed.nodes == tags + opc_manager_subscribed.subscriptions['sub1'].subscribe_data_change.assert_called_once_with( + [opc_manager_subscribed.client.get_node(n) for n in tags] + ) + + +@mark.asyncio +async def test_unsubscribe_no_subscription(opc_manager): + await opc_manager.unsubscribe('sub1') + + opc_manager.logger.warning.assert_called_once_with( + "Subscription 'sub1' not found. Cannot unsubscribe." + ) + assert opc_manager.subscriptions.get('sub1') is None + + +@mark.asyncio +async def test_unsubscribe_success(opc_manager_subscribed): + await opc_manager_subscribed.unsubscribe('sub1') + + assert opc_manager_subscribed.subscriptions.get('sub1') is None + + +@mark.asyncio +async def test_disconnection_fallback_success(opc_manager): + opc_manager.client = AsyncMock() + opc_manager.client.disconnect.return_value = True + result = await opc_manager.disconnection_fallback() + assert result == [] + + +@mark.asyncio +async def test_disconnection_fallback_fail(opc_manager): + opc_manager.client = AsyncMock() + opc_manager.client.disconnect.side_effect = Exception('Test error') + result = await opc_manager.disconnection_fallback() + assert result == [ + {'attempt': 1, 'error': 'Test error', 'traceback': ANY}, + {'attempt': 2, 'error': 'Test error', 'traceback': ANY}, + {'attempt': 3, 'error': 'Test error', 'traceback': ANY}, + {'attempt': 4, 'error': 'Test error', 'traceback': ANY}, + {'attempt': 5, 'error': 'Test error', 'traceback': ANY}, + ] + assert opc_manager.client.disconnect.call_count == 5 + + +@mark.asyncio +async def test_disconnect_success(opc_manager_subscribed): + opc_manager_subscribed.client = MagicMock() + + await opc_manager_subscribed.disconnect() + + opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once() + assert opc_manager_subscribed.client is None + + +@mark.asyncio +async def test_disconnect_no_client(opc_manager_subscribed): + opc_manager_subscribed.client = None + assert await opc_manager_subscribed.disconnect() is None + + opc_manager_subscribed.logger.warning.assert_has_calls( + [ + call('Client already disconnected.'), + ] + ) + + +@mark.asyncio +async def test_disconnect_error_unsubscribe(opc_manager_subscribed): + opc_manager_subscribed.client = MagicMock(disconnect=AsyncMock()) + opc_manager_subscribed.subscriptions['sub1'] = MagicMock( + delete=AsyncMock(side_effect=Exception('Test error')) + ) + + await opc_manager_subscribed.disconnect() + + opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once() + opc_manager_subscribed.client = None + opc_manager_subscribed.logger.error.assert_called_once_with( + 'Failed to clean up subscription: Test error' + ) + + +@mark.asyncio +async def test_disconnect_error(opc_manager_subscribed): + opc_manager_subscribed.client = MagicMock() + opc_manager_subscribed.client.disconnect = MagicMock(side_effect=Exception('Test error')) + + await opc_manager_subscribed.disconnect() + + opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once() + opc_manager_subscribed.client = None + + +@patch('ingestor.managers.opc_manager.metrics') +@mark.asyncio +async def test_disconnect_metrics_on_successful_path(mock_metrics_module, raw_opc_manager): + mock_metrics_module.OPC_CONNECTION_STATUS.reset_mock() + mock_metrics_module.OPC_TAGS_SUBSCRIBED.reset_mock() + + raw_opc_manager.client = MagicMock() + mock_sub1 = MagicMock() + mock_sub2 = MagicMock() + raw_opc_manager.subscriptions = {'sub1': mock_sub1, 'sub2': mock_sub2} + + await raw_opc_manager.disconnect() + + raw_opc_manager.emit_metric.assert_has_calls( + [ + call( + metric_object=mock_metrics_module.OPC_CONNECTION_STATUS, + method='set', + value=0, + tags={ + 'pod_id': raw_opc_manager.pod_id, + 'server_name': raw_opc_manager.name, + 'server_url': raw_opc_manager.url, + }, + ), + call( + metric_object=mock_metrics_module.OPC_TAGS_SUBSCRIBED, + method='set', + value=0, + tags={ + 'pod_id': raw_opc_manager.pod_id, + 'server_name': raw_opc_manager.name, + }, + ), + ], + any_order=True, + ) + + +@patch('ingestor.managers.opc_manager.metrics') +@mark.asyncio +async def test_datachange_notification(metrics, opc_manager_subscribed): + data = MagicMock( + monitored_item=MagicMock( + Value=MagicMock( + Value=MagicMock(Value=42), + SourceTimestamp=datetime.strptime('2021-01-01T00:00:00', '%Y-%m-%dT%H:%M:%S'), + ) + ) + ) + opc_manager_subscribed.nodes = { + 'ns=3;i=1001': { + 'tag_name': 'Counter', + 'cycle_rule': {'cycle_increment': 1.0, 'cycle_count': 2}, + 'topics': ['topic1', 'topic2'], + } + } + + await opc_manager_subscribed.datachange_notification('ns=3;i=1001', None, data) + + opc_manager_subscribed.data_manager.publish.assert_any_call( + 'topic1', + { + 'tag': 'ns=3;i=1001', + 'name': 'Counter', + 'timestamp': '2021-01-01 00:00:00-0300', + 'value': 42, + }, + ) + opc_manager_subscribed.data_manager.publish.assert_any_call( + 'topic2', + { + 'tag': 'ns=3;i=1001', + 'name': 'Counter', + 'timestamp': '2021-01-01 00:00:00-0300', + 'value': 42, + }, + ) + assert opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == 0 + + opc_manager_subscribed.emit_metric.assert_called_once_with( + metric_object=metrics.OPC_CYCLES_WITHOUT_DATA, + method='set', + value=0, + tags={ + 'pod_id': opc_manager_subscribed.pod_id, + 'server_name': opc_manager_subscribed.name, + }, + ) + + +@mark.asyncio +async def test_check_cycles_no_notification(opc_manager): + # Setup: node with cycle_count just below threshold + opc_manager.nodes = { + 'ns=3;i=1001': { + 'tag_name': 'Counter', + 'cycle_rule': {'cycle_increment': 1.0, 'cycle_count': 3.0}, + } + } + + await opc_manager.check_cycles() + + # After one increment, cycle_count = 4.0, still below threshold + assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(4.0) + opc_manager.send_notification_async.assert_not_called() + + +@mark.asyncio +async def test_check_cycles_triggers_notification(opc_manager): + # Setup: node with cycle_count just below threshold, increment will cross threshold + opc_manager.nodes = { + 'ns=3;i=1001': { + 'tag_name': 'Counter', + 'cycle_rule': {'cycle_increment': 2.5, 'cycle_count': 3.0}, + } + } + opc_manager.notification_handler.build_and_send_notification = MagicMock() + + await opc_manager.check_cycles() + + # After increment, cycle_count = 5.5, should trigger notification + assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(5.5) + opc_manager.send_notification_async.assert_called_once_with( + notification_id='TAG_ns=3;i=1001:Counter_LISTENNING_STOPPED', + message='5.5 cycles without receive from ns=3;i=1001:Counter', + block='opc_manager', + level=NotificationLevel.WARNING, + metadata=metadata['metadata'], + ) + + +@patch('ingestor.managers.opc_manager.metrics') +@mark.asyncio +async def test_check_opc_listenning_no_notification(metrics, opc_manager): + opc_manager.non_receive_count = 3 + + result = await opc_manager.check_opc_listenning() + + assert opc_manager.non_receive_count == 4 + opc_manager.send_notification_async.assert_not_called() + assert result is False + + opc_manager.emit_metric.assert_called_once_with( + metric_object=metrics.OPC_CYCLES_WITHOUT_DATA, + method='set', + value=opc_manager.non_receive_count, + tags={ + 'pod_id': opc_manager.pod_id, + 'server_name': opc_manager.name, + }, + ) + + +@mark.asyncio +async def test_check_opc_listenning_warning_notification(opc_manager): + opc_manager.non_receive_count = 4 + + result = await opc_manager.check_opc_listenning() + + assert opc_manager.non_receive_count == 5 + opc_manager.send_notification_async.assert_called_once_with( + notification_id=f'OPC_LISTENNING_STOPPED__{opc_manager.name}', + message=f'5 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}', + block='opc_manager', + level=NotificationLevel.ERROR, + metadata=metadata['metadata'], + ) + assert result is False + + +@patch('ingestor.managers.opc_manager.metrics') +@mark.asyncio +async def test_check_opc_listenning_error_notification_and_retry(metrics, opc_manager): + opc_manager.non_receive_count = 14 + + result = await opc_manager.check_opc_listenning() + + assert opc_manager.non_receive_count == 15 + # Should be called twice: once for 5, once for 15 + assert opc_manager.send_notification_async.call_count == 2 + calls = opc_manager.send_notification_async.call_args_list + # First call: 5 cycles warning + assert calls[0].kwargs == { + 'notification_id': f'OPC_LISTENNING_STOPPED__{opc_manager.name}', + 'message': f'15 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}', + 'block': 'opc_manager', + 'level': NotificationLevel.ERROR, + 'metadata': metadata['metadata'], + } + # Second call: 15 cycles retry + assert calls[1].kwargs == { + 'notification_id': f'OPC_CONNECTION_RETRY__{opc_manager.name}', + 'message': f'Retrying to connect to server {opc_manager.name}', + 'block': 'opc_manager', + 'level': NotificationLevel.ERROR, + 'metadata': metadata['metadata'], + } + assert result is True + + opc_manager.emit_metric.assert_has_calls( + [ + call( + metric_object=metrics.OPC_CYCLES_WITHOUT_DATA, + method='set', + value=opc_manager.non_receive_count, + tags={ + 'pod_id': opc_manager.pod_id, + 'server_name': opc_manager.name, + }, + ), + ] + ) + + opc_manager.emit_metric.assert_has_calls( + [ + call( + metric_object=metrics.OPC_RECONNECTIONS_TOTAL, + tags={ + 'pod_id': opc_manager.pod_id, + 'server_name': opc_manager.name, + }, + ), + ] + ) diff --git a/tests/unit/managers/test_resource_manager.py b/tests/unit/managers/test_resource_manager.py new file mode 100644 index 0000000..d7e236c --- /dev/null +++ b/tests/unit/managers/test_resource_manager.py @@ -0,0 +1,178 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +from pytest import fixture, mark, raises + +from ingestor.managers.resource_manager import ResourceManager + +metadata = { + 'metadata': { + 'model_id': 'test_model', + 'model_name': 'test_model', + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', + }, +} + + +@patch('ingestor.managers.resource_manager.RedisRepository') +def test___init__(redis_repository): + logger = MagicMock() + notification_handler = MagicMock() + metrics_controller = MagicMock() + resource_manager = ResourceManager( + host='localhost', + port=6379, + lease_ttl=10, + heartbeat_ttl=10, + metadata=metadata['metadata'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + assert resource_manager.redis_repository == redis_repository.return_value + redis_repository.assert_called_once_with( + host='localhost', + port=6379, + username=None, + password=None, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + redis_repository.return_value.redis_client.ping.assert_called_once() + + +@patch('ingestor.managers.resource_manager.RedisRepository') +def test___init__connection_failure(redis_repository): + redis_repository.return_value.redis_client.ping.side_effect = Exception('Connection failed') + with raises(Exception, match='Connection failed'): + ResourceManager( + host='localhost', + port=6379, + lease_ttl=10, + heartbeat_ttl=10, + metadata=metadata['metadata'], + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(), + ) + redis_repository.return_value.redis_client.ping.assert_called_once() + redis_repository.logger.error.assert_called_once_with( + 'Failed to connect to Redis: Connection failed' + ) + redis_repository.return_value.redis_client.ping.assert_called_once() + + +@fixture +@patch('ingestor.managers.resource_manager.RedisRepository') +def resource_manager(redis_repository): + resource_manager = ResourceManager( + host='localhost', + port=6379, + lease_ttl=10, + heartbeat_ttl=10, + metadata=metadata['metadata'], + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(), + ) + + resource_manager.send_notification = MagicMock() + resource_manager.send_notification_async = AsyncMock() + resource_manager.emit_metric = AsyncMock() + + resource_manager.redis_repository = AsyncMock() + + return resource_manager + + +@mark.asyncio +async def test_get_tag_slot(resource_manager): + result = await resource_manager.get_tag_slot('id') + assert result == resource_manager.redis_repository.get.return_value + + resource_manager.redis_repository.get.assert_called_once_with( + 'slot:opc_tags:id', metadata=metadata['metadata'] + ) + + +@mark.asyncio +async def test_ingestor_heartbeat(resource_manager): + await resource_manager.ingestor_heartbeat() + resource_manager.redis_repository.set.assert_called_once_with( + 'heartbeat:ingestor:localhost', 1, ttl=10, metadata=metadata['metadata'] + ) + + +@mark.asyncio +async def test_lease_tag(resource_manager): + output = await resource_manager.lease_tag('tag_id') + assert output is resource_manager.redis_repository.set.return_value + resource_manager.redis_repository.set.assert_called_once_with( + 'lease:opc_tags:tag_id', 'localhost', ttl=10, nx=True, metadata=metadata['metadata'] + ) + + +@mark.asyncio +async def test_renew_tag_lease_success(resource_manager): + resource_manager.redis_repository.get.return_value = 'localhost' + resource_manager.redis_repository.expire.return_value = True + result = await resource_manager.renew_tag_lease('tag_id') + assert result is resource_manager.redis_repository.expire.return_value + resource_manager.redis_repository.get.assert_called_once_with( + 'lease:opc_tags:tag_id', metadata=metadata['metadata'] + ) + resource_manager.redis_repository.expire.assert_called_once_with( + 'lease:opc_tags:tag_id', 10, metadata=metadata['metadata'] + ) + + +@mark.asyncio +async def test_renew_tag_lease_failure(resource_manager): + resource_manager.redis_repository.get.return_value = 'other_pod_id' + + result = await resource_manager.renew_tag_lease('tag_id') + assert result is False + + resource_manager.redis_repository.get.assert_called_once_with( + 'lease:opc_tags:tag_id', metadata=metadata['metadata'] + ) + resource_manager.redis_repository.expire.assert_not_called() + + +@mark.asyncio +async def test_drop_tag_lease(resource_manager): + await resource_manager.drop_tag_lease('tag_id') + resource_manager.redis_repository.delete.assert_called_once_with( + 'lease:opc_tags:tag_id', metadata=metadata['metadata'] + ) + + +@mark.asyncio +async def test_get_all_ingestors(resource_manager): + resource_manager.redis_repository.keys.return_value = ['ingestor1', 'ingestor2'] + result = await resource_manager.get_all_ingestors() + assert result == ['ingestor1', 'ingestor2'] + resource_manager.redis_repository.keys.assert_called_once_with( + 'heartbeat:ingestor:*', metadata=metadata['metadata'] + ) + + +@mark.asyncio +async def test_get_all_slots(resource_manager): + resource_manager.redis_repository.keys.return_value = ['slot1', 'slot2'] + result = await resource_manager.get_all_slots() + assert result == ['slot1', 'slot2'] + resource_manager.redis_repository.keys.assert_called_once_with( + 'slot:opc_tags:*', metadata=metadata['metadata'] + ) + + +@mark.asyncio +async def test_get_all_leases(resource_manager): + resource_manager.redis_repository.keys.return_value = ['lease1', 'lease2'] + result = await resource_manager.get_all_leases() + assert result == ['lease1', 'lease2'] + resource_manager.redis_repository.keys.assert_called_once_with( + 'lease:opc_tags:*', metadata=metadata['metadata'] + ) diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py new file mode 100644 index 0000000..558cf1d --- /dev/null +++ b/tests/unit/test_app.py @@ -0,0 +1,319 @@ +import asyncio +import signal as signal_module # To avoid conflict with mock names +from threading import Event +from unittest.mock import AsyncMock, MagicMock, call + +import pytest + +# Import the 'app' module to be tested +from ingestor import app + + +# Custom exception to catch os._exit calls +class OsExitCalledError(Exception): + def __init__(self, code): + super().__init__(f'os._exit({code}) called') + self.code = code + + +# Helper function for the os_exit mock's side_effect +def raise_os_exit_with_code(exit_code): + raise OsExitCalledError(exit_code) + + +@pytest.fixture +def mock_app_env(monkeypatch): + """Fixture to mock dependencies of app.main and app.signal_handler.""" + mocks = { + 'start_http_server': MagicMock(), + 'Ingestor': MagicMock(), + 'metrics_APP_UP_labels_set': MagicMock(), + 'metrics_APP_LOOP_COUNT_labels_inc': MagicMock(), + 'metrics_APP_LOOP_DURATION_labels_observe': MagicMock(), + 'metrics_APP_ERRORS_TOTAL_labels_inc': MagicMock(), + 'os_exit': MagicMock(side_effect=raise_os_exit_with_code), + 'time_time': MagicMock(), + 'asyncio_sleep': AsyncMock(), + 'signal_signal': MagicMock(), + 'traceback_print_exc': MagicMock(), + 'mock_exit_signal': MagicMock(spec=Event), + } + + monkeypatch.setattr(app, 'start_http_server', mocks['start_http_server']) + monkeypatch.setattr(app, 'Ingestor', mocks['Ingestor']) + monkeypatch.setattr(asyncio, 'sleep', mocks['asyncio_sleep']) + + monkeypatch.setattr( + app.metrics.APP_UP, + 'labels', + MagicMock(return_value=MagicMock(set=mocks['metrics_APP_UP_labels_set'])), + ) + monkeypatch.setattr( + app.metrics.APP_LOOP_COUNT, + 'labels', + MagicMock(return_value=MagicMock(inc=mocks['metrics_APP_LOOP_COUNT_labels_inc'])), + ) + monkeypatch.setattr( + app.metrics.APP_LOOP_DURATION, + 'labels', + MagicMock( + return_value=MagicMock(observe=mocks['metrics_APP_LOOP_DURATION_labels_observe']) + ), + ) + monkeypatch.setattr( + app.metrics.APP_ERRORS_TOTAL, + 'labels', + MagicMock(return_value=MagicMock(inc=mocks['metrics_APP_ERRORS_TOTAL_labels_inc'])), + ) + + monkeypatch.setattr(app.os, '_exit', mocks['os_exit']) + monkeypatch.setattr(app, 'time', mocks['time_time']) + monkeypatch.setattr(app.signal, 'signal', mocks['signal_signal']) + monkeypatch.setattr(app.traceback, 'print_exc', mocks['traceback_print_exc']) + + monkeypatch.setattr(app, 'exit_signal', mocks['mock_exit_signal']) + monkeypatch.setattr(app, 'POD_ID', 'test_pod') + + mock_ingestor_instance = mocks['Ingestor'].return_value + mock_ingestor_instance.poll_interval = 0.01 + mock_ingestor_instance.logger = MagicMock() + + # Make async methods async mocks + mock_ingestor_instance.prepare_ingestor = AsyncMock() + mock_ingestor_instance.loop = AsyncMock() + mock_ingestor_instance.shutdown = AsyncMock() + + return mocks + + +@pytest.mark.asyncio +async def test_main_successful_run_one_loop(mock_app_env, capsys): + """Test a successful run where the loop executes once and then exits gracefully.""" + mock_ingestor_instance = mock_app_env['Ingestor'].return_value + mock_exit_signal = mock_app_env['mock_exit_signal'] + + mock_exit_signal.is_set.side_effect = [False, True] + mock_app_env['time_time'].side_effect = [10.0, 11.5] + + with pytest.raises(OsExitCalledError) as excinfo: + await app.main() + assert excinfo.value.code == 0 + + mock_app_env['start_http_server'].assert_called_once_with(9090) + app.metrics.APP_UP.labels.assert_any_call(pod_id='test_pod') + set_calls = mock_app_env['metrics_APP_UP_labels_set'].call_args_list + assert call(1) in set_calls + assert call(0) in set_calls + assert set_calls.index(call(1)) < set_calls.index(call(0)) + + mock_app_env['Ingestor'].assert_called_once_with() + mock_ingestor_instance.prepare_ingestor.assert_called_once() + mock_ingestor_instance.logger.info.assert_any_call('Ingestor prepared. Starting main loop.') + + mock_ingestor_instance.loop.assert_called_once() + app.metrics.APP_LOOP_COUNT.labels.assert_called_with(pod_id='test_pod') + mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].assert_called_once() + + mock_app_env['asyncio_sleep'].assert_has_calls( + [call(mock_ingestor_instance.poll_interval), call(5)] + ) + + app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id='test_pod') + mock_app_env['metrics_APP_LOOP_DURATION_labels_observe'].assert_called_once_with(1.5) + + mock_ingestor_instance.shutdown.assert_called_once() + mock_ingestor_instance.logger.info.assert_any_call('Main loop exit_signaled.') + + captured = capsys.readouterr() + assert 'Prometheus server started on port 9090.' in captured.out + + +@pytest.mark.asyncio +async def test_main_prometheus_server_fails_to_start(mock_app_env, capsys): + """Test the scenario where starting the Prometheus server fails.""" + mock_app_env['start_http_server'].side_effect = OSError('Port already in use') + + with pytest.raises(OsExitCalledError) as excinfo: + await app.main() + assert excinfo.value.code == 1 + + # Check that .set(1) was not called. .set(0) definitely not called. + called_with_1 = False + for call_args in mock_app_env['metrics_APP_UP_labels_set'].call_args_list: + if call_args == call(1): + called_with_1 = True + break + assert not called_with_1, 'APP_UP.set(1) should not have been called if server start failed' + + mock_app_env['Ingestor'].assert_not_called() + captured = capsys.readouterr() + assert 'Failed to start Prometheus server: Port already in use' in captured.out + + +@pytest.mark.asyncio +async def test_main_loop_exception_handling(mock_app_env, capsys): + """Test that an exception in ingestor.loop() is handled gracefully.""" + mock_ingestor_instance = mock_app_env['Ingestor'].return_value + mock_exit_signal = mock_app_env['mock_exit_signal'] + + mock_exit_signal.is_set.side_effect = [False, True] + mock_ingestor_instance.loop.side_effect = Exception('Test loop exception') + mock_app_env['time_time'].side_effect = [10.0, 10.1] + + with pytest.raises(OsExitCalledError) as excinfo: + await app.main() + assert excinfo.value.code == 0 + + mock_ingestor_instance.loop.assert_called_once() + mock_app_env['traceback_print_exc'].assert_called_once() + + app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id='test_pod') + mock_app_env['metrics_APP_ERRORS_TOTAL_labels_inc'].assert_called_once() + + mock_exit_signal.set.assert_called_once() + + mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].assert_not_called() + + app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id='test_pod') + mock_app_env['metrics_APP_LOOP_DURATION_labels_observe'].assert_called_once_with( + pytest.approx(0.1) + ) + + mock_ingestor_instance.shutdown.assert_called_once() + captured = capsys.readouterr() + assert 'Exception in main loop. Setting exit_signal flag.' in captured.out + + +@pytest.mark.asyncio +async def test_main_keyboard_interrupt_handling(mock_app_env, capsys): + """Test that KeyboardInterrupt in ingestor.loop() is handled.""" + mock_ingestor_instance = mock_app_env['Ingestor'].return_value + mock_exit_signal = mock_app_env['mock_exit_signal'] + + mock_exit_signal.is_set.side_effect = [False, True] + mock_ingestor_instance.loop.side_effect = KeyboardInterrupt() + mock_app_env['time_time'].side_effect = [10.0, 10.1] + + with pytest.raises(OsExitCalledError) as excinfo: + await app.main() + assert excinfo.value.code == 0 + + mock_ingestor_instance.loop.assert_called_once() + mock_exit_signal.set.assert_called_once() + mock_ingestor_instance.shutdown.assert_called_once() + captured = capsys.readouterr() + assert 'KeyboardInterrupt received. Setting exit_signal flag.' in captured.out + mock_app_env['metrics_APP_ERRORS_TOTAL_labels_inc'].assert_not_called() + + +def test_signal_handler_sets_exit_signal(mock_app_env): + """Test that the signal_handler function calls exit_signal.set().""" + mock_exit_signal_set = mock_app_env['mock_exit_signal'].set + + app.signal_handler(signal_module.SIGINT, None) + mock_exit_signal_set.assert_called_once() + + +@pytest.mark.asyncio +async def test_main_multiple_loop_iterations(mock_app_env): + """Test the main loop runs for a few iterations.""" + mock_ingestor_instance = mock_app_env['Ingestor'].return_value + mock_exit_signal = mock_app_env['mock_exit_signal'] + + mock_exit_signal.is_set.side_effect = [False, False, False, True] + mock_app_env['time_time'].side_effect = [10.0, 10.1, 10.2, 10.3, 10.4, 10.5] + + with pytest.raises(OsExitCalledError) as excinfo: + await app.main() + assert excinfo.value.code == 0 + + assert mock_ingestor_instance.loop.call_count == 3 + + assert app.metrics.APP_LOOP_COUNT.labels.call_count == 3 + app.metrics.APP_LOOP_COUNT.labels.assert_called_with( + pod_id='test_pod' + ) # Checks last call or any call + assert mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].call_count == 3 + + assert mock_app_env['asyncio_sleep'].call_count == 4 + + assert app.metrics.APP_LOOP_DURATION.labels.call_count == 3 + app.metrics.APP_LOOP_DURATION.labels.assert_called_with(pod_id='test_pod') + duration_calls = mock_app_env['metrics_APP_LOOP_DURATION_labels_observe'].call_args_list + assert duration_calls[0] == call(pytest.approx(0.1, abs=1e-9)) + assert duration_calls[1] == call(pytest.approx(0.1, abs=1e-9)) + assert duration_calls[2] == call(pytest.approx(0.1, abs=1e-9)) + + mock_ingestor_instance.shutdown.assert_called_once() + + +@pytest.mark.asyncio +async def test_main_pod_id_used_in_metrics(mock_app_env): + """Test that the POD_ID from app module is used in metric labels.""" + mock_exit_signal = mock_app_env['mock_exit_signal'] + mock_exit_signal.is_set.side_effect = [False, True] + mock_app_env['time_time'].side_effect = [10.0, 11.0] + + with pytest.raises(OsExitCalledError): + await app.main() + + app.metrics.APP_UP.labels.assert_any_call(pod_id='test_pod') + app.metrics.APP_LOOP_COUNT.labels.assert_any_call(pod_id='test_pod') + app.metrics.APP_LOOP_DURATION.labels.assert_any_call(pod_id='test_pod') + # APP_ERRORS_TOTAL would be checked similarly if it were called in this flow. + + # Check the .set() / .inc() calls on the mocks returned by .labels() + mock_app_env['metrics_APP_UP_labels_set'].assert_any_call(1) + mock_app_env['metrics_APP_UP_labels_set'].assert_any_call(0) + mock_app_env['metrics_APP_LOOP_COUNT_labels_inc'].assert_called_once() + + +@pytest.mark.asyncio +async def test_run_async_main(mock_app_env, capsys): + """Test the run_async_main function that sets up the event loop.""" + mock_ingestor_instance = mock_app_env['Ingestor'].return_value + mock_exit_signal = mock_app_env['mock_exit_signal'] + + mock_exit_signal.is_set.side_effect = [False, True] + mock_app_env['time_time'].side_effect = [10.0, 11.0] + + # Instead of calling run_async_main() which creates a new event loop, + # we test the main() function directly since that's what run_async_main() would call + with pytest.raises(OsExitCalledError) as excinfo: + await app.main() + assert excinfo.value.code == 0 + + # Verify the main function was called through the event loop + mock_app_env['start_http_server'].assert_called_once_with(9090) + mock_ingestor_instance.prepare_ingestor.assert_called_once() + mock_ingestor_instance.loop.assert_called_once() + mock_ingestor_instance.shutdown.assert_called_once() + + +@pytest.mark.asyncio +async def test_main_prepare_ingestor_failure(mock_app_env, capsys): + """Test that prepare_ingestor failure is handled correctly.""" + mock_ingestor_instance = mock_app_env['Ingestor'].return_value + mock_exit_signal = mock_app_env['mock_exit_signal'] + + mock_ingestor_instance.prepare_ingestor.side_effect = Exception('Preparation failed') + mock_exit_signal.is_set.side_effect = [False, True] + + with pytest.raises(OsExitCalledError) as excinfo: + await app.main() + assert excinfo.value.code == 0 + + # Verify error metrics were incremented + app.metrics.APP_ERRORS_TOTAL.labels.assert_called_with(pod_id='test_pod') + mock_app_env['metrics_APP_ERRORS_TOTAL_labels_inc'].assert_called_once() + + # Verify exit signal was set + mock_exit_signal.set.assert_called_once() + + # Verify shutdown was called + mock_ingestor_instance.shutdown.assert_called_once() + + # Verify error was logged + mock_ingestor_instance.logger.error.assert_called_once_with( + 'Failed to prepare ingestor: Preparation failed' + ) diff --git a/tests/unit/test_ingestor.py b/tests/unit/test_ingestor.py new file mode 100644 index 0000000..8bbccc4 --- /dev/null +++ b/tests/unit/test_ingestor.py @@ -0,0 +1,334 @@ +from unittest.mock import ANY, AsyncMock, MagicMock, call, patch + +from pytest import fixture, mark + +from ingestor.ingestor import Ingestor + + +@patch('ingestor.ingestor.getenv') +@patch('ingestor.ingestor.NotificationHandler') +def test___init__(notification_handler, getenv): + getenv.side_effect = [ + 'localhost', # REDIS_HOST + '63790', # REDIS_PORT + 'user', # REDIS_USERNAME + 'password', # REDIS_PASSWORD + '100', # LEASE_TTL + '200', # HEARTBEAT_TTL + 'localhost', # HOSTNAME + '50', # POLL_INTERVAL + 'localhost:27017', # MONGODB_URL + 'sientia', # MONGODB_USERNAME + 'sientia', # MONGODB_PASSWORD + 'sientia', # MONGODB_DATABASE + ] + + ingestor = Ingestor() + + getenv.assert_any_call('REDIS_HOST', 'localhost') + getenv.assert_any_call('REDIS_PORT', '6379') + getenv.assert_any_call('REDIS_USERNAME', None) + getenv.assert_any_call('REDIS_PASSWORD', None) + getenv.assert_any_call('LEASE_TTL', '10') + getenv.assert_any_call('HEARTBEAT_TTL', '20') + getenv.assert_any_call('HOSTNAME', 'localhost') + getenv.assert_any_call('POLL_INTERVAL', '5') + + assert ingestor.redis_host == 'localhost' + assert ingestor.redis_port == 63790 + assert ingestor.redis_username == 'user' + assert ingestor.redis_password == 'password' + assert ingestor.lease_ttl == 100 + assert ingestor.heartbeat_ttl == 200 + assert ingestor.pod_id == 'localhost' + assert ingestor.poll_interval == 50 + assert ingestor.metadata == { + 'model_id': '-', + 'model_name': '-', + 'workflow_name': 'opc_ingestor', + 'schema_name': 'opc_ingestor', + 'pod_id': 'localhost', + } + + notification_handler.assert_called_once_with( + connection_string='mongodb://sientia:sientia@localhost:27017', + database='sientia', + logger=ingestor.logger, + project_name='opc_ingestor', + ) + + +@fixture +@patch('ingestor.ingestor.getenv') +@patch('ingestor.ingestor.NotificationHandler') +def ingestor(_notification_handler, _getenv): + ing = Ingestor() + ing.logger = MagicMock() + + return ing + + +@fixture +def ingestor_manager_started(ingestor): + ingestor.ingestor_manager = AsyncMock( + initialize_opc_from_config=AsyncMock(), + shutdown=AsyncMock(), + update_opc_servers=AsyncMock(), + subscribe_to_tags=AsyncMock(), + unsubscribe_slot=AsyncMock(), + ) + return ingestor + + +@mark.asyncio +async def test_shutdown(ingestor_manager_started): + await ingestor_manager_started.shutdown() + + ingestor_manager_started.ingestor_manager.shutdown.assert_called_once() + + +@mark.asyncio +async def test_handle_acquired_tags_not_acquired(ingestor_manager_started): + await ingestor_manager_started.handle_acquired_tags([]) + + ingestor_manager_started.logger.warning.assert_called_once_with('No slots available') + ingestor_manager_started.ingestor_manager.update_opc_servers.assert_not_called() + ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_not_called() + + +@mark.asyncio +async def test_handle_acquired_tags_success(ingestor_manager_started): + await ingestor_manager_started.handle_acquired_tags(['tag1', 'tag2']) + + ingestor_manager_started.logger.warning.assert_not_called() + ingestor_manager_started.ingestor_manager.update_opc_servers.assert_called_once() + ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_called_once_with( + ['tag1', 'tag2'] + ) + + +@patch('ingestor.ingestor.IngestorManager') +@mark.asyncio +async def test_prepare_ingestor(ingestor_manager_mock, ingestor): + ingestor_manager = ingestor_manager_mock.return_value + ingestor_manager.get_slot_leases.return_value = True + + ingestor_manager.declare_active = AsyncMock() + ingestor_manager.get_slot_leases = AsyncMock() + ingestor.handle_acquired_tags = AsyncMock() + + await ingestor.prepare_ingestor() + + ingestor_manager_mock.assert_called_once_with( + redis_data={ + 'host': ingestor.redis_host, + 'port': ingestor.redis_port, + 'username': ingestor.redis_username, + 'password': ingestor.redis_password, + }, + lease_ttl=ingestor.lease_ttl, + heartbeat_ttl=ingestor.heartbeat_ttl, + poll_interval=ingestor.poll_interval, + mongo_connection_string=ingestor.mongo_connection_string, + mongo_database=ingestor.mongo_database, + metadata=ingestor.metadata, + logger=ingestor.logger, + notification_handler=ingestor.notification_handler, + metrics_controller=ingestor.metrics_controller, + ) + ingestor_manager.declare_active.assert_called_once() + ingestor_manager.get_slot_leases.assert_called_once() + + ingestor.handle_acquired_tags.assert_called_once_with( + ingestor_manager.get_slot_leases.return_value + ) + + +def test_manage_slots_has_slots(ingestor_manager_started): + ingestor_manager_started.handle_acquired_tags = MagicMock() + ingestor_manager_started.ingestor_manager.managed_tags = True + + ingestor_manager_started.manage_no_slots(5) + + ingestor_manager_started.ingestor_manager.get_slot_leases.assert_not_called() + ingestor_manager_started.ingestor_manager.handle_acquired_tags.assert_not_called() + + +def test_manage_no_slots_has_slots_none_available(ingestor_manager_started): + ingestor_manager_started.handle_acquired_tags = MagicMock() + ingestor_manager_started.ingestor_manager.managed_tags = True + + ingestor_manager_started.manage_no_slots(0) + + ingestor_manager_started.ingestor_manager.get_slot_leases.assert_not_called() + ingestor_manager_started.ingestor_manager.handle_acquired_tags.assert_not_called() + + +def test_manage_slots_none_available(ingestor_manager_started): + ingestor_manager_started.handle_acquired_tags = MagicMock() + ingestor_manager_started.ingestor_manager.managed_tags = False + + ingestor_manager_started.manage_no_slots(0) + + ingestor_manager_started.ingestor_manager.get_slot_leases.assert_not_called() + ingestor_manager_started.ingestor_manager.handle_acquired_tags.assert_not_called() + + +@mark.asyncio +async def test_manage_slots_none_available_none_available(ingestor_manager_started): + ingestor_manager_started.handle_acquired_tags = MagicMock() + ingestor_manager_started.ingestor_manager.managed_tags = False + + await ingestor_manager_started.manage_no_slots(2) + + ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with(1) + + +@mark.asyncio +async def test_manage_leases_no_ingestor_manager(ingestor_manager_started): + ingestor_manager_started.ingestor_manager = None + + assert await ingestor_manager_started.manage_leases(2, 2, 5) is None + + +@mark.asyncio +async def test_manage_leases_no_available_slots_no_extra_slots(ingestor_manager_started): + ingestor_manager_started.handle_acquired_tags = MagicMock() + + await ingestor_manager_started.manage_leases(0, 0, 0) + + ingestor_manager_started.ingestor_manager.get_slot_leases.assert_not_called() + ingestor_manager_started.ingestor_manager.handle_acquired_tags.assert_not_called() + ingestor_manager_started.ingestor_manager.drop_slot_leases.assert_not_called() + + +@mark.asyncio +async def test_manage_leases_available_slots_innactive_ingestors(ingestor_manager_started): + ingestor_manager_started.handle_acquired_tags = MagicMock() + + await ingestor_manager_started.manage_leases(2, 2, 5) + + ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with(2) + + ingestor_manager_started.ingestor_manager.drop_slot_leases.assert_not_called() + + +@mark.asyncio +async def test_manage_leases_no_available_slots_extra_sltos(ingestor_manager_started): + ingestor_manager_started.handle_acquired_tags = MagicMock() + ingestor_manager_started.ingestor_manager.managed_tags = { + 'tag1': 'server1', + 'tag2': 'server2', + 'tag3': 'server3', + } + + await ingestor_manager_started.manage_leases(0, 0, 2) + + ingestor_manager_started.ingestor_manager.get_slot_leases.assert_not_called() + ingestor_manager_started.handle_acquired_tags.assert_not_called() + ingestor_manager_started.ingestor_manager.drop_slot_leases.assert_called_once_with( + ['tag2', 'tag3'] + ) + + +@mark.asyncio +async def test_loop(ingestor_manager_started): + ingestor_manager_started.manage_no_slots = AsyncMock() + ingestor_manager_started.manage_leases = AsyncMock() + ingestor_manager_started.update_ingestor_manager = AsyncMock() + ingestor_manager_started.ingestor_manager.check_opc_servers_integrity = AsyncMock() + ingestor_manager_started.ingestor_manager.managed_tags = { + 'slot1': 'server1', + 'slot2': 'server2', + 'slot3': 'server3', + } + ingestor_manager_started.ingestor_manager.get_active_ingestors = AsyncMock( + return_value=['ingestor1', 'ingestor2'] + ) + ingestor_manager_started.ingestor_manager.get_number_of_slots = AsyncMock(return_value=5) + ingestor_manager_started.ingestor_manager.get_number_of_leases = AsyncMock(return_value=1) + + await ingestor_manager_started.loop() + + ingestor_manager_started.ingestor_manager.declare_active.assert_called_once() + ingestor_manager_started.ingestor_manager.get_active_ingestors.assert_called_once() + ingestor_manager_started.ingestor_manager.get_number_of_slots.assert_called_once() + + ingestor_manager_started.manage_no_slots.assert_called_once_with( + ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value + ) + # Explanation: 5 - 2 = 3, 3 - 1 = 2 + ingestor_manager_started.manage_leases.assert_called_once_with(4, 3, 2) + ingestor_manager_started.ingestor_manager.update_slot_config.assert_called_once() + + ingestor_manager_started.ingestor_manager.check_opc_servers_integrity.assert_called_once() + ingestor_manager_started.update_ingestor_manager.assert_called_once() + + +@mark.asyncio +async def test_loop_no_managed(ingestor_manager_started): + ingestor_manager_started.manage_no_slots = AsyncMock() + ingestor_manager_started.manage_leases = AsyncMock() + ingestor_manager_started.update_ingestor_manager = AsyncMock() + ingestor_manager_started.ingestor_manager.check_opc_servers_integrity = AsyncMock() + ingestor_manager_started.ingestor_manager.managed_tags = {} + ingestor_manager_started.ingestor_manager.get_active_ingestors = AsyncMock( + return_value=['ingestor1', 'ingestor2'] + ) + ingestor_manager_started.ingestor_manager.get_number_of_slots = AsyncMock(return_value=5) + + await ingestor_manager_started.loop() + + ingestor_manager_started.ingestor_manager.declare_active.assert_called_once() + ingestor_manager_started.ingestor_manager.get_active_ingestors.assert_called_once() + ingestor_manager_started.ingestor_manager.get_number_of_slots.assert_called_once() + + ingestor_manager_started.manage_no_slots.assert_called_once_with( + ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value + ) + # Explanation: 5 - 2 = 3, 3 - 1 = 2 + ingestor_manager_started.manage_leases.assert_called_once_with(ANY, 3, -1) + ingestor_manager_started.ingestor_manager.update_slot_config.assert_called_once() + ingestor_manager_started.logger.info.assert_any_call('No slots acquired in this loop') + + ingestor_manager_started.ingestor_manager.check_opc_servers_integrity.assert_called_once() + ingestor_manager_started.update_ingestor_manager.assert_called_once() + + +@mark.asyncio +async def test_loop_no_ingestor_manager(ingestor_manager_started): + ingestor_manager_started.ingestor_manager = None + + assert await ingestor_manager_started.loop() is None + + +@mark.asyncio +async def test_update_ingestor_manager(ingestor_manager_started): + ingestor_manager_started.ingestor_manager.managed_tags = { + 'slot_to_create': 'new_config', + 'slot_to_update': 'new_config', + 'slot_to_do_nothing': 'old_config', + } + + old_managed_tags = { + 'slot_to_update': 'old_config', + 'slot_to_delete': 'old_config', + 'slot_to_do_nothing': 'old_config', + } + + await ingestor_manager_started.update_ingestor_manager(old_managed_tags) + + ingestor_manager_started.ingestor_manager.update_opc_servers.assert_called_once() + ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_has_calls( + [call({'slot_to_create': 'new_config'}), call({'slot_to_update': 'new_config'})] + ) + ingestor_manager_started.ingestor_manager.unsubscribe_slot.assert_has_calls( + [call('slot_to_update'), call('slot_to_delete')] + ) + + +@mark.asyncio +async def test_update_ingestor_manager_no_ingestor_manager(ingestor_manager_started): + ingestor_manager_started.ingestor_manager = None + + await ingestor_manager_started.update_ingestor_manager({}) diff --git a/tests/unit/test_metrics.py b/tests/unit/test_metrics.py new file mode 100644 index 0000000..3c50dea --- /dev/null +++ b/tests/unit/test_metrics.py @@ -0,0 +1,191 @@ +# tests/unit/test_metrics.py + +from prometheus_client import Counter, Gauge, Histogram + +import ingestor.metrics as metrics + +# --- Test Functions for Each Metric (Corrected for v0.22.0 _name behavior) --- + + +def test_ingestor_tag_written_count(): + """Verify the definition of INGESTOR_TAG_WRITTEN_COUNT.""" + assert metrics.TAG_WRITTEN_COUNT is not None + assert isinstance(metrics.TAG_WRITTEN_COUNT, Counter) + assert metrics.TAG_WRITTEN_COUNT._name == 'ingestor_tag_written_count' + assert set(metrics.TAG_WRITTEN_COUNT._labelnames) == {'pod_id', 'tag_name', 'collection_name'} + + +def test_app_loop_count(): + """Verify the definition of APP_LOOP_COUNT.""" + assert metrics.APP_LOOP_COUNT is not None + assert isinstance(metrics.APP_LOOP_COUNT, Counter) + assert metrics.APP_LOOP_COUNT._name == 'app_main_loop' # REMOVED _total + assert set(metrics.APP_LOOP_COUNT._labelnames) == {'pod_id'} + + +def test_app_loop_duration(): + """Verify the definition of APP_LOOP_DURATION.""" + assert metrics.APP_LOOP_DURATION is not None + assert isinstance(metrics.APP_LOOP_DURATION, Histogram) + assert ( + metrics.APP_LOOP_DURATION._name == 'app_main_loop_duration_seconds' + ) # Histograms don't have _total + assert set(metrics.APP_LOOP_DURATION._labelnames) == {'pod_id'} + + +def test_app_errors_total(): + """Verify the definition of APP_ERRORS_TOTAL.""" + assert metrics.APP_ERRORS_TOTAL is not None + assert isinstance(metrics.APP_ERRORS_TOTAL, Counter) + assert metrics.APP_ERRORS_TOTAL._name == 'app_errors' # REMOVED _total + assert set(metrics.APP_ERRORS_TOTAL._labelnames) == {'pod_id'} + + +def test_app_up(): + """Verify the definition of APP_UP.""" + assert metrics.APP_UP is not None + assert isinstance(metrics.APP_UP, Gauge) + assert metrics.APP_UP._name == 'app_up' # Gauges don't have _total + assert set(metrics.APP_UP._labelnames) == {'pod_id'} + + +def test_active_ingestors(): + """Verify the definition of ACTIVE_INGESTORS.""" + assert metrics.ACTIVE_INGESTORS is not None + assert isinstance(metrics.ACTIVE_INGESTORS, Gauge) + assert metrics.ACTIVE_INGESTORS._name == 'ingestor_active_total' + assert set(metrics.ACTIVE_INGESTORS._labelnames) == {'pod_id'} + + +def test_slots_total(): + """Verify the definition of SLOTS_TOTAL.""" + assert metrics.SLOTS_TOTAL is not None + assert isinstance(metrics.SLOTS_TOTAL, Gauge) + assert metrics.SLOTS_TOTAL._name == 'ingestor_slots_total' + assert set(metrics.SLOTS_TOTAL._labelnames) == {'pod_id'} + + +def test_leases_total(): + """Verify the definition of LEASES_TOTAL.""" + assert metrics.LEASES_TOTAL is not None + assert isinstance(metrics.LEASES_TOTAL, Gauge) + assert metrics.LEASES_TOTAL._name == 'ingestor_leases_total' + assert set(metrics.LEASES_TOTAL._labelnames) == {'pod_id'} + + +def test_slots_managed(): + """Verify the definition of SLOTS_MANAGED.""" + assert metrics.SLOTS_MANAGED is not None + assert isinstance(metrics.SLOTS_MANAGED, Gauge) + assert metrics.SLOTS_MANAGED._name == 'ingestor_slots_managed_current' + assert set(metrics.SLOTS_MANAGED._labelnames) == {'pod_id'} + + +def test_slots_acquired(): + """Verify the definition of SLOTS_ACQUIRED.""" + assert metrics.SLOTS_ACQUIRED is not None + assert isinstance(metrics.SLOTS_ACQUIRED, Counter) + assert metrics.SLOTS_ACQUIRED._name == 'ingestor_slots_acquired' # REMOVED _total + assert set(metrics.SLOTS_ACQUIRED._labelnames) == {'pod_id'} + + +def test_slots_released(): + """Verify the definition of SLOTS_RELEASED.""" + assert metrics.SLOTS_RELEASED is not None + assert isinstance(metrics.SLOTS_RELEASED, Counter) + assert metrics.SLOTS_RELEASED._name == 'ingestor_slots_released' # REMOVED _total + assert set(metrics.SLOTS_RELEASED._labelnames) == {'pod_id'} + + +def test_opc_managers_active(): + """Verify the definition of OPC_MANAGERS_ACTIVE.""" + assert metrics.OPC_MANAGERS_ACTIVE is not None + assert isinstance(metrics.OPC_MANAGERS_ACTIVE, Gauge) + assert metrics.OPC_MANAGERS_ACTIVE._name == 'ingestor_opc_managers_active' + assert set(metrics.OPC_MANAGERS_ACTIVE._labelnames) == {'pod_id'} + + +def test_opc_subscription_errors(): + """Verify the definition of OPC_SUBSCRIPTION_ERRORS.""" + assert metrics.OPC_SUBSCRIPTION_ERRORS is not None + assert isinstance(metrics.OPC_SUBSCRIPTION_ERRORS, Counter) + assert ( + metrics.OPC_SUBSCRIPTION_ERRORS._name == 'ingestor_opc_subscription_errors' + ) # REMOVED _total + assert set(metrics.OPC_SUBSCRIPTION_ERRORS._labelnames) == { + 'pod_id', + 'server', + 'slot', + } + + +def test_opc_connections_total(): + """Verify the definition of OPC_CONNECTIONS_TOTAL.""" + assert metrics.OPC_CONNECTIONS_TOTAL is not None + assert isinstance(metrics.OPC_CONNECTIONS_TOTAL, Counter) + assert metrics.OPC_CONNECTIONS_TOTAL._name == 'opc_connections_initiated' # REMOVED _total + assert set(metrics.OPC_CONNECTIONS_TOTAL._labelnames) == {'pod_id', 'server_name'} + + +def test_opc_connections_failed(): + """Verify the definition of OPC_CONNECTIONS_FAILED.""" + assert metrics.OPC_CONNECTIONS_FAILED is not None + assert isinstance(metrics.OPC_CONNECTIONS_FAILED, Counter) + assert metrics.OPC_CONNECTIONS_FAILED._name == 'opc_connections_failed' # REMOVED _total + assert set(metrics.OPC_CONNECTIONS_FAILED._labelnames) == {'pod_id', 'server_name'} + + +def test_opc_connection_status(): + """Verify the definition of OPC_CONNECTION_STATUS.""" + assert metrics.OPC_CONNECTION_STATUS is not None + assert isinstance(metrics.OPC_CONNECTION_STATUS, Gauge) + assert metrics.OPC_CONNECTION_STATUS._name == 'opc_connection_status' + assert set(metrics.OPC_CONNECTION_STATUS._labelnames) == { + 'pod_id', + 'server_name', + 'server_url', + } + + +def test_opc_subscriptions_created(): + """Verify the definition of OPC_SUBSCRIPTIONS_CREATED.""" + assert metrics.OPC_SUBSCRIPTIONS_CREATED is not None + assert isinstance(metrics.OPC_SUBSCRIPTIONS_CREATED, Counter) + assert metrics.OPC_SUBSCRIPTIONS_CREATED._name == 'opc_subscriptions_created' # REMOVED _total + assert set(metrics.OPC_SUBSCRIPTIONS_CREATED._labelnames) == { + 'pod_id', + 'server_name', + 'slot_name', + } + + +def test_opc_tags_subscribed(): + """Verify the definition of OPC_TAGS_SUBSCRIBED.""" + assert metrics.OPC_TAGS_SUBSCRIBED is not None + assert isinstance(metrics.OPC_TAGS_SUBSCRIBED, Gauge) + assert metrics.OPC_TAGS_SUBSCRIBED._name == 'opc_tags_subscribed_current' + assert set(metrics.OPC_TAGS_SUBSCRIBED._labelnames) == {'pod_id', 'server_name'} + + +def test_opc_cycles_without_data(): + """Verify the definition of OPC_CYCLES_WITHOUT_DATA.""" + assert metrics.OPC_CYCLES_WITHOUT_DATA is not None + assert isinstance(metrics.OPC_CYCLES_WITHOUT_DATA, Gauge) + assert metrics.OPC_CYCLES_WITHOUT_DATA._name == 'opc_cycles_without_data' + assert set(metrics.OPC_CYCLES_WITHOUT_DATA._labelnames) == {'pod_id', 'server_name'} + + +def test_opc_reconnections_total(): + """Verify the definition of OPC_RECONNECTIONS_TOTAL.""" + assert metrics.OPC_RECONNECTIONS_TOTAL is not None + assert isinstance(metrics.OPC_RECONNECTIONS_TOTAL, Counter) + assert metrics.OPC_RECONNECTIONS_TOTAL._name == 'opc_reconnections_tried' # REMOVED _total + assert set(metrics.OPC_RECONNECTIONS_TOTAL._labelnames) == {'pod_id', 'server_name'} + + +def test_notifications_sent(): + """Verify the definition of NOTIFICATIONS_SENT.""" + assert metrics.NOTIFICATIONS_SENT is not None + assert isinstance(metrics.NOTIFICATIONS_SENT, Counter) + assert metrics.NOTIFICATIONS_SENT._name == 'notifications_sent' # REMOVED _total + assert set(metrics.NOTIFICATIONS_SENT._labelnames) == {'pod_id', 'level', 'block'} diff --git a/validate.sh b/validate.sh new file mode 100755 index 0000000..5e837c6 --- /dev/null +++ b/validate.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# Model Manager Code Validation Script +# This script runs all code quality checks before committing or deploying + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Args +FIX_MODE=false +while [[ $# -gt 0 ]]; do + case "$1" in + --fix) + FIX_MODE=true + shift + ;; + -h|--help) + echo "Usage: $0 [--fix]" + echo " --fix Apply Ruff auto-fixes (format and lint fixes)." + exit 0 + ;; + *) + echo -e "${RED}Unknown option: $1${NC}" + echo "Usage: $0 [--fix]" + exit 2 + ;; + esac +done + +echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Model Manager - Code Validation Suite ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}" +echo "" + +# Check if virtual environment is activated +if [[ -z "${VIRTUAL_ENV}" ]] && [[ -z "${CONDA_DEFAULT_ENV}" ]]; then + echo -e "${YELLOW}⚠️ Warning: No virtual environment detected${NC}" + echo -e "${YELLOW} Consider activating your venv/conda environment${NC}" + echo "" +fi + +# Function to run a validation step +run_step() { + local step_name=$1 + local step_command=$2 + + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${BLUE}▶ ${step_name}${NC}" + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + + if eval "$step_command"; then + echo -e "${GREEN}✅ ${step_name} - PASSED${NC}" + echo "" + return 0 + else + echo -e "${RED}❌ ${step_name} - FAILED${NC}" + echo "" + return 1 + fi +} + +# Track failures +FAILED_STEPS=() + +# Step 1: Code Formatting Check (Ruff) +# - default: check only +# - --fix: write changes +if ! run_step "1. Code Formatting (Ruff)" "if \$FIX_MODE; then ruff format ingestor/ tests/; else ruff format --check ingestor/ tests/; fi"; then + FAILED_STEPS+=("Code Formatting") +fi + +# Step 2: Linting (Ruff) +# - default: check only +# - --fix: apply autofixes +if ! run_step "2. Code Linting (Ruff)" "if \$FIX_MODE; then ruff check --fix ingestor/ tests/; else ruff check ingestor/ tests/; fi"; then + FAILED_STEPS+=("Linting") +fi + +# Step 3: Type Checking (mypy) +if ! run_step "3. Type Checking (mypy)" "mypy ingestor/"; then + FAILED_STEPS+=("Type Checking") +fi + +# Step 4: Security Analysis (Bandit) +if ! run_step "4. Security Analysis (Bandit)" "bandit -r ingestor/ -ll -q"; then + FAILED_STEPS+=("Security Analysis") +fi + +# Step 5: Unit Tests (pytest) +if ! run_step "5. Unit Tests (pytest)" "pytest tests/ --cov=ingestor --cov-report=term-missing --cov-report=xml --cov-report=html --cov-fail-under=80 -q"; then + FAILED_STEPS+=("Unit Tests") +fi + +# Summary +echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Validation Summary ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}" +echo "" + +if [ ${#FAILED_STEPS[@]} -eq 0 ]; then + echo -e "${GREEN}✅ All validation checks passed!${NC}" + echo -e "${GREEN} Your code is ready for commit/deployment.${NC}" + echo "" + exit 0 +else + echo -e "${RED}❌ Validation failed for the following steps:${NC}" + for step in "${FAILED_STEPS[@]}"; do + echo -e "${RED} • ${step}${NC}" + done + echo "" + echo -e "${YELLOW}💡 Tips:${NC}" + echo -e "${YELLOW} • Run 'ruff format ingestor/ tests/' to auto-fix formatting${NC}" + echo -e "${YELLOW} • Run 'ruff check --fix ingestor/ tests/' to auto-fix linting issues${NC}" + echo -e "${YELLOW} • Review mypy errors and add type hints where needed${NC}" + echo -e "${YELLOW} • Check bandit warnings for security issues${NC}" + echo -e "${YELLOW} • Fix failing tests or improve test coverage${NC}" + echo "" + exit 1 +fi \ No newline at end of file