From 4c2e9288df2420f879bfab653022d625106bc4c7 Mon Sep 17 00:00:00 2001 From: gitea_admin Date: Wed, 5 Aug 2026 13:53:40 +0000 Subject: [PATCH] Code import - branch 1.3.0 --- .env.example | 28 + .github/workflows/quality-gate.yml | 17 + .github/workflows/release.yml | 16 + .gitignore | 50 + README.md | 689 +++++ e2e/README.md | 37 + e2e/__init__.py | 0 e2e/conftest.py | 324 +++ e2e/db_schema.sql | 16 + e2e/fixtures/__init__.py | 0 e2e/helpers.py | 404 +++ e2e/pi_web_api_test_server.py | 124 + .../core_scouter_aggregation_avg.json | 24 + .../core_scouter_aggregation_lts.json | 24 + .../core_scouter_aggregation_max.json | 24 + .../core_scouter_aggregation_mdn.json | 24 + .../core_scouter_aggregation_min.json | 24 + .../core_scouter_debug_data_package.json | 22 + .../core_scouter_empty_after_grouping.json | 24 + .../core_scouter_fill_missing_tags.json | 24 + .../core_scouter_happy_path.json | 28 + .../core_scouter_invalid_aggregation.json | 23 + ...re_scouter_null_values_filter_discard.json | 23 + .../core_scouter_null_values_filter_warn.json | 23 + ..._scouter_out_of_bounds_filter_discard.json | 23 + .../core_scouter_postgres_export_failure.json | 21 + .../pi_web_api_scouter_connection_error.json | 23 + ...pi_web_api_scouter_debug_data_package.json | 29 + .../pi_web_api_scouter_empty_response.json | 23 + .../pi_web_api_scouter_happy_path.json | 30 + .../pi_web_api_scouter_invalid_endpoint.json | 23 + .../pi_web_api_scouter_multiple_tags.json | 53 + .../pi_web_api_scouter_timeout.json | 23 + e2e/scenario_inputs/scouter_empty_mongo.json | 23 + e2e/scenario_inputs/scouter_happy_path.json | 38 + .../scouter_incremental_load.json | 43 + .../scouter_no_redis_timestamp_first_run.json | 31 + e2e/scenarios.md | 138 + e2e/test_harness_smoke.py | 51 + e2e/test_pi_web_api_scouter_main_workflow.py | 222 ++ e2e/test_scouter_main_workflow.py | 154 ++ e2e/test_subworkflow_core_scouter.py | 360 +++ get_data_pims.py | 604 ++++ git-requirements-mapping.txt | 1 + init_port_forward.sh | 133 + pi_web_api_fetch_data.py | 306 +++ pyproject.toml | 159 ++ requirements-dev.txt | 23 + requirements-local.txt | 8 + requirements.txt | 8 + run_coverage.sh | 11 + run_local.sh | 18 + scouter/__init__.py | 0 scouter/activities/__init__.py | 0 scouter/activities/activities.py | 132 + scouter/activities/api.py | 157 ++ scouter/activities/gates.py | 319 +++ scouter/activities/mongodb.py | 154 ++ scouter/activities/redis.py | 311 +++ scouter/metrics.py | 25 + scouter/utils/__init__.py | 0 scouter/utils/connectors_config.py | 19 + scouter/utils/quality/filters.py | 81 + scouter/worker/__init__.py | 0 scouter/worker/worker.py | 187 ++ scouter/workflow/__init__.py | 0 scouter/workflow/pi_web_api_scouter.py | 108 + scouter/workflow/scouter.py | 120 + .../workflow/sub_workflows/core_scouter.py | 154 ++ sonar-project.properties | 11 + tests.ipynb | 2433 +++++++++++++++++ tests/__init__.py | 0 tests/activities/__init__.py | 0 tests/activities/test_activities.py | 194 ++ tests/activities/test_api.py | 340 +++ tests/activities/test_gates.py | 409 +++ tests/activities/test_mongo.py | 174 ++ tests/activities/test_redis.py | 525 ++++ tests/test_metrics.py | 32 + tests/utils/__init__.py | 0 tests/utils/quality/test_filters.py | 139 + tests/utils/test_connectors_config.py | 42 + tests/worker/__init__.py | 0 tests/workflow/__init__.py | 0 .../sub_workflows/test_core_scouter.py | 403 +++ tests/workflow/test_pi_web_api_scouter.py | 133 + tests/workflow/test_scouter.py | 129 + 87 files changed, 11302 insertions(+) create mode 100644 .env.example create mode 100644 .github/workflows/quality-gate.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 e2e/README.md create mode 100644 e2e/__init__.py create mode 100644 e2e/conftest.py create mode 100644 e2e/db_schema.sql create mode 100644 e2e/fixtures/__init__.py create mode 100644 e2e/helpers.py create mode 100644 e2e/pi_web_api_test_server.py create mode 100644 e2e/scenario_inputs/core_scouter_aggregation_avg.json create mode 100644 e2e/scenario_inputs/core_scouter_aggregation_lts.json create mode 100644 e2e/scenario_inputs/core_scouter_aggregation_max.json create mode 100644 e2e/scenario_inputs/core_scouter_aggregation_mdn.json create mode 100644 e2e/scenario_inputs/core_scouter_aggregation_min.json create mode 100644 e2e/scenario_inputs/core_scouter_debug_data_package.json create mode 100644 e2e/scenario_inputs/core_scouter_empty_after_grouping.json create mode 100644 e2e/scenario_inputs/core_scouter_fill_missing_tags.json create mode 100644 e2e/scenario_inputs/core_scouter_happy_path.json create mode 100644 e2e/scenario_inputs/core_scouter_invalid_aggregation.json create mode 100644 e2e/scenario_inputs/core_scouter_null_values_filter_discard.json create mode 100644 e2e/scenario_inputs/core_scouter_null_values_filter_warn.json create mode 100644 e2e/scenario_inputs/core_scouter_out_of_bounds_filter_discard.json create mode 100644 e2e/scenario_inputs/core_scouter_postgres_export_failure.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_connection_error.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_debug_data_package.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_empty_response.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_happy_path.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_invalid_endpoint.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_multiple_tags.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_timeout.json create mode 100644 e2e/scenario_inputs/scouter_empty_mongo.json create mode 100644 e2e/scenario_inputs/scouter_happy_path.json create mode 100644 e2e/scenario_inputs/scouter_incremental_load.json create mode 100644 e2e/scenario_inputs/scouter_no_redis_timestamp_first_run.json create mode 100644 e2e/scenarios.md create mode 100644 e2e/test_harness_smoke.py create mode 100644 e2e/test_pi_web_api_scouter_main_workflow.py create mode 100644 e2e/test_scouter_main_workflow.py create mode 100644 e2e/test_subworkflow_core_scouter.py create mode 100644 get_data_pims.py create mode 100644 git-requirements-mapping.txt create mode 100755 init_port_forward.sh create mode 100644 pi_web_api_fetch_data.py create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 requirements-local.txt create mode 100644 requirements.txt create mode 100755 run_coverage.sh create mode 100755 run_local.sh create mode 100644 scouter/__init__.py create mode 100644 scouter/activities/__init__.py create mode 100644 scouter/activities/activities.py create mode 100644 scouter/activities/api.py create mode 100644 scouter/activities/gates.py create mode 100644 scouter/activities/mongodb.py create mode 100644 scouter/activities/redis.py create mode 100644 scouter/metrics.py create mode 100644 scouter/utils/__init__.py create mode 100644 scouter/utils/connectors_config.py create mode 100644 scouter/utils/quality/filters.py create mode 100644 scouter/worker/__init__.py create mode 100644 scouter/worker/worker.py create mode 100644 scouter/workflow/__init__.py create mode 100644 scouter/workflow/pi_web_api_scouter.py create mode 100644 scouter/workflow/scouter.py create mode 100644 scouter/workflow/sub_workflows/core_scouter.py create mode 100644 sonar-project.properties create mode 100644 tests.ipynb create mode 100644 tests/__init__.py create mode 100644 tests/activities/__init__.py create mode 100644 tests/activities/test_activities.py create mode 100644 tests/activities/test_api.py create mode 100644 tests/activities/test_gates.py create mode 100644 tests/activities/test_mongo.py create mode 100644 tests/activities/test_redis.py create mode 100644 tests/test_metrics.py create mode 100644 tests/utils/__init__.py create mode 100644 tests/utils/quality/test_filters.py create mode 100644 tests/utils/test_connectors_config.py create mode 100644 tests/worker/__init__.py create mode 100644 tests/workflow/__init__.py create mode 100644 tests/workflow/sub_workflows/test_core_scouter.py create mode 100644 tests/workflow/test_pi_web_api_scouter.py create mode 100644 tests/workflow/test_scouter.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..9992938 --- /dev/null +++ b/.env.example @@ -0,0 +1,28 @@ +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +POSTGRES_USER=sientia +POSTGRES_PASSWORD=sientia +POSTGRES_DB=sientia +POSTGRES_MIN_CONNECTIONS=5 +POSTGRES_MAX_CONNECTIONS=20 + +MONGODB_USERNAME="root" +MONGODB_PASSWORD="password" +MONGODB_URL="my-release-mongodb.mongodb.svc.cluster.local:27017" +MONGODB_DATABASE="sientia" + +REDIS_HOST=localhost +REDIS_PORT=6379 +REDIS_USERNAME="user" +REDIS_PASSWORD="pass" + +TEMPORAL_HOST=localhost:7233 +TEMPORAL_NAMESPACE=scouter + +PI_WEB_API_BASE_URL="https://piwebapi.link.com/piwebapi" +PI_WEB_API_AUTH_TYPE="basic" +PI_WEB_API_AUTH_TOKEN="password" + +LOG_LEVEL=INFO + +PROJECT_NAME=scouter diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml new file mode 100644 index 0000000..e61b588 --- /dev/null +++ b/.github/workflows/quality-gate.yml @@ -0,0 +1,17 @@ +name: Quality gate + +on: + pull_request: + branches: + - main + types: [ opened, synchronize, reopened ] + +jobs: + quality-gate: + uses: Aignosi/github_workflow_templates/.github/workflows/dataops-module-quality-gate.yml@main + permissions: write-all + with: + project_name: 'scouter' + repositories: 'sientia-dataops-library' + requirements_file: 'requirements-local.txt' + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4c48fb6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,16 @@ +name: Create Release on Merge to Main + +on: + pull_request: + types: [closed] + branches: + - main + +jobs: + release: + if: github.event.pull_request.merged == true + uses: Aignosi/github_workflow_templates/.github/workflows/dataops-module-release.yml@main + permissions: write-all + with: + project_name: 'scouter' + secrets: inherit \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..76ab9b4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,50 @@ +# Ignorar volumes do Docker +docker-compose.override.yml +**/db_data/ +**/kafka-volume/ +**/zookeeper-volume/ +**/mage_data/ +**/minio_data/ +**/venv/ +**/certs/*.pem +**/certs/*.der +**/certs/*.csr +**/deploy/*.yaml +scouter/.file_versions/ +scouter/pipelines/**/triggers.yaml +**/postgres_data/** +# Ignorar arquivos e diretΓ³rios de cache do Python +__pycache__/ +*.pyc +*.pyo +*.pyd + +# Ignorar logs +*.log + +# Ignorar arquivos de configuração locais +.vscode/ +.pytest_cache/ +.idea/ +*.swp + +# Ignorar arquivos temporΓ‘rios +*.tmp +*.bak +*.old +.secret + +# Ignorar coverage +htmlcov/ +.coverage +coverage.xml + +git_log + +.env +.ruff_cache/ +.mypy_cache/ + +.cursor +openspec +collect_scripts \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..739c2bd --- /dev/null +++ b/README.md @@ -0,0 +1,689 @@ +# Sientia DataOps Scouter + +A high-performance, scalable data processing and ML model orchestration system built on Temporal.io for industrial data collection, processing, and analytics. The Scouter system provides enterprise-grade data ingestion from multiple sources with automatic data quality validation, aggregation, and export capabilities. + +## Features + +### Core Functionality +- **Multi-Source Data Ingestion**: Support for Kafka topics, direct OPC server access, PI Web API endpoints, and real-time triggers +- **Temporal Workflow Orchestration**: Robust workflow management with automatic retry policies and fault tolerance +- **Data Quality Gates**: Configurable filtering for null values, out-of-bounds data, and custom validation rules +- **Time-Series Aggregation**: Flexible aggregation functions (average, median, max, min, latest) with configurable parameters +- **Multi-Database Integration**: PostgreSQL for persistent storage, Redis for caching, MongoDB for data retrieval +- **Real-time Monitoring**: Prometheus metrics and comprehensive logging for operational visibility + +### Advanced Capabilities +- **Incremental Data Processing**: Timestamp-based data loading to avoid reprocessing +- **Configurable Data Retention**: Redis-based temporary storage with TTL management +- **Notification System**: Integrated alerting and notification management via MongoDB +- **Scalable Architecture**: Kubernetes-ready deployment with horizontal scaling support +- **Debug Mode**: Optional data package storage for debugging and troubleshooting +- **Worker Autoscaling**: Configurable poller behavior with aggressive autoscaling policies + +# Architecture + +The Scouter system uses a Temporal-based workflow architecture with clear separation of concerns: + +### Key Components + +- **Worker**: Main application orchestrator managing Temporal workers and task queues +- **Workflows**: Temporal workflow definitions for data processing orchestration +- **Activities**: Temporal activities implementing data processing operations +- **Data Services**: Database connectors and data access layer +- **Quality Filters**: Configurable data validation and filtering mechanisms + +## πŸ”„ Workflows + +The Scouter system implements a parent-child workflow pattern for data processing orchestration. + +### 1. Scouter Workflow (`scouter.py`) + +The **Scouter** workflow is the main entry point for data processing pipelines. It orchestrates the complete data ingestion process and implements a robust incremental data processing pattern. + +#### Purpose +- **Data Ingestion Orchestration**: Coordinates data loading from MongoDB collections +- **Timestamp Management**: Tracks last processed timestamps to enable incremental processing +- **Workflow Delegation**: Delegates actual data processing to the CoreScouter workflow +- **Data Continuity**: Ensures no data is lost or reprocessed between executions + +#### Execution Flow +1. **Timestamp Retrieval**: Gets the last processed timestamp from Redis for the specific workflow and schedule +2. **Data Loading**: Loads new data from MongoDB since the last timestamp using the collection name `raw_{schedule_name}` +3. **Timestamp Update**: Updates the last processed timestamp with the most recent data point +4. **Data Processing**: Delegates data processing to the CoreScouter child workflow +5. **Metadata Management**: Maintains workflow execution metadata throughout the process + +#### Key Features +- **Incremental Processing**: Only processes new data since last execution +- **Automatic Retry**: Implements Temporal retry policies for fault tolerance +- **Timeout Management**: 60-second timeout for all activities +- **Error Handling**: Comprehensive error handling with notification integration + +#### Input Parameters +```json +{ + "topic": "sensor_data_topic", + "schedule_name": "hourly_collection", + "model_name": "temperature_sensors", + "model_id": "temp_001", + "trigger_laborious": false, + "filters": {...}, + "schema": "sensor_data", + "table_name": "temperature_readings", + "retention_time": 3600, + "model_tags": {...} +} +``` + +#### Architecture + +```mermaid +flowchart LR + A[1. get_last_data_timestamp] --> B[2. load_latest_data] --> C[3. put_last_data_timestamp] --> D[4. core_scouter πŸ”ƒ] + + A -.-> Redis[(Redis)] + B -.-> MongoDB[(MongoDB)] + C -.-> Redis +``` + + +### 2. CoreScouter Workflow (`core_scouter.py`) + +The **CoreScouter** workflow implements the core data processing pipeline for industrial time-series data. It handles data quality validation, aggregation, and export operations. + +#### Purpose +- **Data Quality Validation**: Applies configurable filters for data integrity +- **Time-Series Aggregation**: Groups and aggregates data using specified functions +- **Data Organization**: Groups data by tags and applies retention policies +- **Persistent Storage**: Exports processed data to PostgreSQL +- **Metrics Collection**: Records processing metrics for monitoring + +#### Execution Flow +1. **Data Quality Gate**: Applies configured filters (null values, out-of-bounds, custom rules) +2. **Data Aggregation**: Groups data by tag and name, applies aggregation functions +3. **Data Grouping**: Organizes data and stores temporarily in Redis with TTL +4. **Data Export**: Persists processed data to PostgreSQL database with timestamp conversion +5. **Metrics Recording**: Writes processing metrics for operational visibility + +**Note**: The data export step uses timestamp conversion to ensure consistent datetime +formatting. The export operation receives the schema, table name, data, and timestamp +conversion configuration. Conflict resolution and unique column constraints are handled +by the underlying PostgreSQL activity implementation. + +#### Aggregation Functions +- **`lts`**: Latest value (most recent data point) +- **`avg`**: Average of all values in the group +- **`mdn`**: Median of all values in the group +- **`max`**: Maximum value in the group +- **`min`**: Minimum value in the group + +#### Key Features +- **Configurable Quality Gates**: Multiple filter types with policy-based configuration +- **Flexible Aggregation**: Tag-specific aggregation function configuration +- **Batch Processing**: Efficient handling of large datasets +- **Asynchronous Export**: Non-blocking data export operations +- **Comprehensive Monitoring**: Detailed metrics and error reporting + +#### Input Parameters +```json +{ + "metadata": {...}, + "workflow_name": "scouter", + "schedule_name": "hourly_collection", + "model_name": "temperature_sensors", + "model_id": "temp_001", + "data": {...}, + "trigger_laborious": false, + "filters": {...}, + "schema": "sensor_data", + "table_name": "temperature_readings", + "retention_time": 3600, + "fill_missing_tags": false, + "debug_data_package": false, + "model_tags": { + "Temperature": { + "data_range": [-50, 150], + "aggr_function": "avg", + "frequency": "60000" + } + } +} +``` + +**Additional Parameters:** +- `fill_missing_tags` (bool): Enable filling of missing tag values with default data +- `debug_data_package` (bool): Store raw and processed data packages in MongoDB for debugging + +#### Architecture + +```mermaid +flowchart LR + A[1. data_quality_gate] --> B[2. aggregate_data] --> C[3. group_and_hold_data] --> D[4. export_data_to_postgres] --> E[5. write_metrics] + E --> F{debug_data_package?} + F -->|yes| G[6. store_data_package] + + B -.-> Redis1[(Redis)] + C -.-> Redis2[(Redis)] + D -.-> PostgreSQL[(PostgreSQL)] + E -.-> Metrics[Prometheus] + G -.-> MongoDB[(MongoDB)] +``` + +#### Debug Mode + +When `debug_data_package` is set to `true`, the workflow stores both raw and processed data packages in MongoDB for debugging and troubleshooting purposes. This is useful for: +- Investigating data processing issues +- Validating data transformations +- Auditing data quality gate decisions + + +### 3. PI Web API Scouter Workflow (`pi_web_api_scouter.py`) + +The **PI Web API Scouter** workflow serves as the entry point for PI Web API data processing pipelines. Unlike the standard Scouter workflow that loads data from MongoDB collections, this workflow directly queries PI Web API endpoints to retrieve tag values and processes them for downstream use. + +#### Purpose +- **Direct API Ingestion**: Retrieves data directly from PI Web API endpoints +- **Real-time Data Processing**: Supports real-time and historical data retrieval +- **Data Normalization**: Normalizes timestamps to ensure consistency across records +- **Workflow Orchestration**: Delegates data processing to the CoreScouter workflow +- **Error Handling**: Comprehensive error handling with retry policies + +#### Execution Flow +1. **Tag Value Retrieval**: Retrieves tag values from PI Web API using configured WebIds and time periods +2. **Data Normalization**: Normalizes timestamps to ensure all records in a batch share the same timestamp value +3. **Data Validation**: Validates retrieved data and handles empty responses +4. **Data Processing**: Delegates data processing to the CoreScouter child workflow + +**Note**: The timestamp normalization process converts all timestamps to string format and then sets all records to the maximum timestamp value (lexicographically) found in the dataset. This ensures consistency across all records in a single batch. + +#### Key Features +- **Configurable Time Periods**: Supports flexible time period configurations (e.g., '*-1d', '*-1h') +- **Data Point Limits**: Configurable maximum data points per tag via `max_count` parameter +- **Timeout Management**: Configurable API request timeouts for reliable operation +- **Empty Data Handling**: Gracefully handles empty responses without processing +- **Standardized Processing**: Uses CoreScouter for consistent data quality and export operations + +#### Input Parameters +```json +{ + "model_name": "pi_sensors", + "model_id": "pi_001", + "schedule_name": "hourly_pi_collection", + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 10, + "api_timeout": 30 + }, + "model_tags": { + "Temperature": { + "webid": "F1AbCdEfGhIjKlMnOpQrStUvWxYz", + "aggr_function": "avg", + "data_range": [-50, 150] + }, + "Pressure": { + "webid": "F2AbCdEfGhIjKlMnOpQrStUvWxYz", + "aggr_function": "max", + "data_range": [0, 100] + } + }, + "trigger_laborious": false, + "filters": { + "OUT_OF_BOUNDS_FILTER": {"policy": "DISCARD"}, + "NULL_VALUES_FILTER": {"policy": "DISCARD"} + }, + "schema": "sensor_data", + "table_name": "pi_readings", + "retention_time": 3600, + "fill_missing_tags": false, + "debug_data_package": false +} +``` + +**PI Web API Query Parameters:** +- `endpoint` (str): PI Web API endpoint path (e.g., '/streamsets/recorded') +- `period` (str): Time period configuration (e.g., '*-1d' for last day, '*-1h' for last hour) +- `max_count` (int, optional): Maximum data points per tag. Defaults to 1 +- `api_timeout` (int): Request timeout in seconds for PI Web API calls + +**Model Tags Configuration:** +- `webid` (str): PI Web API WebId for the tag +- `aggr_function` (str): Aggregation method (avg, mdn, max, min, lts) +- `data_range` (list[int]): [min, max] values for data validation + +#### Architecture + +```mermaid +flowchart LR + A[1. get_tag_values] --> B{data empty?} + B -->|yes| C[Exit] + B -->|no| D[2. core_scouter πŸ”ƒ] + + A -.-> PI_API[(PI Web API)] + D -.-> CoreScouter[CoreScouter Workflow] +``` + +#### Data Normalization + +The `get_tag_values` activity normalizes timestamps to ensure consistency: +1. Converts all timestamps to string format using the configured datetime format +2. Identifies the maximum timestamp value (lexicographically) in the dataset +3. Sets all records to use this normalized timestamp value + +This normalization ensures that all records in a single batch share the same timestamp, which is useful for batch processing and data consistency in downstream operations. + +## πŸ“‹ Prerequisites + +- Python 3.11+ +- Temporal server/cluster +- PostgreSQL database +- Redis server +- MongoDB server +- Kafka cluster (for data ingestion) +- PI Web API server (for PI Web API Scouter workflow) + +**Note**: External dependencies must be available either through: +- Kubernetes cluster deployment +- Docker Compose setup +- Cloud-managed services +- Local installations + +## πŸš€ Installation + +### Local Development Setup + +1. **Clone the repository** + ```bash + git clone + cd sientia-dataops-scouter + ``` + +2. **Create virtual environment** + ```bash + python3.11 -m venv venv + source ./venv/bin/activate + ``` + +3. **Install dependencies** + ```bash + pip install -r requirements.txt + ``` + +4. **Create environment configuration file** + ```bash + cp .env.example .env + # Edit .env with your connection details + ``` + +5. **Configure external dependencies** + + You'll need to set up port forwarding or connections to external services. For example: + + ```bash + # Port forwarding from Kubernetes cluster + kubectl port-forward svc/redis-master 6379:6379 + kubectl port-forward svc/mongodb 27017:27017 + kubectl port-forward svc/kafka 9092:9092 + + # Or connect to external services + # Ensure services are accessible on localhost with appropriate ports + ``` + +## πŸ“¦ How to Run + +### Running the Scouter Application + +Use the provided script to run the application locally: + +```bash +# Make script executable (first time only) +chmod +x run_local.sh + +# Run the application +./run_local.sh +``` + +The script will: +- Activate the virtual environment +- Load environment variables from `.env` +- Start the scouter worker application + +### Running Tests and Coverage + +Use the provided script to run tests with coverage: + +```bash +# Make script executable (first time only) +chmod +x run_coverage.sh + +# Run tests with coverage +./run_coverage.sh +``` + +The script will: +- Activate the virtual environment +- Run pytest with coverage reporting +- Generate HTML coverage report +- Open the coverage report in your browser + +### Manual Test Execution + +You can also run tests manually: + +```bash +# Activate virtual environment +source ./venv/bin/activate + +# Run all tests +pytest + +# Run with coverage +pytest --cov=scouter --cov-report=html + +# Run specific test categories +pytest tests/activities/ +pytest tests/workflow/ +``` + +### Manual Application Execution + +For manual execution without scripts: + +```bash +# Activate virtual environment +source ./venv/bin/activate + +# Load environment variables (if using .env file) +if [ -f .env ]; then + export $(cat .env | grep -v '^#' | xargs) +fi + +# Start the scouter worker +python -m scouter.worker.worker +``` + +### Environment Configuration + +Before running the application, ensure your `.env` file contains the necessary configuration. + +## πŸ§ͺ Testing + +### Test Structure +``` +tests/ +β”œβ”€β”€ activities/ # Activity implementation tests +β”œβ”€β”€ workflow/ # Workflow orchestration tests +β”œβ”€β”€ utils/ # Utility function tests +└── integration/ # End-to-end workflow tests +``` + +### Test Execution +```bash +# Install test dependencies +pip install pytest pytest-cov pytest-asyncio + +# Run tests with coverage +pytest --cov=scouter --cov-report=html + +# Run specific test modules +pytest tests/activities/test_redis.py +pytest tests/workflow/test_scouter.py +``` + +## πŸ“Š Monitoring and Metrics + +The Scouter system exposes comprehensive Prometheus metrics: + +### Application Metrics +- `app_up`: Application health status (1=healthy, 0=unhealthy) +- `scouter_laborious_data_written_count`: Data export operation count +- `scouter_tag_changes_monitor`: Tag value change monitoring + +### Temporal Metrics +- Workflow execution counts and durations +- Activity execution success/failure rates +- Task queue processing metrics +- Worker health and performance indicators + +### Database Metrics +- Connection pool utilization +- Query execution times +- Error rates and retry counts + +## βš™οΈ Configuration + +### Environment Variables + +| Variable | Description | Default | Required | +|----------|-------------|---------|----------| +| `TEMPORAL_HOST` | Temporal server address | `localhost:7233` | Yes | +| `TEMPORAL_NAMESPACE` | Temporal namespace | `scouter` | No | +| `POSTGRES_HOST` | PostgreSQL hostname | `localhost` | Yes | +| `POSTGRES_PORT` | PostgreSQL port | `5432` | Yes | +| `POSTGRES_USER` | PostgreSQL username | `sientia` | Yes | +| `POSTGRES_PASSWORD` | PostgreSQL password | `sientia` | Yes | +| `POSTGRES_DBNAME` | PostgreSQL database | `sientia` | Yes | +| `REDIS_HOST` | Redis hostname | `localhost` | Yes | +| `REDIS_PORT` | Redis port | `6379` | Yes | +| `MONGODB_URL` | MongoDB connection URI | `localhost:27017` | Yes | +| `KAFKA_BOOTSTRAP_SERVERS` | Kafka broker addresses | `localhost:9092` | No | +| `PI_WEB_API_BASE_URL` | PI Web API base URL | - | Yes (for PI Web API Scouter) | +| `PI_WEB_API_AUTH_TYPE` | PI Web API authentication type ('basic' or 'bearer') | - | Yes (for PI Web API Scouter) | +| `PI_WEB_API_AUTH_TOKEN` | PI Web API authentication token | - | Yes (for PI Web API Scouter) | +| `HTTP_METRICS_PORT` | Prometheus metrics port | `9090` | No | +| `HTTP_SDK_METRICS_PORT` | Temporal SDK metrics port | `9091` | No | +| `PROJECT_NAME` | Project identifier for notifications | `scouter` | No | + +### Worker Configuration + +The worker supports advanced configuration for optimizing performance and latency: + +| Variable | Description | Default | Recommended | +|----------|-------------|---------|-------------| +| `MAX_CONCURRENT_WORKFLOW_TASKS` | Maximum concurrent workflow tasks | `200` | 100-500 | +| `MAX_CONCURRENT_ACTIVITIES` | Maximum concurrent activities | `200` | 100-500 | +| `MAX_CONCURRENT_LOCAL_ACTIVITIES` | Maximum concurrent local activities | `200` | 100-500 | +| `MAX_CACHED_WORKFLOWS` | Maximum cached workflow instances | `200` | 100-500 | + +### Poller Autoscaling Configuration + +The worker implements aggressive autoscaling policies for workflow and activity pollers: + +**Workflow Poller Behavior:** +| Variable | Description | Default | +|----------|-------------|---------| +| `WORKFLOW_POLLER_BEHAVIOUR_MINIMUM` | Minimum workflow pollers | `10` | +| `WORKFLOW_POLLER_BEHAVIOUR_INITIAL` | Initial workflow pollers | `100` | +| `WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM` | Maximum workflow pollers | `200` | + +**Activity Poller Behavior:** +| Variable | Description | Default | +|----------|-------------|---------| +| `ACTIVITY_POLLER_BEHAVIOUR_MINIMUM` | Minimum activity pollers | `10` | +| `ACTIVITY_POLLER_BEHAVIOUR_INITIAL` | Initial activity pollers | `100` | +| `ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM` | Maximum activity pollers | `200` | + +### Workflow Configuration + +MongoDB pipeline configuration: + +#### Scouter Workflow Input Parameters + +```json +{ + "schedule_name": "scouter-opcua-orchestrated-pipeline", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "5s", # Workflow execution frequency + "max_retry_policy": 1, # Maximum number of retries for the workflow + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", # Tag expected frequency (used in Ingestor) + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60, # Time that tag data is cached in Redis + "active": true, + "updated_at": { + "$date": "2025-08-13T18:35:01.600Z" + } +} +``` + +## πŸ”§ Development + +### Project Structure +``` +scouter/ +β”œβ”€β”€ activities/ # Temporal activity implementations +β”‚ β”œβ”€β”€ activities.py # Main activities orchestrator +β”‚ β”œβ”€β”€ api.py # PI Web API operations (tag value retrieval) +β”‚ β”œβ”€β”€ redis.py # Redis operations (caching, timestamps) +β”‚ β”œβ”€β”€ gates.py # Data quality gates and filtering +β”‚ └── mongodb.py # MongoDB operations (data loading) +β”œβ”€β”€ workflow/ # Temporal workflow definitions +β”‚ β”œβ”€β”€ scouter.py # Main data ingestion workflow +β”‚ β”œβ”€β”€ pi_web_api_scouter.py # PI Web API data ingestion workflow +β”‚ └── sub_workflows/ # Sub-workflow implementations +β”‚ └── core_scouter.py # Core data processing workflow +β”œβ”€β”€ worker/ # Worker implementation +β”‚ └── worker.py # Main worker orchestrator +β”œβ”€β”€ utils/ # Utility functions +β”‚ β”œβ”€β”€ connectors_config.py # Database configuration +β”‚ └── quality/ # Data quality filters +β”œβ”€β”€ metrics.py # Prometheus metrics definitions +└── __init__.py +``` + +### Activity Implementations + +The Activities class combines multiple service classes through multiple inheritance: + +- **Postgres** (from sientia-dataops-library): PostgreSQL data export and persistence +- **Redis**: Timestamp management, data caching, and temporary storage +- **Gates**: Data quality validation and filtering logic +- **MongoDB**: Data loading from raw collections +- **API**: PI Web API tag value retrieval and data normalization + +All activities support: +- Comprehensive logging and error handling +- Notification integration for errors and alerts +- Prometheus metrics collection +- Graceful shutdown and resource cleanup + +### Adding New Features + +1. **Follow Temporal patterns** for new workflows and activities +2. **Add comprehensive docstrings** for all public methods +3. **Include Prometheus metrics** for monitoring +4. **Add unit tests** for new functionality +5. **Update this README** with new features and configuration + +## πŸ› Troubleshooting + +### Common Issues + +1. **Temporal Connection Failures** + - Verify Temporal server is running and accessible + - Check namespace configuration and permissions + - Review server logs for connection issues + +2. **Database Connection Issues** + - Verify all database services are running + - Check connection credentials and network access + - Ensure proper connection pool configuration + +3. **Workflow Execution Failures** + - Review activity error logs and notifications + - Check data quality filter configurations + - Verify input data format and required fields + +4. **Performance Issues** + - Monitor Prometheus metrics for bottlenecks + - Review database query performance + - Check Temporal worker configuration + +### Debug Mode + +Enable debug logging by setting the log level: +```bash +export LOG_LEVEL=DEBUG +``` + +## ⚑ Performance Tuning + +### Key Parameters + +- **Worker Concurrency**: Adjust `MAX_CONCURRENT_WORKFLOW_TASKS` and `MAX_CONCURRENT_ACTIVITIES` (default: 200) +- **Poller Autoscaling**: Configure minimum, initial, and maximum poller counts for optimal throughput +- **Connection Pools**: Optimize database connection pool sizes (configured in `build_*_config()` functions) +- **Data Retention**: Configure Redis TTL via `retention_time` parameter (in seconds) +- **Workflow Caching**: Set `MAX_CACHED_WORKFLOWS` to balance memory usage and performance + +### Scaling Considerations + +- **Horizontal Scaling**: Deploy multiple worker instances (each registers to `scouter-queue`) +- **Poller Autoscaling**: Workers implement aggressive autoscaling (10-200 pollers) for latency optimization +- **Database Performance**: Connection pooling is configured in `utils/connectors_config.py` +- **Worker Placement**: Use pod anti-affinity rules in Kubernetes for optimal distribution +- **Resource Limits**: Configure appropriate CPU/memory limits based on concurrency settings + +## 🀝 Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes with comprehensive testing +4. Update documentation and docstrings +5. Submit a pull request + +### Code Quality Standards + +- Follow PEP 8 style guidelines +- Include comprehensive docstrings for all public methods +- Maintain test coverage above 80% +- Use type hints where appropriate +- Follow Temporal.io best practices + +## πŸ“„ License + +This project is licensed under the terms specified in the LICENSE file. + +## πŸ†˜ Support + +For support and questions: +- Check the troubleshooting section above +- Review the metrics and logs for error patterns +- Open an issue in the project repository +- Contact the development team + +--- + +**Note**: The Scouter system is designed for production use in industrial data processing environments. Ensure proper security configuration and network isolation for production deployments. diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000..05f8b33 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,37 @@ +# Scouter end-to-end tests + +Production-faithful E2E tests for `Scouter`, `PIWebAPIScouter`, and `CoreScouter` against real backing services. + +## Requirements + +- Docker (for testcontainers: MongoDB, Redis, PostgreSQL) +- Python 3.11+ with dev dependencies: `pip install -r requirements-dev.txt` + +## Run locally + +```bash +pytest e2e/ --override-ini testpaths=e2e -m e2e -v +``` + +Stop on first failure: + +```bash +pytest e2e/ --override-ini testpaths=e2e -m e2e -x +``` + +## Coverage (separate from unit tests) + +```bash +COVERAGE_FILE=.coverage.e2e pytest e2e/ --override-ini testpaths=e2e -m e2e --cov=scouter --cov-branch +coverage combine .coverage .coverage.e2e && coverage report +``` + +## Scenario catalog + +See [scenarios.md](scenarios.md) for numbered scenarios and how they map to `test_scenario_*` functions. Section `## 0` of that file lists the harness smoke tests in `test_harness_smoke.py` (infra liveness checks, not business scenarios). + +## Production code is not mocked + +E2E uses real testcontainers, an in-process PI Web API HTTP server, `WorkflowEnvironment.start_local()`, and production `Activities` wiring. The only stand-ins are `mock_logger` and the optional `notification_inserts` spy. If a scenario fails due to a production defect, it is marked `xfail(strict=True)` and tracked in `openspec/changes/standardize-and-complete-e2e-tests/notes.md` when applicable. + +Unit tests under `tests/` remain Docker-free and run with the default `pytest` invocation. diff --git a/e2e/__init__.py b/e2e/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/e2e/conftest.py b/e2e/conftest.py new file mode 100644 index 0000000..e04d215 --- /dev/null +++ b/e2e/conftest.py @@ -0,0 +1,324 @@ +""" +Pytest configuration and fixtures for production-faithful Scouter E2E tests. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import pytest_asyncio +from pymongo import MongoClient +from redis import Redis +from sqlalchemy import create_engine, text +from testcontainers.mongodb import MongoDbContainer +from testcontainers.postgres import PostgresContainer +from testcontainers.redis import RedisContainer +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import PollerBehaviorAutoscaling, Worker + +from e2e.helpers import SCOUTER_TASK_QUEUE, postgres_connection_parts +from e2e.pi_web_api_test_server import PIWebAPITestServer +from scouter.activities.activities import Activities +from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter +from scouter.workflow.scouter import Scouter +from scouter.workflow.sub_workflows.core_scouter import CoreScouter +from sientia_do.notifications.handlers import CoreNotificationHandler +from sientia_do.observability.logger import Logger + +E2E_DATABASE = 'scouter_e2e_test' +E2E_NOTIFICATION_COLLECTION = 'notification_queue' +DB_SCHEMA_PATH = Path(__file__).resolve().parent / 'db_schema.sql' + + +def _activity_list(activities: Activities) -> list: + """Return bound activity callables for the E2E worker.""" + return [ + activities.load_latest_data, + activities.get_last_data_timestamp, + activities.put_last_data_timestamp, + activities.get_tag_values, + activities.data_quality_gate, + activities.aggregate_data, + activities.group_and_hold_data, + activities.export_data_to_postgres, + activities.write_metrics, + activities.store_data_package, + ] + + +@pytest.fixture(scope='session') +def postgres_container(): + """ + Session-scoped PostgreSQL testcontainer. + + Return: + Running PostgresContainer instance + """ + container = PostgresContainer('postgres:15') + container.start() + yield container + container.stop() + + +@pytest.fixture(scope='session') +def mongo_container(): + """ + Session-scoped MongoDB testcontainer. + + Return: + Running MongoDbContainer instance + """ + container = MongoDbContainer('mongo:7') + container.start() + yield container + container.stop() + + +@pytest.fixture(scope='session') +def redis_container(): + """ + Session-scoped Redis testcontainer. + + Return: + Running RedisContainer instance + """ + container = RedisContainer('redis:7') + container.start() + yield container + container.stop() + + +@pytest.fixture(scope='session') +def postgres_engine(postgres_container): + """ + SQLAlchemy engine bound to the Postgres testcontainer for the session. + + Return: + SQLAlchemy Engine + """ + engine = create_engine(postgres_container.get_connection_url()) + yield engine + engine.dispose() + + +@pytest.fixture(scope='session') +def mongo_uri(mongo_container): + """ + MongoDB connection string for the testcontainer. + + Return: + Connection URI string + """ + return mongo_container.get_connection_url() + + +@pytest.fixture(scope='session') +def redis_client(redis_container): + """ + Redis client connected to the testcontainer. + + Return: + redis.Redis client with decode_responses=True + """ + host = redis_container.get_container_host_ip() + port = int(redis_container.get_exposed_port(6379)) + client = Redis(host=host, port=port, decode_responses=True) + yield client + client.close() + + +@pytest.fixture(autouse=True) +def setup_postgres_schema_and_table(postgres_engine): + """ + Apply db_schema.sql before each test so laborious_data is empty and current. + """ + sql = DB_SCHEMA_PATH.read_text(encoding='utf-8') + with postgres_engine.begin() as conn: + conn.exec_driver_sql(sql) + yield + + +@pytest.fixture(autouse=True) +def reset_mongo_collections(mongo_uri): + """ + Drop all collections in the E2E Mongo database between tests. + """ + client = MongoClient(mongo_uri) + try: + db = client[E2E_DATABASE] + for name in db.list_collection_names(): + db.drop_collection(name) + finally: + client.close() + yield + + +@pytest.fixture(autouse=True) +def reset_redis(redis_client): + """ + Flush the Redis testcontainer between tests. + """ + redis_client.flushdb() + yield + + +@pytest.fixture +def mock_logger(): + """ + Logger stand-in (only permitted MagicMock in the E2E harness). + + Return: + MagicMock with Logger spec + """ + logger = MagicMock(spec=Logger) + logger.info = MagicMock() + logger.debug = MagicMock() + logger.error = MagicMock() + logger.warning = MagicMock() + logger.custom_info = MagicMock() + return logger + + +@pytest.fixture +def notification_handler(mock_logger, mongo_uri): + """ + Real CoreNotificationHandler backed by the Mongo testcontainer. + + Return: + CoreNotificationHandler instance + """ + handler = CoreNotificationHandler( + connection_string=mongo_uri, + database=E2E_DATABASE, + logger=mock_logger, + project_name='scouter-e2e', + notification_topic=E2E_NOTIFICATION_COLLECTION, + ) + yield handler + handler.shutdown() + + +@pytest.fixture +def notification_inserts(notification_handler): + """ + Spy wrapper around notification collection insert_one (still writes to Mongo). + + Return: + MagicMock wrapping insert_one + """ + collection = notification_handler.mongo_collection + spy = MagicMock(wraps=collection.insert_one) + collection.insert_one = spy + return spy + + +@pytest.fixture(scope='session') +def pi_web_api_server(): + """ + Session-scoped in-process PI Web API HTTP stub. + + Return: + Started PIWebAPITestServer instance + """ + server = PIWebAPITestServer() + server.start() + yield server + server.stop() + + +@pytest.fixture(autouse=True) +def cleanup_pi_web_api_server(pi_web_api_server): + """ + Reset PI Web API stub state between tests. + """ + pi_web_api_server.clear() + yield + + +@pytest_asyncio.fixture(scope='session') +async def temporal_env(): + """ + Session-scoped WorkflowEnvironment using the real local Temporal dev server. + + Return: + WorkflowEnvironment from start_local() + """ + async with await WorkflowEnvironment.start_local() as env: + yield env + + +@pytest.fixture +def test_activities( + postgres_container, + mongo_uri, + redis_container, + pi_web_api_server, + mock_logger, + notification_handler, +): + """ + Production Activities wired to testcontainers and the PI Web API stub. + + Return: + Live Activities instance (no unittest.mock.patch) + """ + pg_parts = postgres_connection_parts( + postgres_container.get_connection_url()) + redis_host = redis_container.get_container_host_ip() + redis_port = int(redis_container.get_exposed_port(6379)) + + activities = Activities( + postgres_config={ + **pg_parts, + 'min_connections': 1, + 'max_connections': 5, + }, + redis_config={ + 'host': redis_host, + 'port': redis_port, + 'username': '', + 'password': '', + }, + mongodb_config={ + 'connection_string': mongo_uri, + 'database_name': E2E_DATABASE, + }, + api_config={ + 'base_url': pi_web_api_server.base_url, + 'auth_type': 'bearer', + 'auth_token': 'e2e-test-token', + }, + logger=mock_logger, + notification_handler=notification_handler, + ) + yield activities + activities.shutdown() + + +@pytest_asyncio.fixture +async def temporal_worker(temporal_env, test_activities): + """ + Temporal worker registering all Scouter workflows and activities on the E2E queue. + + Return: + Running temporalio.worker.Worker + """ + async with Worker( + temporal_env.client, + task_queue=SCOUTER_TASK_QUEUE, + workflows=[Scouter, PIWebAPIScouter, CoreScouter], + activities=_activity_list(test_activities), + activity_executor=ThreadPoolExecutor( + max_workers=50, thread_name_prefix='e2e-activity'), + max_concurrent_workflow_tasks=50, + max_concurrent_activities=50, + max_concurrent_local_activities=50, + workflow_task_poller_behavior=PollerBehaviorAutoscaling( + minimum=1, initial=2, maximum=10), + activity_task_poller_behavior=PollerBehaviorAutoscaling( + minimum=1, initial=2, maximum=10), + ) as worker: + yield worker diff --git a/e2e/db_schema.sql b/e2e/db_schema.sql new file mode 100644 index 0000000..3e4111b --- /dev/null +++ b/e2e/db_schema.sql @@ -0,0 +1,16 @@ +-- Single source of truth for E2E Postgres DDL (non-partitioned mirror of production sientia_data.laborious_data). + +DROP TABLE IF EXISTS sientia_data.laborious_data; +DROP SCHEMA IF EXISTS sientia_data CASCADE; + +CREATE SCHEMA sientia_data; + +CREATE TABLE sientia_data.laborious_data ( + id SERIAL NOT NULL, + model_id int4 NOT NULL, + variable text NOT NULL, + value numeric NULL, + "timestamp" timestamptz NOT NULL, + created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL, + PRIMARY KEY (id, created_at) +); diff --git a/e2e/fixtures/__init__.py b/e2e/fixtures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/e2e/helpers.py b/e2e/helpers.py new file mode 100644 index 0000000..97b8fbb --- /dev/null +++ b/e2e/helpers.py @@ -0,0 +1,404 @@ +""" +Shared helpers for Scouter end-to-end tests. +""" + +from __future__ import annotations + +import json +import re +import uuid +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from pymongo import MongoClient +from redis import Redis +from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Engine +from temporalio.client import Client + +SCENARIO_INPUTS_DIR = Path(__file__).resolve().parent / 'scenario_inputs' +SCOUTER_TASK_QUEUE = 'scouter-test-queue' + +_NOW_MARKER = re.compile(r'^@now(?:([+-])(\d+)([smhd]))?$') + + +def _resolve_timestamp_marker(value: str) -> datetime: + """ + Resolve @now relative timestamp markers to timezone-aware datetimes. + + Args: + - value: Marker string such as @now, @now-1h, or @now+30m + + Return: + Resolved datetime in UTC + """ + match = _NOW_MARKER.match(value.strip()) + if not match: + raise ValueError(f'Invalid timestamp marker: {value}') + + now = datetime.now(UTC) + if match.group(1) is None: + return now + + sign, amount, unit = match.group(1), int(match.group(2)), match.group(3) + delta_kwargs = {'seconds': 0, 'minutes': 0, 'hours': 0, 'days': 0} + if unit == 's': + delta_kwargs['seconds'] = amount + elif unit == 'm': + delta_kwargs['minutes'] = amount + elif unit == 'h': + delta_kwargs['hours'] = amount + elif unit == 'd': + delta_kwargs['days'] = amount + + delta = timedelta(**delta_kwargs) + return now - delta if sign == '-' else now + delta + + +def _resolve_payload(node: Any) -> Any: + """ + Recursively resolve @now markers inside JSON-loaded structures. + + Args: + - node: JSON node (dict, list, or scalar) + + Return: + Structure with markers replaced by datetimes or formatted strings + """ + if isinstance(node, dict): + return {key: _resolve_payload(value) for key, value in node.items()} + if isinstance(node, list): + return [_resolve_payload(item) for item in node] + if isinstance(node, str) and node.startswith('@now'): + resolved = _resolve_timestamp_marker(node) + return resolved.strftime('%Y-%m-%d %H:%M:%S.%f%z') + return node + + +def load_scenario_input(name: str, **overrides: Any) -> dict[str, Any]: + """ + Load a scenario JSON file and apply optional overrides. + + Args: + - name: Scenario slug with or without .json suffix + - overrides: Top-level keys merged into the loaded document + + Return: + Parsed scenario document with @now markers resolved + """ + slug = name.removesuffix('.json') + path = SCENARIO_INPUTS_DIR / f'{slug}.json' + with path.open(encoding='utf-8') as handle: + payload = json.load(handle) + resolved = _resolve_payload(payload) + if overrides: + resolved.update(overrides) + return resolved + + +def make_workflow_id(prefix: str) -> str: + """ + Build a unique Temporal workflow id for E2E runs. + + Args: + - prefix: Human-readable prefix for the workflow id + + Return: + Unique workflow id string + """ + return f'{prefix}-{uuid.uuid4().hex[:12]}' + + +async def start_and_await_workflow( + client: Client, + workflow_run: Any, + input_data: dict[str, Any], + workflow_id: str, + *, + task_queue: str = SCOUTER_TASK_QUEUE, + timeout: float = 300.0, +) -> Any: + """ + Start a workflow on the E2E task queue and await its result. + + Args: + - client: Temporal client from WorkflowEnvironment + - workflow_run: Workflow run method (e.g. Scouter.run) + - input_data: Workflow input payload + - workflow_id: Unique workflow id + - task_queue: Task queue name + - timeout: Maximum seconds to wait for completion + + Return: + Workflow result (None for Scouter-family workflows) + """ + return await client.execute_workflow( + workflow_run, + input_data, + id=workflow_id, + task_queue=task_queue, + execution_timeout=timedelta(seconds=timeout), + ) + + +def _coerce_mongo_document(document: dict[str, Any]) -> dict[str, Any]: + """ + Convert string timestamps in seed documents to BSON datetimes for Mongo filters. + + Args: + - document: Raw document dict from scenario JSON + + Return: + Document with inserted_at as datetime when present + """ + doc = dict(document) + inserted_at = doc.get('inserted_at') + if isinstance(inserted_at, str): + doc['inserted_at'] = datetime.strptime(inserted_at, DATETIME_FORMAT_MS_WITH_TZ).replace( + tzinfo=UTC + ) + return doc + + +def seed_raw_collection( + mongo_uri: str, + database: str, + schedule_name: str, + documents: list[dict[str, Any]], +) -> None: + """ + Insert raw Mongo documents into raw_. + + Args: + - mongo_uri: MongoDB connection string + - database: Database name + - schedule_name: Schedule slug used in collection name + - documents: Documents to insert + """ + collection_name = f'raw_{schedule_name}' + client = MongoClient(mongo_uri) + try: + collection = client[database][collection_name] + if documents: + collection.insert_many([_coerce_mongo_document(doc) for doc in documents]) + finally: + client.close() + + +def seed_last_data_timestamp( + redis_client: Redis, + workflow_name: str, + schedule_name: str, + value: str, +) -> None: + """ + Pre-seed last_data_timestamp Redis key the same way RedisRepository.set stores strings. + + Args: + - redis_client: Connected Redis client + - workflow_name: Workflow name segment in the key + - schedule_name: Schedule name segment in the key + - value: Timestamp string to store + """ + key = f'last_data_timestamp:{workflow_name}:{schedule_name}' + redis_client.set(key, json.dumps(value)) + + +def count_laborious_rows(engine: Engine, model_id: str | int | None = None) -> int: + """ + Count rows in sientia_data.laborious_data, optionally filtered by model_id. + + Args: + - engine: SQLAlchemy engine bound to the Postgres testcontainer + - model_id: Optional model id filter + + Return: + Row count + """ + query = 'SELECT COUNT(*) FROM sientia_data.laborious_data' + params: dict[str, Any] = {} + if model_id is not None: + query += ' WHERE model_id = :model_id' + params['model_id'] = int(model_id) + + with engine.connect() as conn: + return conn.execute(text(query), params).scalar() or 0 + + +def fetch_laborious_rows(engine: Engine, model_id: str | int | None = None) -> list[dict[str, Any]]: + """ + Fetch laborious_data rows as plain dicts. + + Args: + - engine: SQLAlchemy engine bound to the Postgres testcontainer + - model_id: Optional model id filter + + Return: + List of row dicts with variable and value keys + """ + query = 'SELECT model_id, variable, value, timestamp FROM sientia_data.laborious_data' + params: dict[str, Any] = {} + if model_id is not None: + query += ' WHERE model_id = :model_id' + params['model_id'] = int(model_id) + query += ' ORDER BY variable' + + with engine.connect() as conn: + rows = conn.execute(text(query), params).mappings().all() + return [dict(row) for row in rows] + + +def count_held_data_keys(redis_client: Redis) -> int: + """ + Count Redis keys matching held_data_*. + + Args: + - redis_client: Connected Redis client + + Return: + Number of matching keys + """ + return len(redis_client.keys('held_data_*')) + + +def count_notifications( + mongo_uri: str, + database: str, + *, + notification_id: str | None = None, + level: str | None = None, +) -> int: + """ + Count notification documents in the E2E notification collection. + + Args: + - mongo_uri: MongoDB connection string + - database: Database name + - notification_id: Optional notification_id filter + - level: Optional level filter (WARNING, ERROR, ...) + + Return: + Matching document count + """ + client = MongoClient(mongo_uri) + try: + collection = client[database]['notification_queue'] + query: dict[str, Any] = {} + if notification_id: + query['notification_id'] = notification_id + if level: + query['level'] = level + return collection.count_documents(query) + finally: + client.close() + + +def default_model_tags( + *, + names: list[str], + aggr: str = 'avg', + data_range: list[float] | None = None, + frequency: int = 60000, +) -> dict[str, dict[str, Any]]: + """ + Build a minimal model_tags map for E2E scenarios. + + Args: + - names: Tag names to include + - aggr: Aggregation function (aggr_func field) + - data_range: Optional [min, max] validation range + - frequency: Collection frequency in milliseconds + + Return: + model_tags dict keyed by tag name + """ + if data_range is None: + data_range = [0, 100] + return { + name: { + 'webid': f'webid_{name}', + 'aggr_func': aggr, + 'data_range': data_range, + 'frequency': frequency, + } + for name in names + } + + +def default_scouter_input( + *, + model_id: str = '1', + model_name: str = 'Test Model', + schedule_name: str = 'test-schedule', + workflow_name: str = 'scouter', + **overrides: Any, +) -> dict[str, Any]: + """ + Return a base workflow input dict for Scouter / CoreScouter E2E runs. + + Args: + - model_id: Model identifier + - model_name: Human-readable model name + - schedule_name: Schedule slug + - workflow_name: Parent workflow name + - overrides: Additional keys merged into the payload + + Return: + Workflow input dictionary + """ + payload: dict[str, Any] = { + 'topic': 'e2e-topic', + 'model_id': model_id, + 'model_name': model_name, + 'schedule_name': schedule_name, + 'workflow_name': workflow_name, + 'trigger_laborious': False, + 'filters': {}, + 'schema': 'sientia_data', + 'table_name': 'laborious_data', + 'retention_time': 3600, + 'fill_missing_tags': False, + 'model_tags': default_model_tags(names=['tag1']), + } + payload.update(overrides) + return payload + + +def apply_pi_web_api_server_config(server: Any, config: dict[str, Any] | None) -> None: + """ + Configure the in-process PI Web API stub from a scenario pi_web_api_server block. + + Args: + - server: PIWebAPITestServer instance + - config: Optional mode/rows/timeout_sleep_seconds dict from scenario JSON + """ + if not config: + return + server.set_mode( + config.get('mode', 'success'), + rows=config.get('rows'), + timeout_sleep_seconds=config.get('timeout_sleep_seconds', 60), + ) + + +def postgres_connection_parts(connection_url: str) -> dict[str, Any]: + """ + Parse a SQLAlchemy Postgres URL into Activities postgres_config fields. + + Args: + - connection_url: SQLAlchemy connection URL from testcontainers + + Return: + Dict with host, port, user, password, dbname keys + """ + parsed = urlparse(connection_url) + return { + 'host': parsed.hostname or 'localhost', + 'port': parsed.port or 5432, + 'user': parsed.username or 'test', + 'password': parsed.password or 'test', + 'dbname': (parsed.path or '/test').lstrip('/'), + } diff --git a/e2e/pi_web_api_test_server.py b/e2e/pi_web_api_test_server.py new file mode 100644 index 0000000..912c214 --- /dev/null +++ b/e2e/pi_web_api_test_server.py @@ -0,0 +1,124 @@ +""" +In-process HTTP server emulating PI Web API streamsets/recorded responses for E2E tests. +""" + +from __future__ import annotations + +import json +import time +from typing import Any, Literal + +from pytest_httpserver import HTTPServer +from werkzeug import Request +from werkzeug.wrappers import Response + +PIWebAPIMode = Literal['success', 'empty', 'error', 'timeout'] +STREAMSETS_RECORDED_PATH = '/streamsets/recorded' + + +class PIWebAPITestServer: + """ + Thread-backed PI Web API stub using pytest-httpserver (real HTTP for pycurl clients). + """ + + def __init__(self) -> None: + self._httpserver = HTTPServer(host='127.0.0.1', port=0) + self._mode: PIWebAPIMode = 'success' + self._rows: list[dict[str, Any]] = [] + self._timeout_sleep_seconds = 60 + self.requests: list[Request] = [] + + @property + def host(self) -> str: + return self._httpserver.host + + @property + def port(self) -> int: + return self._httpserver.port + + @property + def base_url(self) -> str: + return f'http://{self.host}:{self.port}' + + def start(self) -> None: + """Start the HTTP server and register the streamsets handler.""" + self._httpserver.start() + self._register_handler() + + def stop(self) -> None: + """Stop the HTTP server.""" + self._httpserver.stop() + + def clear(self) -> None: + """Clear recorded requests and reset mode to success with no rows.""" + self.requests.clear() + self._mode = 'success' + self._rows = [] + self._httpserver.clear() + self._register_handler() + + def set_mode( + self, + mode: PIWebAPIMode, + rows: list[dict[str, Any]] | None = None, + *, + timeout_sleep_seconds: int = 60, + ) -> None: + """ + Configure the next responses from the stub server. + + Args: + - mode: Response mode (success, empty, error, timeout) + - rows: Optional list of row dicts with keys name, webid, timestamp, value + - timeout_sleep_seconds: Sleep duration for timeout mode (must exceed client timeout) + """ + self._mode = mode + if rows is not None: + self._rows = rows + self._timeout_sleep_seconds = timeout_sleep_seconds + self._register_handler() + + def _register_handler(self) -> None: + self._httpserver.expect_request( + STREAMSETS_RECORDED_PATH, + method='GET', + ).respond_with_handler(self._handle_streamsets_recorded) + + def _handle_streamsets_recorded(self, request: Request): + self.requests.append(request) + + if self._mode == 'timeout': + time.sleep(self._timeout_sleep_seconds) + return self._json_response({'Items': []}, status=200) + + if self._mode == 'error': + return self._json_response({'error': 'internal'}, status=500) + + if self._mode == 'empty' or not self._rows: + return self._json_response({'Items': []}, status=200) + + items_by_name: dict[str, list[dict[str, Any]]] = {} + for row in self._rows: + name = row['name'] + items_by_name.setdefault(name, []).append( + { + 'Timestamp': row['timestamp'], + 'Value': row['value'], + 'Good': True, + 'Questionable': False, + } + ) + + items = [ + {'Name': name, 'Items': points} + for name, points in items_by_name.items() + ] + return self._json_response({'Items': items}, status=200) + + @staticmethod + def _json_response(payload: dict[str, Any], *, status: int) -> Response: + return Response( + json.dumps(payload), + status=status, + mimetype='application/json', + ) diff --git a/e2e/scenario_inputs/core_scouter_aggregation_avg.json b/e2e/scenario_inputs/core_scouter_aggregation_avg.json new file mode 100644 index 0000000..2863249 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_aggregation_avg.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "24", "model_name": "Core Avg", "schedule_name": "core-avg", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-avg", + "model_name": "Core Avg", + "model_id": "24", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_avg", "value": 10.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:01:00+0000", "name": "tag_avg", "value": 20.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:02:00+0000", "name": "tag_avg", "value": 30.0, "tag": "w1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_avg": {"webid": "w1", "aggr_func": "avg", "data_range": [0, 1000], "frequency": 60000} + } + }, + "expected_value": 20.0 +} diff --git a/e2e/scenario_inputs/core_scouter_aggregation_lts.json b/e2e/scenario_inputs/core_scouter_aggregation_lts.json new file mode 100644 index 0000000..95612c5 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_aggregation_lts.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "28", "model_name": "Core Lts", "schedule_name": "core-lts", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-lts", + "model_name": "Core Lts", + "model_id": "28", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_lts", "value": 100.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:01:00+0000", "name": "tag_lts", "value": 200.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:02:00+0000", "name": "tag_lts", "value": 300.0, "tag": "w1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_lts": {"webid": "w1", "aggr_func": "lts", "data_range": [0, 1000], "frequency": 60000} + } + }, + "expected_value": 300.0 +} diff --git a/e2e/scenario_inputs/core_scouter_aggregation_max.json b/e2e/scenario_inputs/core_scouter_aggregation_max.json new file mode 100644 index 0000000..e8ce960 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_aggregation_max.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "26", "model_name": "Core Max", "schedule_name": "core-max", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-max", + "model_name": "Core Max", + "model_id": "26", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_max", "value": 5.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:01:00+0000", "name": "tag_max", "value": 15.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:02:00+0000", "name": "tag_max", "value": 10.0, "tag": "w1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_max": {"webid": "w1", "aggr_func": "max", "data_range": [0, 1000], "frequency": 60000} + } + }, + "expected_value": 15.0 +} diff --git a/e2e/scenario_inputs/core_scouter_aggregation_mdn.json b/e2e/scenario_inputs/core_scouter_aggregation_mdn.json new file mode 100644 index 0000000..93e4fda --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_aggregation_mdn.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "25", "model_name": "Core Mdn", "schedule_name": "core-mdn", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-mdn", + "model_name": "Core Mdn", + "model_id": "25", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_mdn", "value": 1.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:01:00+0000", "name": "tag_mdn", "value": 9.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:02:00+0000", "name": "tag_mdn", "value": 5.0, "tag": "w1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_mdn": {"webid": "w1", "aggr_func": "mdn", "data_range": [0, 1000], "frequency": 60000} + } + }, + "expected_value": 5.0 +} diff --git a/e2e/scenario_inputs/core_scouter_aggregation_min.json b/e2e/scenario_inputs/core_scouter_aggregation_min.json new file mode 100644 index 0000000..3cfc8f4 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_aggregation_min.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "27", "model_name": "Core Min", "schedule_name": "core-min", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-min", + "model_name": "Core Min", + "model_id": "27", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_min", "value": 50.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:01:00+0000", "name": "tag_min", "value": 30.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:02:00+0000", "name": "tag_min", "value": 40.0, "tag": "w1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_min": {"webid": "w1", "aggr_func": "min", "data_range": [0, 1000], "frequency": 60000} + } + }, + "expected_value": 30.0 +} diff --git a/e2e/scenario_inputs/core_scouter_debug_data_package.json b/e2e/scenario_inputs/core_scouter_debug_data_package.json new file mode 100644 index 0000000..8f0eb92 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_debug_data_package.json @@ -0,0 +1,22 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "30", "model_name": "Core Debug", "schedule_name": "core-debug", "workflow_name": "subworkflow.core_scouter"}}, + "workflow_name": "subworkflow.core_scouter", + "schedule_name": "core-debug", + "model_name": "Core Debug", + "model_id": "30", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": 11.0, "tag": "webid1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "debug_data_package": true, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_empty_after_grouping.json b/e2e/scenario_inputs/core_scouter_empty_after_grouping.json new file mode 100644 index 0000000..236c269 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_empty_after_grouping.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "31", "model_name": "Core Empty Group", "schedule_name": "core-empty-group", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-empty-group", + "model_name": "Core Empty Group", + "model_id": "31", + "data": { + "timestamp": [], + "name": [], + "value": [], + "tag": [] + }, + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_fill_missing_tags.json b/e2e/scenario_inputs/core_scouter_fill_missing_tags.json new file mode 100644 index 0000000..677f5bd --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_fill_missing_tags.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "29", "model_name": "Core Fill Tags", "schedule_name": "core-fill", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-fill", + "model_name": "Core Fill Tags", + "model_id": "29", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": 10.0, "tag": "webid1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": true, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag2": {"webid": "webid2", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag3": {"webid": "webid3", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + }, + "expected_missing_tags": ["tag2", "tag3"] +} diff --git a/e2e/scenario_inputs/core_scouter_happy_path.json b/e2e/scenario_inputs/core_scouter_happy_path.json new file mode 100644 index 0000000..ad1e889 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_happy_path.json @@ -0,0 +1,28 @@ +{ + "workflow_input": { + "metadata": { + "metadata": { + "model_id": "20", + "model_name": "Core Happy", + "schedule_name": "core-happy", + "workflow_name": "pi_web_api_scouter" + } + }, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-happy", + "model_name": "Core Happy", + "model_id": "20", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": 10.5, "tag": "webid1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_invalid_aggregation.json b/e2e/scenario_inputs/core_scouter_invalid_aggregation.json new file mode 100644 index 0000000..b0c8a4a --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_invalid_aggregation.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "32", "model_name": "Core Bad Aggr", "schedule_name": "core-bad-aggr", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-bad-aggr", + "model_name": "Core Bad Aggr", + "model_id": "32", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_ok", "value": 10.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_bad", "value": 20.0, "tag": "w2"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_ok": {"webid": "w1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag_bad": {"webid": "w2", "aggr_func": "bogus", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_null_values_filter_discard.json b/e2e/scenario_inputs/core_scouter_null_values_filter_discard.json new file mode 100644 index 0000000..cfb74df --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_null_values_filter_discard.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "21", "model_name": "Core Null Discard", "schedule_name": "core-null-discard", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-null-discard", + "model_name": "Core Null Discard", + "model_id": "21", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": null, "tag": "webid1"}, + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag2", "value": 50.0, "tag": "webid2"} + ], + "trigger_laborious": false, + "filters": {"NULL_VALUES_FILTER": {"policy": "DISCARD"}}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag2": {"webid": "webid2", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_null_values_filter_warn.json b/e2e/scenario_inputs/core_scouter_null_values_filter_warn.json new file mode 100644 index 0000000..96cc560 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_null_values_filter_warn.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "22", "model_name": "Core Null Warn", "schedule_name": "core-null-warn", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-null-warn", + "model_name": "Core Null Warn", + "model_id": "22", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": null, "tag": "webid1"}, + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag2", "value": 50.0, "tag": "webid2"} + ], + "trigger_laborious": false, + "filters": {"NULL_VALUES_FILTER": {"policy": "WARN"}}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag2": {"webid": "webid2", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_out_of_bounds_filter_discard.json b/e2e/scenario_inputs/core_scouter_out_of_bounds_filter_discard.json new file mode 100644 index 0000000..a7225ad --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_out_of_bounds_filter_discard.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "23", "model_name": "Core OOB Discard", "schedule_name": "core-oob-discard", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-oob-discard", + "model_name": "Core OOB Discard", + "model_id": "23", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": 150.0, "tag": "webid1"}, + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag2", "value": 50.0, "tag": "webid2"} + ], + "trigger_laborious": false, + "filters": {"OUT_OF_BOUNDS_FILTER": {"policy": "DISCARD"}}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag2": {"webid": "webid2", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_postgres_export_failure.json b/e2e/scenario_inputs/core_scouter_postgres_export_failure.json new file mode 100644 index 0000000..982b976 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_postgres_export_failure.json @@ -0,0 +1,21 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "33", "model_name": "Core PG Fail", "schedule_name": "core-pg-fail", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-pg-fail", + "model_name": "Core PG Fail", + "model_id": "33", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": 10.0, "tag": "webid1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_connection_error.json b/e2e/scenario_inputs/pi_web_api_scouter_connection_error.json new file mode 100644 index 0000000..29244c3 --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_connection_error.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "model_id": "14", + "model_name": "PI Error", + "schedule_name": "pi-error", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 1, + "api_timeout": 30 + } + }, + "pi_web_api_server": {"mode": "error"} +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_debug_data_package.json b/e2e/scenario_inputs/pi_web_api_scouter_debug_data_package.json new file mode 100644 index 0000000..d49ced1 --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_debug_data_package.json @@ -0,0 +1,29 @@ +{ + "workflow_input": { + "model_id": "12", + "model_name": "PI Debug Package", + "schedule_name": "pi-debug", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "debug_data_package": true, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 5, + "api_timeout": 30 + } + }, + "pi_web_api_server": { + "mode": "success", + "rows": [ + {"name": "tag1", "timestamp": "2024-06-01T12:00:00Z", "value": 7.5} + ] + } +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_empty_response.json b/e2e/scenario_inputs/pi_web_api_scouter_empty_response.json new file mode 100644 index 0000000..83591f2 --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_empty_response.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "model_id": "13", + "model_name": "PI Empty", + "schedule_name": "pi-empty", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 1, + "api_timeout": 30 + } + }, + "pi_web_api_server": {"mode": "empty"} +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_happy_path.json b/e2e/scenario_inputs/pi_web_api_scouter_happy_path.json new file mode 100644 index 0000000..7c4ae70 --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_happy_path.json @@ -0,0 +1,30 @@ +{ + "workflow_input": { + "model_id": "10", + "model_name": "PI Web API Happy", + "schedule_name": "pi-happy", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag2": {"webid": "webid2", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 10, + "api_timeout": 30 + } + }, + "pi_web_api_server": { + "mode": "success", + "rows": [ + {"name": "tag1", "timestamp": "2024-06-01T12:00:00Z", "value": 10.5}, + {"name": "tag2", "timestamp": "2024-06-01T12:00:00Z", "value": 20.3} + ] + } +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_invalid_endpoint.json b/e2e/scenario_inputs/pi_web_api_scouter_invalid_endpoint.json new file mode 100644 index 0000000..cc40d1b --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_invalid_endpoint.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "model_id": "16", + "model_name": "PI Invalid Endpoint", + "schedule_name": "pi-invalid", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/invalid/endpoint", + "period": "*-1d", + "max_count": 1, + "api_timeout": 30 + } + }, + "pi_web_api_server": {"mode": "success", "rows": []} +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_multiple_tags.json b/e2e/scenario_inputs/pi_web_api_scouter_multiple_tags.json new file mode 100644 index 0000000..c2c06a1 --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_multiple_tags.json @@ -0,0 +1,53 @@ +{ + "workflow_input": { + "model_id": "11", + "model_name": "PI Multi Tag", + "schedule_name": "pi-multi", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_avg": {"webid": "w1", "aggr_func": "avg", "data_range": [0, 1000], "frequency": 60000}, + "tag_mdn": {"webid": "w2", "aggr_func": "mdn", "data_range": [0, 1000], "frequency": 60000}, + "tag_max": {"webid": "w3", "aggr_func": "max", "data_range": [0, 1000], "frequency": 60000}, + "tag_min": {"webid": "w4", "aggr_func": "min", "data_range": [0, 1000], "frequency": 60000}, + "tag_lts": {"webid": "w5", "aggr_func": "lts", "data_range": [0, 1000], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 3, + "api_timeout": 30 + } + }, + "pi_web_api_server": { + "mode": "success", + "rows": [ + {"name": "tag_avg", "timestamp": "2024-06-01T12:00:00Z", "value": 10}, + {"name": "tag_avg", "timestamp": "2024-06-01T12:01:00Z", "value": 20}, + {"name": "tag_avg", "timestamp": "2024-06-01T12:02:00Z", "value": 30}, + {"name": "tag_mdn", "timestamp": "2024-06-01T12:00:00Z", "value": 1}, + {"name": "tag_mdn", "timestamp": "2024-06-01T12:01:00Z", "value": 9}, + {"name": "tag_mdn", "timestamp": "2024-06-01T12:02:00Z", "value": 5}, + {"name": "tag_max", "timestamp": "2024-06-01T12:00:00Z", "value": 5}, + {"name": "tag_max", "timestamp": "2024-06-01T12:01:00Z", "value": 15}, + {"name": "tag_max", "timestamp": "2024-06-01T12:02:00Z", "value": 10}, + {"name": "tag_min", "timestamp": "2024-06-01T12:00:00Z", "value": 50}, + {"name": "tag_min", "timestamp": "2024-06-01T12:01:00Z", "value": 30}, + {"name": "tag_min", "timestamp": "2024-06-01T12:02:00Z", "value": 40}, + {"name": "tag_lts", "timestamp": "2024-06-01T12:00:00Z", "value": 100}, + {"name": "tag_lts", "timestamp": "2024-06-01T12:01:00Z", "value": 200}, + {"name": "tag_lts", "timestamp": "2024-06-01T12:02:00Z", "value": 300} + ] + }, + "expected_values": { + "tag_avg": 20.0, + "tag_mdn": 5.0, + "tag_max": 15.0, + "tag_min": 30.0, + "tag_lts": 300.0 + } +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_timeout.json b/e2e/scenario_inputs/pi_web_api_scouter_timeout.json new file mode 100644 index 0000000..5d70d8e --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_timeout.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "model_id": "15", + "model_name": "PI Timeout", + "schedule_name": "pi-timeout", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 1, + "api_timeout": 2 + } + }, + "pi_web_api_server": {"mode": "timeout", "timeout_sleep_seconds": 5} +} diff --git a/e2e/scenario_inputs/scouter_empty_mongo.json b/e2e/scenario_inputs/scouter_empty_mongo.json new file mode 100644 index 0000000..0738993 --- /dev/null +++ b/e2e/scenario_inputs/scouter_empty_mongo.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "topic": "e2e-topic", + "model_id": "3", + "model_name": "Scouter Empty Mongo", + "schedule_name": "scouter-empty", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": { + "webid": "webid1", + "aggr_func": "avg", + "data_range": [0, 100], + "frequency": 60000 + } + } + }, + "raw_documents": [] +} diff --git a/e2e/scenario_inputs/scouter_happy_path.json b/e2e/scenario_inputs/scouter_happy_path.json new file mode 100644 index 0000000..0ca6510 --- /dev/null +++ b/e2e/scenario_inputs/scouter_happy_path.json @@ -0,0 +1,38 @@ +{ + "workflow_input": { + "topic": "e2e-topic", + "model_id": "1", + "model_name": "Scouter E2E Model", + "schedule_name": "scouter-happy", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": { + "webid": "webid1", + "aggr_func": "avg", + "data_range": [0, 100], + "frequency": 60000 + } + } + }, + "raw_documents": [ + { + "inserted_at": "2024-06-01 12:00:00.000000+0000", + "timestamp": "2024-06-01 12:00:00+0000", + "name": "tag1", + "value": 10.5, + "tag": "webid1" + }, + { + "inserted_at": "2024-06-01 12:01:00.000000+0000", + "timestamp": "2024-06-01 12:01:00+0000", + "name": "tag1", + "value": 20.0, + "tag": "webid1" + } + ] +} diff --git a/e2e/scenario_inputs/scouter_incremental_load.json b/e2e/scenario_inputs/scouter_incremental_load.json new file mode 100644 index 0000000..4dc0f73 --- /dev/null +++ b/e2e/scenario_inputs/scouter_incremental_load.json @@ -0,0 +1,43 @@ +{ + "workflow_input": { + "topic": "e2e-topic", + "model_id": "2", + "model_name": "Scouter Incremental", + "schedule_name": "scouter-incremental", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": { + "webid": "webid1", + "aggr_func": "avg", + "data_range": [0, 100], + "frequency": 60000 + } + } + }, + "redis_seed": { + "last_data_timestamp": "2024-06-01 10:00:00.000000+0000" + }, + "raw_documents": [ + { + "inserted_at": "2024-06-01 09:00:00.000000+0000", + "timestamp": "2024-06-01 09:00:00+0000", + "name": "tag1", + "value": 1.0, + "tag": "webid1" + }, + { + "inserted_at": "2024-06-01 11:00:00.000000+0000", + "timestamp": "2024-06-01 11:00:00+0000", + "name": "tag1", + "value": 99.0, + "tag": "webid1" + } + ], + "expected_newer_count": 1, + "expected_last_timestamp": "2024-06-01 11:00:00.000000+0000" +} diff --git a/e2e/scenario_inputs/scouter_no_redis_timestamp_first_run.json b/e2e/scenario_inputs/scouter_no_redis_timestamp_first_run.json new file mode 100644 index 0000000..5711b5a --- /dev/null +++ b/e2e/scenario_inputs/scouter_no_redis_timestamp_first_run.json @@ -0,0 +1,31 @@ +{ + "workflow_input": { + "topic": "e2e-topic", + "model_id": "4", + "model_name": "Scouter First Run", + "schedule_name": "scouter-first-run", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": { + "webid": "webid1", + "aggr_func": "lts", + "data_range": [0, 100], + "frequency": 60000 + } + } + }, + "raw_documents": [ + { + "inserted_at": "2024-06-01 08:00:00.000000+0000", + "timestamp": "2024-06-01 08:00:00+0000", + "name": "tag1", + "value": 42.0, + "tag": "webid1" + } + ] +} diff --git a/e2e/scenarios.md b/e2e/scenarios.md new file mode 100644 index 0000000..caac4c5 --- /dev/null +++ b/e2e/scenarios.md @@ -0,0 +1,138 @@ +# Scouter E2E scenario catalog + +## Execution context + +- **MongoDB**, **Redis**, and **PostgreSQL** run as session-scoped testcontainers with autouse cleanup between tests. +- **PI Web API** is an in-process HTTP server (`e2e/pi_web_api_test_server.py`) speaking the wire format consumed by `PIWebAPIClient`. +- **Temporal** uses `WorkflowEnvironment.start_local()` and a single worker on `scouter-test-queue`. +- **Production code is not mocked** (except `Logger` and optional notification insert spy). + +--- + +## 0. Harness smoke tests + +Diagnostic-only checks under `e2e/test_harness_smoke.py`. They are not business scenarios; they exist to fail fast when the harness itself (Docker / containers / Temporal worker wiring) is broken, before the numbered suite runs. + +### 0.0.1 Postgres schema ready (`test_postgres_schema_ready`) + +Confirms the autouse fixture executed `db_schema.sql` and `sientia_data.laborious_data` exists in the Postgres testcontainer. + +### 0.0.2 Activities construct (`test_activities_construct`) + +Confirms the production `Activities` instance initializes against the Mongo/Redis/Postgres testcontainers without hanging (no `patch(...)` involved). + +### 0.0.3 Temporal PI happy path (`test_temporal_pi_happy_path`) + +End-to-end liveness check: `WorkflowEnvironment.start_local()` + worker + in-process PI server + `PIWebAPIScouter` complete without raising. Functional assertions for this flow live in scenario **2.1.1**. + +--- + +## 1. Scouter workflow + +### 1.1.1 Happy path + +Seed `raw_` with multiple documents, run `Scouter`, assert Postgres rows and Redis `last_data_timestamp:scouter:`. + +### 1.2.1 Incremental load + +Pre-seed Redis timestamp; seed older and newer Mongo docs; assert only newer rows export and timestamp advances. + +### 1.3.1 Empty Mongo early exit + +Empty `raw_`; workflow exits without Postgres rows or Redis timestamp key. + +### 1.3.2 No Redis timestamp first run + +No prior Redis key; all seeded Mongo docs load and timestamp is written after success. + +--- + +## 2. PIWebAPIScouter workflow + +### 2.1.1 Happy path + +PI server `success` mode with two tags; assert Postgres rows, Redis hold key, one HTTP request recorded. + +### 2.1.2 Multiple tags + +Five tags with `avg` / `mdn` / `max` / `min` / `lts`; assert five distinct `variable` values and exact aggregated numbers in Postgres. + +### 2.1.3 Debug data package + +`debug_data_package=True`; assert `data_package_pi_web_api_scouter_*` Redis key with `data` and `held_data`. + +### 2.2.1 Empty response early exit + +Server `empty` mode; zero Postgres rows for `model_id`, one request recorded. + +### 2.3.1 PI Web API connection error + +Server `error` mode (HTTP 5xx); workflow fails; `PI_WEB_API_REQUEST_ERROR` notification in Mongo. + +### 2.3.2 PI Web API timeout + +Server `timeout` mode; workflow fails; `PI_WEB_API_REQUEST_ERROR` notification sent. + +### 2.3.3 Invalid endpoint + +Workflow uses `/invalid/endpoint` (404); workflow fails; `PI_WEB_API_REQUEST_ERROR` notification sent. + +--- + +## 3. CoreScouter subworkflow + +### 3.1.1 Complete processing success + +Single tag, no filters; Postgres row and `held_data_*` Redis key; no `data_package_*` key. + +### 3.1.2 Null values filter discard + +`NULL_VALUES_FILTER` DISCARD; one WARNING notification; only valid row in Postgres. + +### 3.1.3 Null values filter warn + +`NULL_VALUES_FILTER` WARN; notification sent; both rows in Postgres. + +### 3.1.4 Out of bounds filter discard + +`OUT_OF_BOUNDS_FILTER` DISCARD; in-range row only; WARNING notification. + +### 3.1.5 Aggregation avg + +Three points; Postgres `value` equals arithmetic mean (20.0). + +### 3.1.6 Aggregation mdn + +Median equals 5.0 in Postgres. + +### 3.1.7 Aggregation max + +Maximum equals 15.0 in Postgres. + +### 3.1.8 Aggregation min + +Minimum equals 30.0 in Postgres. + +### 3.1.9 Aggregation lts + +Last-by-timestamp value equals 300.0 in Postgres. + +### 3.1.10 Fill missing tags + +`fill_missing_tags=True`; `held_data_*` contains missing tag keys with `None`. + +### 3.1.11 Debug data package + +`debug_data_package=True`; `data_package_*` Redis key decodes to dict with `data` and `held_data`. + +### 3.2.1 Empty after grouping early exit + +Empty column-oriented `data`; no Postgres rows; no populated `held_data_*`. + +### 3.3.1 Invalid aggregation function + +`aggr_func= bogus`; `AGGREGATION_ISSUES` ERROR notification; bogus tag absent from Postgres. + +### 3.3.2 Postgres export failure surfaces + +Drop `value` column before run; workflow fails; ERROR notification in Mongo. diff --git a/e2e/test_harness_smoke.py b/e2e/test_harness_smoke.py new file mode 100644 index 0000000..276cee0 --- /dev/null +++ b/e2e/test_harness_smoke.py @@ -0,0 +1,51 @@ +"""Fast smoke checks for E2E fixture wiring.""" + +import pytest + +from e2e.helpers import ( + apply_pi_web_api_server_config, + load_scenario_input, + make_workflow_id, + start_and_await_workflow, +) +from e2e.pi_web_api_test_server import PIWebAPITestServer +from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + + +@pytest.mark.e2e +def test_postgres_schema_ready(postgres_engine): + """Verify autouse schema setup created laborious_data.""" + with postgres_engine.connect() as conn: + count = conn.exec_driver_sql( + 'SELECT COUNT(*) FROM information_schema.tables ' + "WHERE table_schema = 'sientia_data' AND table_name = 'laborious_data'" + ).scalar() + assert count == 1 + + +@pytest.mark.e2e +def test_activities_construct(test_activities): + """Verify Activities initializes against testcontainers without hanging.""" + assert test_activities.redis_repository is not None + assert test_activities.mongodb_repository is not None + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_temporal_pi_happy_path( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, +): + """Minimal Temporal path: PIWebAPIScouter happy path completes.""" + scenario = load_scenario_input('pi_web_api_scouter_happy_path') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + scenario['workflow_input'], + make_workflow_id('smoke-pi-happy'), + timeout=60.0, + ) diff --git a/e2e/test_pi_web_api_scouter_main_workflow.py b/e2e/test_pi_web_api_scouter_main_workflow.py new file mode 100644 index 0000000..5696310 --- /dev/null +++ b/e2e/test_pi_web_api_scouter_main_workflow.py @@ -0,0 +1,222 @@ +""" +End-to-end tests for the PIWebAPIScouter main workflow. +""" + +import json + +import pytest +from redis import Redis +from temporalio.client import WorkflowFailureError +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + +from e2e.conftest import E2E_DATABASE +from e2e.helpers import ( + apply_pi_web_api_server_config, + count_laborious_rows, + count_notifications, + fetch_laborious_rows, + load_scenario_input, + make_workflow_id, + start_and_await_workflow, +) +from e2e.pi_web_api_test_server import PIWebAPITestServer +from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_2_1_1_happy_path( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + postgres_engine, + redis_client: Redis, +): + scenario = load_scenario_input('pi_web_api_scouter_happy_path') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-happy'), + ) + + model_id = workflow_input['model_id'] + assert count_laborious_rows(postgres_engine, model_id) >= 1 + assert len(redis_client.keys('held_data_*')) >= 1 + assert len(pi_web_api_server.requests) == 1 + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_2_1_2_multiple_tags( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + postgres_engine, +): + scenario = load_scenario_input('pi_web_api_scouter_multiple_tags') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-multi'), + ) + + rows = { + row['variable']: float(row['value']) + for row in fetch_laborious_rows(postgres_engine, workflow_input['model_id']) + } + for tag, expected in scenario['expected_values'].items(): + assert tag in rows + assert rows[tag] == pytest.approx(expected) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_2_1_3_debug_data_package( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + redis_client: Redis, +): + scenario = load_scenario_input('pi_web_api_scouter_debug_data_package') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-debug'), + ) + + keys = redis_client.keys('data_package_pi_web_api_scouter_*') + assert len(keys) >= 1 + payload = json.loads(redis_client.get(keys[0])) + assert 'data' in payload and 'held_data' in payload + + +@pytest.mark.e2e +@pytest.mark.asyncio +@pytest.mark.xfail( + strict=True, + reason='Empty PI DataFrame lacks timestamp column in get_tag_values; tracked in fix-pi-empty-response-handling', +) +async def test_scenario_2_2_1_empty_response_early_exit( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + postgres_engine, +): + scenario = load_scenario_input('pi_web_api_scouter_empty_response') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-empty'), + ) + + assert count_laborious_rows(postgres_engine, workflow_input['model_id']) == 0 + assert len(pi_web_api_server.requests) == 1 + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_2_3_1_pi_web_api_connection_error( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + mongo_uri: str, +): + scenario = load_scenario_input('pi_web_api_scouter_connection_error') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + with pytest.raises(WorkflowFailureError): + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-conn-error'), + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='PI_WEB_API_REQUEST_ERROR', + level='ERROR', + ) + >= 1 + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_2_3_2_pi_web_api_timeout( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + mongo_uri: str, +): + scenario = load_scenario_input('pi_web_api_scouter_timeout') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + with pytest.raises(WorkflowFailureError): + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-timeout'), + timeout=180.0, + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='PI_WEB_API_REQUEST_ERROR', + ) + >= 1 + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_2_3_3_invalid_endpoint( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + mongo_uri: str, +): + scenario = load_scenario_input('pi_web_api_scouter_invalid_endpoint') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + with pytest.raises(WorkflowFailureError): + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-invalid-endpoint'), + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='PI_WEB_API_REQUEST_ERROR', + ) + >= 1 + ) diff --git a/e2e/test_scouter_main_workflow.py b/e2e/test_scouter_main_workflow.py new file mode 100644 index 0000000..42ccbe1 --- /dev/null +++ b/e2e/test_scouter_main_workflow.py @@ -0,0 +1,154 @@ +""" +End-to-end tests for the Scouter main workflow (Mongo load path). +""" + +import json + +import pytest +from redis import Redis + +from e2e.conftest import E2E_DATABASE +from e2e.helpers import ( + count_laborious_rows, + load_scenario_input, + make_workflow_id, + seed_last_data_timestamp, + seed_raw_collection, + start_and_await_workflow, +) +from scouter.workflow.scouter import Scouter +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_1_1_1_happy_path( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + redis_client: Redis, + postgres_engine, +): + scenario = load_scenario_input('scouter_happy_path') + workflow_input = scenario['workflow_input'] + seed_raw_collection( + mongo_uri, + E2E_DATABASE, + workflow_input['schedule_name'], + scenario['raw_documents'], + ) + + await start_and_await_workflow( + temporal_env.client, + Scouter.run, + workflow_input, + make_workflow_id('scouter-happy'), + ) + + model_id = workflow_input['model_id'] + assert count_laborious_rows(postgres_engine, model_id) >= 1 + + key = f"last_data_timestamp:scouter:{workflow_input['schedule_name']}" + assert redis_client.get(key) is not None + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_1_2_1_incremental_load( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + redis_client: Redis, + postgres_engine, +): + scenario = load_scenario_input('scouter_incremental_load') + workflow_input = scenario['workflow_input'] + redis_seed = scenario['redis_seed'] + seed_last_data_timestamp( + redis_client, + 'scouter', + workflow_input['schedule_name'], + redis_seed['last_data_timestamp'], + ) + seed_raw_collection( + mongo_uri, + E2E_DATABASE, + workflow_input['schedule_name'], + scenario['raw_documents'], + ) + + await start_and_await_workflow( + temporal_env.client, + Scouter.run, + workflow_input, + make_workflow_id('scouter-incremental'), + ) + + model_id = workflow_input['model_id'] + assert count_laborious_rows(postgres_engine, model_id) == scenario['expected_newer_count'] + + stored = json.loads( + redis_client.get(f"last_data_timestamp:scouter:{workflow_input['schedule_name']}") + ) + assert stored == scenario['expected_last_timestamp'] + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_1_3_1_empty_mongo_early_exit( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + redis_client: Redis, + postgres_engine, +): + scenario = load_scenario_input('scouter_empty_mongo') + workflow_input = scenario['workflow_input'] + seed_raw_collection( + mongo_uri, + E2E_DATABASE, + workflow_input['schedule_name'], + scenario['raw_documents'], + ) + + await start_and_await_workflow( + temporal_env.client, + Scouter.run, + workflow_input, + make_workflow_id('scouter-empty'), + ) + + assert count_laborious_rows(postgres_engine, workflow_input['model_id']) == 0 + key = f"last_data_timestamp:scouter:{workflow_input['schedule_name']}" + assert redis_client.get(key) is None + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_1_3_2_no_redis_timestamp_first_run( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + redis_client: Redis, + postgres_engine, +): + scenario = load_scenario_input('scouter_no_redis_timestamp_first_run') + workflow_input = scenario['workflow_input'] + seed_raw_collection( + mongo_uri, + E2E_DATABASE, + workflow_input['schedule_name'], + scenario['raw_documents'], + ) + + await start_and_await_workflow( + temporal_env.client, + Scouter.run, + workflow_input, + make_workflow_id('scouter-first-run'), + ) + + assert count_laborious_rows(postgres_engine, workflow_input['model_id']) >= 1 + key = f"last_data_timestamp:scouter:{workflow_input['schedule_name']}" + assert redis_client.get(key) is not None diff --git a/e2e/test_subworkflow_core_scouter.py b/e2e/test_subworkflow_core_scouter.py new file mode 100644 index 0000000..39abed4 --- /dev/null +++ b/e2e/test_subworkflow_core_scouter.py @@ -0,0 +1,360 @@ +""" +End-to-end tests for the CoreScouter subworkflow. +""" + +import json + +import pytest +from redis import Redis +from sqlalchemy import text +from temporalio.client import WorkflowFailureError +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + +from e2e.conftest import E2E_DATABASE +from e2e.helpers import ( + count_laborious_rows, + count_notifications, + fetch_laborious_rows, + load_scenario_input, + make_workflow_id, + start_and_await_workflow, +) +from scouter.activities.activities import Activities +from scouter.workflow.sub_workflows.core_scouter import CoreScouter + + +def _held_data_blob(test_activities: Activities, workflow_input: dict) -> dict | None: + """ + Read held_data Redis payload via the production RedisRepository. + + Return: + Decoded held-data dict or None + """ + key = ( + f"held_data_{workflow_input['workflow_name']}_{workflow_input['schedule_name']}" + ) + metadata = workflow_input['metadata']['metadata'] + return test_activities.redis_repository.get(key, metadata=metadata) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_1_complete_processing_success( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + test_activities: Activities, + postgres_engine, + redis_client: Redis, +): + scenario = load_scenario_input('core_scouter_happy_path') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-happy'), + ) + + assert count_laborious_rows(postgres_engine, workflow_input['model_id']) >= 1 + assert _held_data_blob(test_activities, workflow_input) is not None + assert len(redis_client.keys('data_package_*')) == 0 + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_2_null_values_filter_discard( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + postgres_engine, +): + scenario = load_scenario_input('core_scouter_null_values_filter_discard') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-null-discard'), + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='DATA_QUALITY_GATE_ISSUES__NULL_VALUES_FILTER', + level='WARNING', + ) + == 1 + ) + rows = fetch_laborious_rows(postgres_engine, workflow_input['model_id']) + assert len(rows) == 1 + assert rows[0]['variable'] == 'tag2' + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_3_null_values_filter_warn( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + postgres_engine, +): + scenario = load_scenario_input('core_scouter_null_values_filter_warn') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-null-warn'), + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='DATA_QUALITY_GATE_ISSUES__NULL_VALUES_FILTER', + level='WARNING', + ) + == 1 + ) + assert count_laborious_rows(postgres_engine, workflow_input['model_id']) == 2 + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_4_out_of_bounds_filter_discard( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + postgres_engine, +): + scenario = load_scenario_input('core_scouter_out_of_bounds_filter_discard') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-oob-discard'), + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='DATA_QUALITY_GATE_ISSUES__OUT_OF_BOUNDS_FILTER', + level='WARNING', + ) + == 1 + ) + rows = fetch_laborious_rows(postgres_engine, workflow_input['model_id']) + assert len(rows) == 1 + assert rows[0]['variable'] == 'tag2' + + +async def _run_aggregation_scenario( + temporal_env: WorkflowEnvironment, + postgres_engine, + slug: str, + workflow_id_prefix: str, +) -> None: + scenario = load_scenario_input(slug) + workflow_input = scenario['workflow_input'] + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id(workflow_id_prefix), + ) + rows = fetch_laborious_rows(postgres_engine, workflow_input['model_id']) + tag_name = next(iter(workflow_input['model_tags'])) + value = next(row['value'] for row in rows if row['variable'] == tag_name) + assert float(value) == pytest.approx(scenario['expected_value']) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_5_aggregation_avg( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + await _run_aggregation_scenario( + temporal_env, postgres_engine, 'core_scouter_aggregation_avg', 'core-avg' + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_6_aggregation_mdn( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + await _run_aggregation_scenario( + temporal_env, postgres_engine, 'core_scouter_aggregation_mdn', 'core-mdn' + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_7_aggregation_max( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + await _run_aggregation_scenario( + temporal_env, postgres_engine, 'core_scouter_aggregation_max', 'core-max' + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_8_aggregation_min( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + await _run_aggregation_scenario( + temporal_env, postgres_engine, 'core_scouter_aggregation_min', 'core-min' + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_9_aggregation_lts( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + await _run_aggregation_scenario( + temporal_env, postgres_engine, 'core_scouter_aggregation_lts', 'core-lts' + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_10_fill_missing_tags( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + test_activities: Activities, +): + scenario = load_scenario_input('core_scouter_fill_missing_tags') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-fill-tags'), + ) + + held = _held_data_blob(test_activities, workflow_input) + assert held is not None + for tag in scenario['expected_missing_tags']: + assert tag in held + assert held[tag] is None + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_11_debug_data_package( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + redis_client: Redis, +): + scenario = load_scenario_input('core_scouter_debug_data_package') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-debug-pkg'), + ) + + keys = redis_client.keys('data_package_*') + assert len(keys) >= 1 + payload = json.loads(redis_client.get(keys[0])) + assert 'data' in payload and 'held_data' in payload + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_2_1_empty_after_grouping_early_exit( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + test_activities: Activities, + postgres_engine, +): + scenario = load_scenario_input('core_scouter_empty_after_grouping') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-empty-group'), + ) + + assert count_laborious_rows(postgres_engine, workflow_input['model_id']) == 0 + assert _held_data_blob(test_activities, workflow_input) is None + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_3_1_invalid_aggregation_function( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + postgres_engine, +): + scenario = load_scenario_input('core_scouter_invalid_aggregation') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-bad-aggr'), + ) + + variables = {row['variable'] for row in fetch_laborious_rows(postgres_engine, workflow_input['model_id'])} + assert 'tag_bad' not in variables + assert 'tag_ok' in variables + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_3_2_postgres_export_failure_surfaces( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + postgres_engine, +): + scenario = load_scenario_input('core_scouter_postgres_export_failure') + workflow_input = scenario['workflow_input'] + + with postgres_engine.begin() as conn: + conn.execute(text('ALTER TABLE sientia_data.laborious_data DROP COLUMN value')) + + with pytest.raises(WorkflowFailureError): + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-pg-fail'), + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='ERROR_EXPORTING_DATA_TO_POSTGRES', + ) + >= 1 + ) diff --git a/get_data_pims.py b/get_data_pims.py new file mode 100644 index 0000000..ab65b2e --- /dev/null +++ b/get_data_pims.py @@ -0,0 +1,604 @@ +import requests # type: ignore +import pandas as pd # type: ignore +from typing import Optional, Dict, List + + +class PIMSClient: + """ + Cliente para interagir com a API do PIMS (PI System) da Votorantim. + + Esta classe fornece mΓ©todos para autenticação e busca de dados de streams/tags + do sistema PIMS atravΓ©s da API REST. + """ + + def __init__(self, base_url: str, api_key: Optional[str] = None, api_key_header: Optional[str] = "apikey", additional_headers: Optional[Dict[str, str]] = None): + """ + Inicializa o cliente PIMS. + + Args: + base_url (str): URL base da API (ex: https://votorantim.apimanagement.br10.hana.ondemand.com/v2/webapi/piwebapi) + api_key (Optional[str]): API Key para autenticação + api_key_header (Optional[str]): Nome do header onde a chave deve ser enviada (ex: "X-API-Key", "Ocp-Apim-Subscription-Key", "apikey") + additional_headers (Optional[Dict[str, str]]): CabeΓ§alhos adicionais para incluir em todas as requisiçáes + """ + self.base_url = base_url.rstrip('/') + self.api_key = api_key + self.api_key_header = api_key_header + self.session = requests.Session() + self.additional_headers = additional_headers or {} + self._authenticated = False + + def authenticate(self) -> bool: + """ + Configura a autenticação via cabeΓ§alhos. + + Returns: + bool: True se a configuração foi bem-sucedida, False caso contrΓ‘rio + """ + try: + default_headers: Dict[str, str] = { + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + if self.api_key and self.api_key_header: + default_headers[self.api_key_header] = self.api_key + + # Mescla cabeΓ§alhos adicionais (sobrescrevem os padrΓ΅es se necessΓ‘rio) + default_headers.update(self.additional_headers) + + self.session.headers.update(default_headers) + + # NΓ£o faz chamada de teste aqui para evitar 401 em endpoints protegidos; assume headers configurados + self._authenticated = True + return True + + except requests.exceptions.RequestException as e: + print(f"Erro na configuração da autenticação: {e}") + return False + + def get_stream_data(self, web_ids: Dict[str, str], start_time: str = "*-3d", end_time: str = "*") -> Optional[pd.DataFrame]: + """ + Busca dados de mΓΊltiplos streams/tags. + + Args: + web_ids (Dict[str, str]): DicionΓ‘rio no formato {tag_name: web_id} + start_time (str): Data/hora de inΓ­cio (formato: "*-3d" ou "yyyy-mm-dd") + end_time (str): Data/hora de fim (formato: "*" ou "yyyy-mm-dd") + + Returns: + pd.DataFrame: DataFrame onde as colunas sΓ£o o nome da tag, o Γ­ndice Γ© o Timestamp, e os valores sΓ£o os valores das tags + """ + if not self._authenticated: + if not self.authenticate(): + print("Falha na autenticação") + return None + + all_series = [] + + for tag_name, web_id in web_ids.items(): + try: + url = f"{self.base_url}/streams/{web_id}/recorded" + params = { + "startTime": start_time, + "endTime": end_time + } + response = self.session.get(url, params=params) + response.raise_for_status() + + data = response.json() + if 'Items' in data and data['Items']: + df = pd.DataFrame(data['Items']) + if 'Timestamp' in df.columns and 'Value' in df.columns: + # Converte timestamp + try: + df['Timestamp'] = pd.to_datetime( + df['Timestamp'], + format='ISO8601', + utc=True, + errors='coerce' + ) + except TypeError: + df['Timestamp'] = pd.to_datetime( + df['Timestamp'], + utc=True, + errors='coerce' + ) + # Arredonda timestamps para precisΓ£o de segundos + df['Timestamp'] = df['Timestamp'].dt.floor('s') + # Normaliza valores: quando a API retorna um dict, tenta extrair um nΓΊmero + def _extract_numeric(v): + if isinstance(v, dict): + # Casos comuns: {'Value': } ou aninhados + inner = v.get('Value') + if isinstance(inner, (int, float)): + return inner + # Tenta outros campos conhecidos + for k in ('NumericValue',): + inner2 = v.get(k) + if isinstance(inner2, (int, float)): + return inner2 + return None + if isinstance(v, (int, float)): + return v + # Converte strings numΓ©ricas, demais viram NaN + return pd.to_numeric(v, errors='coerce') + + df['Value'] = df['Value'].apply(_extract_numeric) + df['Value'] = pd.to_numeric(df['Value'], errors='coerce') + + df.set_index('Timestamp', inplace=True) + # Agrega valores por segundo para remover Γ­ndices duplicados + series = ( + df['Value'] + .groupby(level=0) + .mean() + .sort_index() + .rename(tag_name) + ) + all_series.append(series) + else: + print(f"Nenhum dado encontrado para a tag '{tag_name}' no perΓ­odo especificado") + + except requests.exceptions.RequestException as e: + print(f"Erro ao buscar dados do stream {tag_name}: {e}") + + if all_series: + result_df = pd.concat(all_series, axis=1) + return result_df + else: + print("Nenhum dado encontrado para as tags informadas.") + return pd.DataFrame() + + def search_streams(self, name_filter: Optional[str] = None, tag_filter: Optional[str] = None) -> Optional[List[Dict]]: + """ + Busca streams disponΓ­veis com filtros opcionais. + + Args: + name_filter (str): Filtro por nome do stream + tag_filter (str): Filtro por tag + + Returns: + List[Dict]: Lista de streams encontrados ou None se houver erro + """ + if not self._authenticated: + if not self.authenticate(): + print("Falha na autenticação") + return None + + try: + search_url = f"{self.base_url}/streams" + params = {} + + if name_filter: + params['nameFilter'] = name_filter + if tag_filter: + params['tag'] = tag_filter + + response = self.session.get(search_url, params=params) + response.raise_for_status() + + return response.json().get('Items', []) + + except requests.exceptions.RequestException as e: + print(f"Erro ao buscar streams: {e}") + return None + + def get_stream_info(self, web_id: str) -> Optional[Dict]: + """ + ObtΓ©m informaçáes detalhadas de um stream especΓ­fico. + + Args: + web_id (str): WebID do stream + + Returns: + Dict: Informaçáes do stream ou None se houver erro + """ + if not self._authenticated: + if not self.authenticate(): + print("Falha na autenticação") + return None + + try: + info_url = f"{self.base_url}/streams/{web_id}" + response = self.session.get(info_url) + response.raise_for_status() + + return response.json() + + except requests.exceptions.RequestException as e: + print(f"Erro ao buscar informaçáes do stream: {e}") + return None + + def get_web_ids_by_tags(self, data_server_id: str, tag_names: List[str]) -> Dict[str, Optional[str]]: + """ + Retorna os WebIds para uma lista de tags (pontos) em um Data Server especΓ­fico. + + Args: + data_server_id (str): ID/WebId do Data Server (ex: "F1DS-...") + tag_names (List[str]): Lista com os nomes exatos das tags + + Returns: + Dict[str, Optional[str]]: DicionΓ‘rio mapeando tag -> WebId (ou None se nΓ£o encontrada) + """ + + # Garante autenticação, similar ao script de teste + if not self._authenticated: + if not self.authenticate(): + print("Falha na autenticação") + return {tag: None for tag in tag_names} + + results: Dict[str, Optional[str]] = {} + + for tag in tag_names: + try: + # Monta a URL seguindo a lΓ³gica do script de teste fornecido no contexto + url = f"{self.base_url}/dataservers/{data_server_id}/points" + params = {"namefilter": tag} + print(url, params) + response = self.session.get(url, params=params) + response.raise_for_status() + + data = response.json() + + # Corrige: procurar a lista 'Items' como no script de teste + items = data.get("Items", []) if isinstance(data, dict) else [] + + web_id_value: Optional[str] = None + if items: + # Emula exatamente o resultado do script: pega primeiro item se disponΓ­vel + first_item = items[0] + if isinstance(first_item, dict): + web_id_value = first_item.get("WebId") + + results[tag] = web_id_value + except requests.exceptions.RequestException as e: + print(f"Erro ao buscar WebId para tag '{tag}': {e}") + results[tag] = None + + return results + + def multi_tags_agregadas( + self, + web_ids: List[str], + start_time: str, + end_time: str, + summary_duration: str = "15m", + summary_type: str = "average", + selected_fields: str = "Items.Name;Items.Items.Type;Items.Items.Value.Timestamp;Items.Items.Value.Value;Items.Items.Value.Good", + batch_size: int = 50, + ) -> Optional[Dict]: + """ + Chama o endpoint /streamsets/summary com mΓΊltiplos webids via GET e retorna o JSON bruto. + Para evitar URLs muito longas, realiza chamadas em lotes e agrega os resultados. + + Args: + web_ids (List[str]): Lista de WebIds a consultar + start_time (str): InΓ­cio do perΓ­odo (ex.: "2024-09-05" ou "*-1d") + end_time (str): Fim do perΓ­odo (ex.: "2024-09-06" ou "*") + summary_duration (str): Duração do resumo (ex.: "15m") + summary_type (str): Tipo de resumo (ex.: "average", "minimum", "maximum", etc.) + selected_fields (str): Campos a retornar + batch_size (int): Tamanho do lote de WebIds por requisição + + Returns: + Optional[Dict]: JSON com "Items" unificados ou None em caso de erro + """ + if not self._authenticated: + if not self.authenticate(): + print("Falha na autenticação") + return None + + try: + url = f"{self.base_url}/streamsets/summary" + all_items: List[Dict] = [] + + for i in range(0, len(web_ids), batch_size): + chunk = web_ids[i:i + batch_size] + # ConstrΓ³i lista de tuplas para repetir 'webid' como mΓΊltiplos params + params: List[tuple] = [("webid", wid) for wid in chunk] + params.extend([ + ("startTime", start_time), + ("endtime", end_time), # conforme imagem + ("summaryDuration", summary_duration), + ("summaryType", summary_type), + ("selectedFields", selected_fields), + ]) + + response = self.session.get(url, params=params, timeout=600000) + response.raise_for_status() + data = response.json() + items = data.get("Items", []) if isinstance(data, dict) else [] + if isinstance(items, list): + all_items.extend(items) + + return {"Items": all_items} + + except requests.exceptions.RequestException as e: + print(f"Erro ao buscar dados brutos de mΓΊltiplas tags: {e}") + return None + + def multi_tags_agregadas_df( + self, + web_ids: List[str], + start_time: str, + end_time: str, + summary_duration: str = "1m", + summary_type: str = "average", + selected_fields: str = "Items.Name;Items.Items.Type;Items.Items.Value.Timestamp;Items.Items.Value.Value;Items.Items.Value.Good" + ) -> pd.DataFrame: + """ + Chama /streamsets/summary para mΓΊltiplos webids e retorna DataFrame: + - Γ­ndice: Timestamp (precisΓ£o de segundos) + - colunas: nome da tag + - cΓ©lulas: Value (numΓ©rico) + """ + raw = self.multi_tags_agregadas( + web_ids=web_ids, + start_time=start_time, + end_time=end_time, + summary_duration=summary_duration, + summary_type=summary_type, + selected_fields=selected_fields, + ) + + if not raw or 'Items' not in raw or not isinstance(raw['Items'], list): + return pd.DataFrame() + + records = [] + + def _extract_numeric(v): + if isinstance(v, dict): + inner = v.get('Value') + if isinstance(inner, (int, float)): + return inner + for k in ('NumericValue',): + inner2 = v.get(k) + if isinstance(inner2, (int, float)): + return inner2 + return None + if isinstance(v, (int, float)): + return v + return pd.to_numeric(v, errors='coerce') + + for entry in raw['Items']: + tag_name = entry.get('Name') + series_items = entry.get('Items') or [] + for it in series_items: + v = (it.get('Value') or {}) if isinstance(it, dict) else {} + ts = v.get('Timestamp') if isinstance(v, dict) else None + val = v.get('Value') if isinstance(v, dict) else None + val = _extract_numeric(val) + if ts is not None: + records.append({ + 'Timestamp': ts, + 'Tag': tag_name, + 'Value': val, + }) + + if not records: + return pd.DataFrame() + + df = pd.DataFrame.from_records(records) + # Converte e arredonda timestamps + try: + df['Timestamp'] = pd.to_datetime(df['Timestamp'], format='ISO8601', utc=True, errors='coerce') + except TypeError: + df['Timestamp'] = pd.to_datetime(df['Timestamp'], utc=True, errors='coerce') + df['Timestamp'] = df['Timestamp'].dt.floor('s') + + # Pivot: Γ­ndice timestamp, colunas nome da tag, valores numΓ©ricos + df_pivot = df.pivot_table(index='Timestamp', columns='Tag', values='Value', aggfunc='mean') + df_pivot.sort_index(inplace=True) + return df_pivot + + def multi_tags_brutas( + self, + web_ids: List[str], + start_time: str, + end_time: str, + max_count: int = 10000, + batch_size: int = 50, + ) -> Optional[Dict]: + """ + Chama o endpoint /streamsets/recorded com mΓΊltiplos webids via GET e retorna o JSON bruto. + Para evitar URLs muito longas, realiza chamadas em lotes e agrega os resultados. + + Args: + web_ids (List[str]): Lista de WebIds a consultar + start_time (str): InΓ­cio do perΓ­odo (ex.: "2024-09-01" ou "*-1d") + end_time (str): Fim do perΓ­odo (ex.: "2024-09-09" ou "*") + max_count (int): NΓΊmero mΓ‘ximo de registros a retornar por requisição + batch_size (int): Tamanho do lote de WebIds por requisição + + Returns: + Optional[Dict]: JSON com "Items" unificados ou None em caso de erro + """ + if not self._authenticated: + if not self.authenticate(): + print("Falha na autenticação") + return None + + try: + url = f"{self.base_url}/streamsets/recorded" + all_items: List[Dict] = [] + + for i in range(0, len(web_ids), batch_size): + chunk = web_ids[i:i + batch_size] + # ConstrΓ³i lista de tuplas para repetir 'webid' como mΓΊltiplos params + params: List[tuple] = [("webid", wid) for wid in chunk] + params.extend([ + ("startTime", start_time), + ("endTime", end_time), + ("maxCount", str(max_count)), + ]) + + response = self.session.get(url, params=params, timeout=600000) + response.raise_for_status() + data = response.json() + items = data.get("Items", []) if isinstance(data, dict) else [] + if isinstance(items, list): + all_items.extend(items) + + return {"Items": all_items} + + except requests.exceptions.RequestException as e: + print(f"Erro ao buscar dados brutos de mΓΊltiplas tags: {e}") + return None + + def multi_tags_brutas_df( + self, + web_ids: List[str], + start_time: str, + end_time: str, + max_count: int = 10000, + ) -> pd.DataFrame: + """ + Chama /streamsets/recorded para mΓΊltiplos webids e retorna DataFrame: + - Γ­ndice: Timestamp (precisΓ£o de segundos) + - colunas: nome da tag + - cΓ©lulas: Value (numΓ©rico) + + Args: + web_ids (List[str]): Lista de WebIds a consultar + start_time (str): InΓ­cio do perΓ­odo (ex.: "2024-09-01" ou "*-1d") + end_time (str): Fim do perΓ­odo (ex.: "2024-09-09" ou "*") + max_count (int): NΓΊmero mΓ‘ximo de registros a retornar por requisição + + Returns: + pd.DataFrame: DataFrame com timestamp como Γ­ndice e tags como colunas + """ + raw = self.multi_tags_brutas( + web_ids=web_ids, + start_time=start_time, + end_time=end_time, + max_count=max_count, + ) + + if not raw or 'Items' not in raw or not isinstance(raw['Items'], list): + return pd.DataFrame() + + records = [] + + def _extract_numeric(v): + if isinstance(v, dict): + inner = v.get('Value') + if isinstance(inner, (int, float)): + return inner + for k in ('NumericValue',): + inner2 = v.get(k) + if isinstance(inner2, (int, float)): + return inner2 + return None + if isinstance(v, (int, float)): + return v + return pd.to_numeric(v, errors='coerce') + + for entry in raw['Items']: + tag_name = entry.get('Name') + # Para /streamsets/recorded, cada entry tem uma lista 'Items' com objetos contendo Timestamp e Value diretamente + series_items = entry.get('Items') or [] + + # Processa items aninhados + for it in series_items: + if isinstance(it, dict): + # Estrutura: {'Timestamp': '2024-09-01T23:00:00Z', 'Value': 4431.94141, ...} + if 'Timestamp' in it and 'Value' in it: + ts = it.get('Timestamp') + val = it.get('Value') + val = _extract_numeric(val) + if ts is not None: + records.append({ + 'Timestamp': ts, + 'Tag': tag_name, + 'Value': val, + }) + + if not records: + return pd.DataFrame() + + df = pd.DataFrame.from_records(records) + # Converte e arredonda timestamps + try: + df['Timestamp'] = pd.to_datetime(df['Timestamp'], format='ISO8601', utc=True, errors='coerce') + except TypeError: + df['Timestamp'] = pd.to_datetime(df['Timestamp'], utc=True, errors='coerce') + df['Timestamp'] = df['Timestamp'].dt.floor('s') + + # Pivot: Γ­ndice timestamp, colunas nome da tag, valores numΓ©ricos + # Agrega valores duplicados no mesmo timestamp usando mΓ©dia + df_pivot = df.pivot_table(index='Timestamp', columns='Tag', values='Value', aggfunc='mean') + df_pivot.sort_index(inplace=True) + return df_pivot + + def close(self): + """Fecha a sessΓ£o HTTP.""" + self.session.close() + + + + +# Exemplo de uso +if __name__ == "__main__": + # Configuração do cliente + base_url = "https://votorantim.apimanagement.br10.hana.ondemand.com/v2/webapi/piwebapi" + api_key = "zK4WbZAZGBwSaQ5GJzhPpp06P1PGueqP" + + # Cria instΓ’ncia do cliente + pims_client = PIMSClient(base_url, api_key) + + # Exemplo: buscar dados de um stream especΓ­fico + tag_forms = [ + 'CI-J3J01S1', 'CI-J3P01T1A', 'CI-J3P03S1', 'CI-W3A05F1', + 'CI-W3A50A1', 'CI-W3A50A2', 'CI-W3A50A3', 'CI-W3A50P1', 'CI-W3A50T1', + 'CI-W3A55P1', 'CI-W3A55T1', 'CI-W3A65_Cl', 'CI-W3A65_SO3', 'CI-W3A71P1', + 'CI-W3A71P2', 'CI-W3A71P3', 'CI-W3E01F1', 'CI-W3K01S1', 'CI-W3K01T1', + 'CI-W3K01T2', 'CI-W3K01T3', 'CI-W3K01T4', 'CI-W3K14P1', 'CI-W3P17S1', 'CI-W3V04P1', + 'CI-W3V04P3', 'CI-W3V21F1', 'CI-W3V21P1', 'CI-W3V30F1', 'CI-W3V33P1', + 'CI-W3W01A1', 'CI-W3W01A2', 'CI-W3W01A3', 'CI-W3W01G1', 'CI-W3W01P1', + 'CI-W3W01P2', 'CI-W3W03I1', 'CI-W3W03S1', 'CI-W3X21IN', 'CI-W3_C3S', + 'CI-W3_CAO', 'CI-W3_MA', 'CI-W3_MS', 'CI-W3_PL' + ] + web_ids = pims_client.get_web_ids_by_tags("F1DS-7fYgsRTtUOa7V9NIwSujAUElIQVZD", tag_forms) + + web_ids_list: List[str] = [wid for wid in web_ids.values() if isinstance(wid, str)] + + from datetime import datetime, timedelta + + # ParΓ’metros iniciais apenas atΓ© o dia (granulometria diΓ‘ria) + inicio = datetime(2023, 1, 1) # Somente a data, sem horas/minutos/segundos + fim = datetime.today().replace(hour=0, minute=0, second=0, microsecond=0) # AtΓ© hoje Γ  00:00 (comeΓ§o do dia atual) + delta = timedelta(days=1) + + + dfs = [] # lista para armazenar os dataframes parciais + + while inicio < fim: + proximo = min(inicio + delta, fim) # garante que nΓ£o passa da data atual + + print(f"Buscando de {inicio:%Y-%m-%d} atΓ© {proximo:%Y-%m-%d}...") + + df_parcial = pims_client.multi_tags_brutas_df( + web_ids_list, + inicio.strftime("%Y-%m-%d"), + proximo.strftime("%Y-%m-%d"), + max_count=1000 + ) + + dfs.append(df_parcial) + inicio = proximo # avanΓ§a o cursor + # break + + # concatena todos em um ΓΊnico dataframe + df_final = pd.concat(dfs, ignore_index=False) + df_final.reset_index(inplace=True) + df_final.rename(columns={'index': 'timestamp'}, inplace=True) + # df_final = df_final.ffill() + # df_final = df_final.bfill() + + # save to csv + if not df_final.empty: + df_final.to_parquet("data_brutos_pims_no_fill.parquet", index=False) + + print(df_final.head()) + print(df_final.shape) + print(df_final.columns) \ No newline at end of file diff --git a/git-requirements-mapping.txt b/git-requirements-mapping.txt new file mode 100644 index 0000000..1362493 --- /dev/null +++ b/git-requirements-mapping.txt @@ -0,0 +1 @@ +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git:sientia-do \ No newline at end of file diff --git a/init_port_forward.sh b/init_port_forward.sh new file mode 100755 index 0000000..e9c0e3d --- /dev/null +++ b/init_port_forward.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +# Usage: +# 1) Edit the PORT_FORWARDS list below with entries of: +# +# 2) Run: ./init_port_forward.sh +# +# The script will start all port-forwards in the background and keep running +# until interrupted (Ctrl+C). On exit, it will clean up started port-forward processes. + +set -euo pipefail + +# Define your namespace/service/port combinations here +# Example entries: +# "default my-service 8080 80" +# "observability grafana 3000 3000" +PORT_FORWARDS=( + "mongodb my-release-mongodb 27017 27017" + "paradedb paradedb-rw 5432 5432" + "redis redis-master 6379 6379" + "temporal temporal-frontend 7233 7233" +) + +if [ ${#PORT_FORWARDS[@]} -eq 0 ]; then + echo "No port-forward entries defined. Edit PORT_FORWARDS in $(basename "$0")." + exit 1 +fi + +PIDS=() + +cleanup() { + echo "\nStopping port-forward processes..." + for pid in "${PIDS[@]}"; do + if kill -0 "$pid" >/dev/null 2>&1; then + kill "$pid" >/dev/null 2>&1 || true + fi + done +} + +trap cleanup EXIT INT TERM + +timestamp() { date '+%Y-%m-%d %H:%M:%S'; } + +# Allow overriding kubectl binary if needed +KUBECTL=${KUBECTL:-kubectl} + +is_port_free() { + local port="$1" + # Consider port free if nothing is listening locally on it + if command -v ss >/dev/null 2>&1; then + ! ss -ltn | awk '{print $4}' | grep -E "(^|:|\\])${port}$" >/dev/null 2>&1 + else + if command -v lsof >/dev/null 2>&1; then + ! lsof -tiTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1 + else + # Fallback: attempt to open a TCP connection; expect failure when nothing is listening + ! (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null + fi + fi +} + +free_port_if_stuck() { + local port="$1" + # Try multiple tools to free a stuck listener (often old kubectl PF) + if command -v lsof >/dev/null 2>&1; then + local pids + pids=$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null || true) + if [ -n "${pids}" ]; then + echo "[$(timestamp)] Found listeners on ${port}: ${pids}; terminating" + kill ${pids} 2>/dev/null || true + sleep 0.5 + fi + fi + if ! is_port_free "${port}"; then + if command -v fuser >/dev/null 2>&1; then + echo "[$(timestamp)] Forcing free of ${port} via fuser" + fuser -k "${port}/tcp" 2>/dev/null || true + sleep 0.5 + fi + fi +} + +run_port_forward() { + local namespace="$1" + local service_name="$2" + local local_port="$3" + local service_port="$4" + + # simple and robust supervisor loop with gentle backoff on failures + local delay=2 + local max_delay=20 + while true; do + free_port_if_stuck "${local_port}" + if ! is_port_free "${local_port}"; then + echo "[$(timestamp)] ns=${namespace} svc=${service_name} ${local_port}:${service_port} -> local port busy, retrying in 3s" + sleep 3 + continue + fi + + echo "[$(timestamp)] Starting port-forward: ns=${namespace} svc=${service_name} ${local_port}:${service_port}" + ${KUBECTL} -n "${namespace}" port-forward "svc/${service_name}" "${local_port}:${service_port}" \ + --address=127.0.0.1 --pod-running-timeout=2m --request-timeout=0 + rc=$? + + # If kubectl exits (e.g., connection reset by peer), wait a bit and retry + echo "[$(timestamp)] Port-forward exited (rc=${rc}): ns=${namespace} svc=${service_name} ${local_port}:${service_port}" + sleep "${delay}" + # Exponential backoff up to max_delay + if [ ${delay} -lt ${max_delay} ]; then + delay=$(( delay * 2 )) + if [ ${delay} -gt ${max_delay} ]; then + delay=${max_delay} + fi + fi + done +} + +ONLY_SERVICE_NAME="${ONLY_SERVICE_NAME:-}" + +for entry in "${PORT_FORWARDS[@]}"; do + read -r NAMESPACE SERVICE_NAME LOCAL_PORT SERVICE_PORT <<< "$entry" + if [ -n "${ONLY_SERVICE_NAME}" ] && [ "${SERVICE_NAME}" != "${ONLY_SERVICE_NAME}" ]; then + continue + fi + run_port_forward "${NAMESPACE}" "${SERVICE_NAME}" "${LOCAL_PORT}" "${SERVICE_PORT}" & + PIDS+=("$!") +done + +echo "All port-forwards started: ${#PIDS[@]} process(es). Press Ctrl+C to stop." + +# Do not exit the script if one port-forward fails; they self-restart +set +e +wait diff --git a/pi_web_api_fetch_data.py b/pi_web_api_fetch_data.py new file mode 100644 index 0000000..26761d4 --- /dev/null +++ b/pi_web_api_fetch_data.py @@ -0,0 +1,306 @@ +""" +Script to fetch WebIds from PI Web API and then retrieve historical values +for a list of tags over a given period in 30-day chunks, storing results +in a DataFrame indexed by timestamp. + +Based on pi_web_api_client.py and tests.ipynb. Run with project venv active. +Use # %% cell separators: run each cell in order (Run Cell / Shift+Enter). +""" + +# %% 1. Imports and configuration +import asyncio +import concurrent.futures +import json +import os +import time +from typing import Any + +import pandas as pd +import requests +from unittest.mock import MagicMock, AsyncMock + +from sientia_do.repository.pi_web_api_client import PIWebAPIClient + +BASE_URL = 'https://pivision.votorantimcimentos.com/piwebapi' +AUTH_TOKEN = 'dmlkX3ZjbmV0XHN2Yy5waW9zaS5wcmQud2ViYXBpOlN2Y1ByRFdlQkBQaQ==' +WEBID_LOOKUP_PATH = 'dataservers/F1DS-7fYgsRTtUOa7V9NIwSujAUElIQVZD/points' +PERIOD_DAYS = (365 * 3) + 50 +CHUNK_DAYS = 5 +ENDPOINT = '/streamsets/recorded' +API_TIMEOUT = 60 +WEB_IDS_SAVE_PATH = 'web_ids.json' +TAG_NAMES = [ + 'CI-J3J01S1', 'CI-J3P01T1A', 'CI-J3P03S1', 'CI-W3A05F1', + 'CI-W3A50A1', 'CI-W3A50A2', 'CI-W3A50A3', 'CI-W3A50P1', 'CI-W3A50T1', + 'CI-W3A55P1', 'CI-W3A55T1', 'CI-W3A65_Cl', 'CI-W3A65_SO3', 'CI-W3A71P1', + 'CI-W3A71P2', 'CI-W3A71P3', 'CI-W3E01F1', 'CI-W3K01S1', 'CI-W3K01T1', + 'CI-W3K01T2', 'CI-W3K01T3', 'CI-W3K01T4', 'CI-W3K14P1', 'CI-W3P17S1', 'CI-W3V04P1', + 'CI-W3V04P3', 'CI-W3V21F1', 'CI-W3V21P1', 'CI-W3V30F1', 'CI-W3V33P1', + 'CI-W3W01A1', 'CI-W3W01A2', 'CI-W3W01G1', 'CI-W3W01P1', + 'CI-W3W01P2', 'CI-W3W03I1', 'CI-W3W03S1', 'CI-W3X21IN', 'CI-W3_C3S', + 'CI-W3_CAO', 'CI-W3_MA', 'CI-W3_MS', 'CI-W3_PL' +] + +print(f'Config: BASE_URL={BASE_URL}, PERIOD_DAYS={PERIOD_DAYS}, CHUNK_DAYS={CHUNK_DAYS}, tags={len(TAG_NAMES)}, WEB_IDS_SAVE_PATH={WEB_IDS_SAVE_PATH}') + + +def run_async(coro): + """ + Run a coroutine from sync code. Works in scripts and in Jupyter (where an event loop is already running). + + Args: + coro: Coroutine to run. + + Return: + Result of the coroutine. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + with concurrent.futures.ThreadPoolExecutor() as pool: + future = pool.submit(asyncio.run, coro) + return future.result() + + +# %% 2. Helper: fetch WebIds from PI Web API +def fetch_webids( + tag_names: list[str], + base_url: str, + auth_token: str, + webid_lookup_path: str, + delay_seconds: float = 0.5, +) -> dict[str, dict[str, Any]]: + """ + Resolve WebIds for the given tag names via PI Web API points endpoint. + + Args: + tag_names: List of tag names to resolve. + base_url: PI Web API base URL (no trailing slash). + auth_token: Basic auth token (base64-encoded user:password). + webid_lookup_path: Path relative to base_url, with {tag} placeholder for namefilter. + delay_seconds: Delay between requests to avoid rate limiting. + + Returns: + dict mapping tag name to {'webid': str, 'aggr_func': str, 'data_range': list}. + """ + base_url = base_url.rstrip('/') + url_template = f'{base_url}/{webid_lookup_path}' + if '?' in url_template: + url_template = f'{url_template}&namefilter={{tag}}' + else: + url_template = f'{url_template}?namefilter={{tag}}' + + headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'X-Requested-With': 'piwebapistreams', + 'Authorization': f'Basic {auth_token}', + } + + web_ids: dict[str, dict[str, Any]] = {} + for idx, tag in enumerate(tag_names, start=1): + url = url_template.format(tag=tag) + resp = requests.get(url, headers=headers, timeout=API_TIMEOUT) + resp.raise_for_status() + data = resp.json() + items = data.get('Items', []) + if not items: + raise ValueError(f'No point found for tag: {tag}') + web_ids[tag] = { + 'webid': items[0]['WebId'], + 'aggr_func': 'lts', + 'data_range': [-100000, 100000], + } + print(f' Resolved tag {idx}/{len(tag_names)}: {tag}') + time.sleep(delay_seconds) + + return web_ids + + +# %% 3. Helper: load WebIds from JSON +def load_web_ids(json_path: str | None) -> dict[str, dict[str, Any]] | None: + """ + Load web_ids from a JSON file if path is provided. + + Args: + json_path: Path to JSON file with tag -> {webid, ...} structure. + + Return: + Loaded dict or None if json_path is None or file missing. + """ + if not json_path or not os.path.isfile(json_path): + return None + with open(json_path, encoding='utf-8') as f: + out = json.load(f) + print(f' Loaded {len(out)} web_ids from {json_path}') + return out + + +# %% 4. Helper: fetch values in chunks (async) +async def fetch_values_chunked( + web_ids: dict[str, dict[str, Any]], + period_days: int, + chunk_days: int, + base_url: str, + auth_token: str, + endpoint: str, + request_timeout: int, +) -> pd.DataFrame: + """ + Fetch historical values for web_ids over period_days in chunks of chunk_days. + + Args: + web_ids: Dict mapping tag name to at least {'webid': str}. + period_days: Total period to fetch (e.g. 180 for last 180 days). + chunk_days: Size of each time chunk in days (e.g. 30). + base_url: PI Web API base URL. + auth_token: Basic auth token. + endpoint: PI Web API endpoint (e.g. /streamsets/recorded). + request_timeout: Request timeout in seconds. + + Returns: + DataFrame with timestamp index and one column per tag (values). + """ + logger = MagicMock() + notification_handler = AsyncMock() + metrics_controller = AsyncMock() + + client = PIWebAPIClient( + base_url=base_url, + auth_config={'type': 'basic', 'token': auth_token}, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + headers_config={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'x-requested-with': 'piwebapistreams', + 'User-Agent': 'PiWebApiFetchData/1.0', + }, + ) + + metadata: dict[str, Any] = {} + chunks: list[pd.DataFrame] = [] + + try: + for i in range(period_days, 0, -chunk_days): + j = i - chunk_days + start_time_pi = f'*-{i}d' + end_time_pi = f'*-{j}d' if j > 0 else '*' + print(f' Fetching chunk: {start_time_pi} to {end_time_pi}') + df = await client.get_latest_values_df( + web_ids=web_ids, + endpoint=endpoint, + start_time=start_time_pi, + end_time=end_time_pi, + max_count=None, + request_timeout=request_timeout, + metadata=metadata, + ) + if not df.empty: + chunks.append(df) + print(f' Chunk size: {df.shape}') + print(f' Chunk sample: {df.head(3)}') + else: + print(f' Chunk size: 0 ') + await asyncio.sleep(1) + finally: + client.close() + + if not chunks: + return pd.DataFrame() + + data = pd.concat(chunks, ignore_index=True) + + print(f"Amount of names: {len(data['name'].unique())}") + + raw_count = len(data) + # data['timestamp'] = pd.to_datetime(data['timestamp'], utc=True).dt.floor('s') + data_timestamp_na = data[data['timestamp'].isna()] + + print(f"NA timestamp: {data_timestamp_na}") + + print(f"Amount of names: {len(data['name'].unique())}") + + data = data.sort_values('timestamp') + data = data.drop_duplicates(subset=['timestamp', 'name'], keep='last') + + print(f"Amount of names: {len(data['name'].unique())}") + + dedup_count = len(data) + print(f' Total raw rows: {raw_count}, after dedup: {dedup_count}') + + pivot = data.pivot(index='timestamp', columns='name', values='value') + pivot.sort_index(inplace=True) + print(f' Pivot shape: {pivot.shape} (index=timestamp, columns=tags)') + return pivot + + +# %% 5. Step: load or fetch WebIds (saved to WEB_IDS_SAVE_PATH after fetch for continuity) +print('Step 5: Load or fetch WebIds') +print(f' Trying WEB_IDS_SAVE_PATH={WEB_IDS_SAVE_PATH}') +web_ids = load_web_ids(WEB_IDS_SAVE_PATH) +if web_ids is None: + if not AUTH_TOKEN: + raise ValueError('Set AUTH_TOKEN at top to fetch WebIds.') + print(f'Fetching WebIds for {len(TAG_NAMES)} tags...') + web_ids = fetch_webids( + tag_names=TAG_NAMES, + base_url=BASE_URL, + auth_token=AUTH_TOKEN, + webid_lookup_path=WEBID_LOOKUP_PATH, + ) + print(f'Resolved {len(web_ids)} WebIds.') + with open(WEB_IDS_SAVE_PATH, 'w', encoding='utf-8') as f: + json.dump(web_ids, f, indent=4) + print(f'Saved web_ids to {WEB_IDS_SAVE_PATH} for continuity.') +else: + print(f'Loaded {len(web_ids)} WebIds from {WEB_IDS_SAVE_PATH}.') +web_ids + + +# %% 6. Step: fetch values in chunks +print('Step 6: Fetch values in chunks') +print(f' Period: {PERIOD_DAYS} days, chunk size: {CHUNK_DAYS} days, tags: {list(web_ids.keys())}') +df = run_async( + fetch_values_chunked( + web_ids=web_ids, + period_days=PERIOD_DAYS, + chunk_days=CHUNK_DAYS, + base_url=BASE_URL, + auth_token=AUTH_TOKEN, + endpoint=ENDPOINT, + request_timeout=API_TIMEOUT, + ) +) +if df.empty: + print(' Done. No data returned.') +else: + print(f' Done. Shape: {df.shape}, index range: {df.index.min()} to {df.index.max()}') +df + + +# %% 7. Step: inspect and optionally save +print('Step 7: Inspect and optionally save') +if df.empty: + print(' DataFrame is empty.') +else: + print(f' Shape: {df.shape}, columns: {list(df.columns)}') + print(f' Index (timestamp) range: {df.index.min()} to {df.index.max()}') +df.head() +df.to_csv('pi_web_api_data.csv') +# df.to_parquet('pi_web_api_data.parquet') + +# %% + +from pandas import read_csv, to_datetime + +data = read_csv('pi_web_api_data.csv') +data['timestamp'] = to_datetime(data['timestamp']) +print(data['timestamp'].min()) +print(data['timestamp'].max()) + +# %% +print(data.shape) +# %% diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..cc111b7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,159 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "scouter" +version = "0.0.0" +description = "Sientia DataOps Scouter - ML Model Orchestration System" +readme = "README.md" +requires-python = ">=3.11" +authors = [ + {name = "Aignosi", email = "dev@aignosi.com"} +] + +[tool.ruff] +line-length = 100 +target-version = "py311" +exclude = [ + ".git", + ".venv", + "venv", + "__pycache__", + "*.pyc", + ".pytest_cache", + "htmlcov", +] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "UP", # pyupgrade + "N", # pep8-naming + "YTT", # flake8-2020 + "S", # flake8-bandit + "BLE", # flake8-blind-except + "A", # flake8-builtins + "C90", # mccabe complexity +] + +ignore = [ + "BLE001", # ignore blind except, we need to send notifications with any error + "E501", # line too long (handled by formatter) + "S101", # use of assert (needed for tests) + "S105", # possible hardcoded password (false positives) + "S106", # possible hardcoded password (false positives) + "N802", # function name should be lowercase (temporal decorators) + "N806", # variable in function should be lowercase +] + +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = [ + "S101", # assert allowed in tests + "S105", # hardcoded passwords ok in tests + "S106", # hardcoded passwords ok in tests +] + +[tool.ruff.lint.mccabe] +max-complexity = 15 + +[tool.ruff.format] +quote-style = "single" +indent-style = "space" +line-ending = "auto" + +[tool.mypy] +python_version = "3.11" +warn_return_any = false +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = false +warn_no_return = true +strict_equality = true +ignore_missing_imports = true + +# Ignore missing imports for external packages +[[tool.mypy.overrides]] +module = "temporalio.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sientia_do.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mlflow.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "prometheus_client.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sientia.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pandas.*" +ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "-v", + "--strict-markers", +] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "session" +asyncio_default_test_loop_scope = "session" +markers = [ + "asyncio: marks tests as async", + "e2e: end-to-end tests against real backing services (Docker required)", + "integration: marks tests as integration tests", + "unit: marks tests as unit tests", +] + +[tool.coverage.run] +source = ["scouter"] +omit = [ + "*/tests/*", + "*/venv/*", + "*/__pycache__/*", + "*/site-packages/*", +] +branch = true + +[tool.coverage.report] +precision = 2 +show_missing = true +skip_covered = false +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "def __str__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", + "class .*\\bProtocol\\):", + "@(abc\\.)?abstractmethod", +] + +[tool.coverage.html] +directory = "htmlcov" + +[tool.bandit] +exclude_dirs = ["tests", "venv", ".venv"] +skips = ["B101", "B601"] # Skip assert and shell injection in controlled environments \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..3539fa9 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,23 @@ +# Development and Testing Dependencies +# These packages are only needed for development, testing, and code quality checks +# Install with: pip install -r requirements-dev.txt + +# Code Quality & Linting +ruff>=0.1.0 # Fast Python linter and formatter (replaces flake8, black, isort) +mypy>=1.7.0 # Static type checker +bandit>=1.7.5 # Security vulnerability scanner +pandas-stubs>=2.0.0 # Type stubs for pandas +types-requests>=2.31.0 # Type stubs for requests + +# Testing +pytest>=7.4.0 # Testing framework +pytest-cov>=4.1.0 # Coverage plugin for pytest +pytest-asyncio>=0.21.0 # Async test support (already in main requirements) + +# E2E Testing Dependencies +pytest-httpserver>=1.0.10 + +# Development Tools +ipython>=8.12.0 # Enhanced Python shell +ipdb>=0.13.13 # IPython debugger +testcontainers[postgres,mongodb,redis]>=4.0 \ No newline at end of file diff --git a/requirements-local.txt b/requirements-local.txt new file mode 100644 index 0000000..d1adfd1 --- /dev/null +++ b/requirements-local.txt @@ -0,0 +1,8 @@ +temporalio +psycopg2-binary +sqlalchemy +redis +pymongo +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.1 +prometheus-client +pycurl \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6470c2c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +temporalio +psycopg2-binary +sqlalchemy +redis +pymongo +sientia_do +prometheus-client +pycurl \ No newline at end of file diff --git a/run_coverage.sh b/run_coverage.sh new file mode 100755 index 0000000..dabc16d --- /dev/null +++ b/run_coverage.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Exit on any error +set -e + +echo "Activating virtual environment..." +source ./venv/bin/activate + +pytest --cov=scouter --cov-report=html + +xdg-open htmlcov/index.html \ No newline at end of file diff --git a/run_local.sh b/run_local.sh new file mode 100755 index 0000000..009279d --- /dev/null +++ b/run_local.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Exit on any error +set -e + +echo "Activating virtual environment..." +source ./venv/bin/activate + +echo "Loading environment variables from .env..." +if [ -f .env ]; then + export $(cat .env | grep -v '^#' | xargs) + echo "Environment variables loaded from .env" +else + echo "Warning: .env file not found. Continuing without environment variables." +fi + +echo "Starting scouter application..." +python -m scouter.worker.worker diff --git a/scouter/__init__.py b/scouter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scouter/activities/__init__.py b/scouter/activities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scouter/activities/activities.py b/scouter/activities/activities.py new file mode 100644 index 0000000..cb638a0 --- /dev/null +++ b/scouter/activities/activities.py @@ -0,0 +1,132 @@ +from sientia_do.observability.metrics_controller import MetricsController +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + from os import getenv + from typing import Any + + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.observability.logger import Logger + from sientia_do.temporal.activities.postgres_sync import Postgres + + from scouter.activities.api import API + from scouter.activities.gates import Gates + from scouter.activities.mongodb import MongoDB + from scouter.activities.redis import Redis + + +class Activities(Postgres, Redis, Gates, MongoDB, API): + """ + Unified activities class that combines multiple data processing services. + + This class provides a comprehensive interface for all data processing activities + by inheriting from specialized service classes. It handles: + - PostgreSQL operations for data persistence + - Redis operations for caching and temporary storage + - Data quality gates and filtering + - MongoDB operations for data retrieval + - PI Web API operations for external data ingestion + - Notification handling and logging + + The class implements the multiple inheritance pattern to provide a unified + interface while maintaining separation of concerns across different data services. + """ + + def __init__( + self, + postgres_config: dict[str, Any], + redis_config: dict[str, Any], + mongodb_config: dict[str, Any], + api_config: dict[str, Any], + logger: Logger, + notification_handler: NotificationHandler, + ): + """ + Initialize the Activities class with all required services. + + Args: + postgres_config (dict[str, Any]): PostgreSQL connection configuration. + Required fields: host, port, user, password, dbname, min_connections, max_connections + redis_config (dict[str, Any]): Redis connection configuration. + Required fields: host, port, username, password + mongodb_config (dict[str, Any]): MongoDB connection configuration. + Required fields: connection_string, database_name + api_config (dict[str, Any]): PI Web API configuration. + Required fields: base_url, auth_type, auth_token + logger (Logger): Logger instance for application logging + notification_handler (NotificationHandler): Handler for system notifications + """ + metrics_controller = MetricsController( + logger=logger, + ) + + # Initialize Postgres + Postgres.__init__( + self, + host=postgres_config['host'], + port=postgres_config['port'], + user=postgres_config['user'], + password=postgres_config['password'], + dbname=postgres_config['dbname'], + min_connections=postgres_config['min_connections'], + max_connections=postgres_config['max_connections'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + # Initialize Redis + Redis.__init__( + self, + host=redis_config['host'], + port=redis_config['port'], + logger=logger, + notification_handler=notification_handler, + username=redis_config['username'], + password=redis_config['password'], + metrics_controller=metrics_controller, + ) + + # Initialize Gates + Gates.__init__( + self, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + # Initialize MongoDB + MongoDB.__init__( + self, + connection_string=mongodb_config['connection_string'], + database_name=mongodb_config['database_name'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + # Initialize API + API.__init__( + self, + base_url=api_config['base_url'], + auth_type=api_config['auth_type'], + auth_token=api_config['auth_token'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + self.pod_id = getenv('HOSTNAME', 'localhost') + + def shutdown(self): + """ + Gracefully shutdown all service connections. + + This method ensures proper cleanup of database connections and resources + to prevent connection leaks and ensure graceful application termination. + """ + Postgres.close(self) + MongoDB.close(self) + Redis.close(self) + Gates.close(self) + API.close(self) diff --git a/scouter/activities/api.py b/scouter/activities/api.py new file mode 100644 index 0000000..161d3a6 --- /dev/null +++ b/scouter/activities/api.py @@ -0,0 +1,157 @@ +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + import traceback + from typing import Any + + from sientia_do.notifications.handlers import NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.observability.logger import Logger + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.repository.pi_web_api_client_sync import PIWebAPIClient + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ + + +class API(SientiaMonitoring): + """ + PI Web API operations for data retrieval. + + This class provides Temporal activities for interacting with the PI Web API + to retrieve tag values and historical data. It implements: + - Tag value retrieval from PI Web API endpoints + - Data quality filtering and validation + - Error handling with notifications + - Metrics collection for monitoring + + The class wraps the PIWebAPIClient to provide Temporal-aware activity methods + that can be used in workflow orchestration. + """ + + def __init__( + self, + base_url: str, + auth_type: str, + auth_token: str, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ) -> None: + """ + Initialize API activity with PI Web API client. + + Args: + base_url (str): Base URL of the PI Web API server + auth_type (str): Authentication type ('basic' or 'bearer') + auth_token (str): Authentication token + logger (Logger): Logger instance for operation logging + notification_handler (NotificationHandler): Handler for system notifications + metrics_controller (MetricsController): Controller for metrics collection + """ + SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) + self.pi_web_api_client = PIWebAPIClient( + base_url=base_url, + auth_config={ + 'type': auth_type, + 'token': auth_token, + }, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + headers_config={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'x-requested-with': 'piwebapistreams', + 'User-Agent': 'Aig-Scouter-Agent/1.0', + }, + ) + + def close(self) -> None: + """ + Close the PI Web API client and shutdown monitoring services. + + This method performs cleanup operations: + - Closes the PI Web API client connection + - Shuts down SientiaMonitoring services (metrics, notifications) + """ + self.pi_web_api_client.close() + SientiaMonitoring.shutdown(self) + + @activity.defn(name='get_tag_values') + def get_tag_values(self, input_data: dict[str, Any]) -> list[dict]: + """ + Retrieve tag values from PI Web API for specified WebIds. + + This activity fetches historical or real-time data from the PI Web API + for a set of configured tags. It returns the data as a list of dictionaries + suitable for further processing in the workflow. + + The timestamps are normalized to ensure consistency across all records in the + response. After converting timestamps to string format, all timestamps are + set to the maximum timestamp value (lexicographically) found in the dataset. + This ensures all records in a single batch share the same timestamp value. + + Args: + input_data (dict[str, Any]): Activity input parameters. + Required fields: + - metadata (dict[str, Any]): Workflow execution metadata + - endpoint (str): PI Web API endpoint path + - web_ids (dict[str, str | None]): Tag names mapped to WebIds + - period (dict[str, str]): Time period with 'start_time' field + - api_timeout (int): Request timeout in seconds + - max_count (int, optional): Maximum data points per tag. Defaults to 1 + + Returns: + list[dict]: List of data records, each containing: + - timestamp: Normalized timestamp string (all records share the same value) + - name: Tag name + - value: Numeric value + - tag: WebId + + Raises: + PIMSRequestError: If API request fails + Exception: If data retrieval or processing fails + """ + metadata = input_data['metadata'] + endpoint = input_data['endpoint'] + web_ids = input_data['web_ids'] + period = input_data['period'] + end_time = input_data.get('end_time', '*') + max_count = input_data.get('max_count', 1) + api_timeout = input_data['api_timeout'] + + self.info(f'Getting tag values from {endpoint}', metadata=metadata) + self.debug(f'Web IDs: {web_ids}', metadata=metadata) + try: + latest_values = self.pi_web_api_client.get_latest_values_df( + endpoint=endpoint, + web_ids=web_ids, + start_time=period, + end_time=end_time, + max_count=max_count, + metadata=metadata, + request_timeout=api_timeout, + ) + + except Exception as e: + self.send_notification( + metadata=metadata, + notification_id='PI_WEB_API_REQUEST_ERROR', + message=f'Error getting tag values from PI Web API: {e}', + block='get_tag_values', + level=NotificationLevel.ERROR, + attachment_content=traceback.format_exc(), + ) + raise e + + latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ) + + self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata) + + # Normalize the package timestamp + valid_timestamp_values = latest_values['timestamp'].dropna() + latest_values['timestamp'] = valid_timestamp_values.max() + + self.info(f'Gathered {len(latest_values)} tag values', metadata=metadata) + + return latest_values.to_dict(orient='records') diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py new file mode 100644 index 0000000..9040113 --- /dev/null +++ b/scouter/activities/gates.py @@ -0,0 +1,319 @@ +from collections.abc import Hashable + +from sientia_do.observability.metrics_controller import MetricsController +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + import traceback + from typing import Any + + from pandas import DataFrame + from sientia_do.notifications.handlers import NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.observability.logger import Logger + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + + from scouter import metrics + from scouter.utils.quality.filters import null_values_filter, out_of_bounds_filter + +quality_gate_filters = { + 'NULL_VALUES_FILTER': null_values_filter, + 'OUT_OF_BOUNDS_FILTER': out_of_bounds_filter, +} + + +class Gates(SientiaMonitoring): + """ + Data quality gates and filtering operations. + + This class implements data quality validation and filtering for industrial + time-series data. It provides: + - Configurable data quality filters + - Data aggregation functions for time-series data + - Comprehensive error handling and notification + - Metrics collection for quality monitoring + + The class supports multiple aggregation strategies and quality filters to + ensure data integrity and enable flexible data processing workflows. + """ + + def __init__( + self, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ): + """ + Initialize the Gates class with logging and notification services. + + Args: + logger (Logger): Logger instance for operation logging + notification_handler (NotificationHandler): Handler for system notifications + metrics_controller (MetricsController): Metrics controller instance + """ + SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) + + def close(self): + """ + Close the Gates class. + """ + SientiaMonitoring.shutdown(self) + + def apply_aggregation( + self, values: DataFrame, aggr_function: str, metadata: dict[str, Any] + ) -> float | None | str: + """ + Apply aggregation function to a group of time-series data. + + This method applies the specified aggregation function to a group of + data points. It handles edge cases and provides comprehensive error + reporting for invalid aggregation functions. + + Args: + values (DataFrame): Group of data points to aggregate (pre-sorted by timestamp) + aggr_function (str): Aggregation function to apply. + Supported functions: 'lts' (latest), 'avg' (average), 'mdn' (median), + 'max' (maximum), 'min' (minimum) + metadata (dict[str, Any]): Workflow metadata for error reporting + + Returns: + float | None | str: Aggregated value, None if no valid data, or 'continue' for errors + + Raises: + NotificationError: If invalid aggregation function is specified + """ + aggregation_map = { + 'lts': lambda x: x.iloc[-1], + 'avg': lambda x: x.mean(), + 'mdn': lambda x: x.median(), + 'max': lambda x: x.max(), + 'min': lambda x: x.min(), + } + + if aggr_function not in aggregation_map: + self.send_notification( + metadata=metadata, + notification_id='AGGREGATION_ISSUES', + message=f'Invalid aggregation function: {aggr_function}', + block='aggregate_data', + level=NotificationLevel.ERROR, + attachment_content=traceback.format_exc(), + ) + return 'continue' + + if len(values) == 1: + return values['value'].iloc[0] + + if aggr_function == 'lts': + return aggregation_map['lts'](values['value']) + + clean_values = values['value'].dropna() + + if clean_values.empty: + return None + + return aggregation_map[aggr_function](clean_values) + + @activity.defn(name='aggregate_data') + def aggregate_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]: + """ + Aggregate time-series data by tag and name using specified functions. + + This activity processes time-series data by grouping it by tag and name, + then applying the configured aggregation functions. It handles data + validation and provides comprehensive error reporting. + + Args: + input_data (dict[str, Any]): Activity input parameters. + Required fields: + - data (dict[str, Any]): Time-series data to aggregate + - model_tags (dict[str, Any]): Tag configuration with aggregation functions + + Returns: + dict[Hashable, Any]: Aggregated data organized by tag and name + + Raises: + Exception: If aggregation operation fails + """ + + metadata = input_data['metadata'] + + try: + # Convert input data to DataFrame + df = DataFrame(input_data['data']) + + self.info(f'Aggregating time series data for {len(df)} rows', metadata=metadata) + + # Sort once by timestamp for all data (more efficient than sorting each group) + df = df.sort_values(['tag', 'name', 'timestamp']) + + # Group by tag and name + # sort=False since we already sorted + grouped = df.groupby(['tag', 'name'], sort=False) + + # Prepare aggregation functions mapping + model_tags = input_data['model_tags'] + + # Process groups efficiently + results = [] + for (tag, name), group in grouped: + # Get the aggregation function from model_tags + aggr_function = model_tags.get(name, {}).get('aggr_func', 'lts') + + # Get the latest timestamp (last row since data is sorted) + latest_timestamp = group['timestamp'].iloc[-1] + + aggr_value = self.apply_aggregation(group, aggr_function, metadata) + + if aggr_value == 'continue': + continue + + # Store the result directly in list for better performance + results.append( + { + 'tag': tag, + 'name': name, + 'value': aggr_value, + 'timestamp': latest_timestamp, + 'aggregation_function': aggr_function, + } + ) + + self.info(f'Aggregated data has {len(results)} rows', metadata=metadata) + + # Convert to DataFrame only once at the end if we have results + if results: + result_df = DataFrame(results) + + self.debug(f'Final aggregated data:\n{result_df.to_string()}', metadata=metadata) + + return result_df.to_dict() + else: + # Return empty DataFrame dict structure + return DataFrame().to_dict() + + except Exception as e: + trace = traceback.format_exc() + + self.send_notification( + metadata=metadata, + notification_id='AGGREGATION_ISSUES', + message=f'Error aggregating data: {e}', + block='aggregate_data', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + + self.error(trace, metadata=metadata) + raise e + + @activity.defn(name='data_quality_gate') + def data_quality_gate(self, input_data: dict[str, Any]) -> dict[Hashable, Any]: + """ + Apply data quality filters to incoming data. + + This activity applies configurable quality filters to validate incoming + data. It supports multiple filter types and provides comprehensive + error reporting for quality issues. + + Args: + input_data (dict[str, Any]): Activity input parameters. + Required fields: + - data (dict[str, Any]): Data to validate + - filters (dict[str, str]): Filter configuration + - model_tags (dict[str, Any]): Tag-specific validation rules + + Returns: + dict[Hashable, Any]: Filtered data that passes quality validation + + Raises: + Exception: If quality validation fails + """ + + metadata = input_data['metadata'] + + filters = input_data['filters'] + data = DataFrame(input_data['data']) + model_tags = input_data['model_tags'] + + self.info(f'Applying quality gate to data to {len(data)} rows', metadata=metadata) + + tags = list(model_tags.keys()) + + data = data[data['name'].isin(tags)] + + for filter_name, config in filters.items(): + policy = config['policy'] + if filter_name not in quality_gate_filters: + self.warning(f'Filter {filter_name} not found', metadata=metadata) + continue + + try: + filtered_data = quality_gate_filters[filter_name](data, model_tags) + + except Exception as e: + trace = traceback.format_exc() + self.send_notification( + metadata=metadata, + notification_id='DATA_QUALITY_GATE_ISSUES', + message=f'Error applying filter {filter_name}: {e}', + block='data_quality_gate', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + + self.error(trace, metadata=metadata) + + else: + if filtered_data.empty: + continue + + message = f'{len(filtered_data)} rows has quality issues: {filter_name}: {policy}' + attachment = filtered_data.to_string() + + self.send_notification( + metadata=metadata, + notification_id=f'DATA_QUALITY_GATE_ISSUES__{filter_name}', + message=message, + block='data_quality_gate', + level=NotificationLevel.WARNING, + attachment_content=attachment, + ) + + if policy == 'DISCARD': + data = data[~data.index.isin(filtered_data.index)] + + self.info(f'Data quality gate applied, final data has {len(data)} rows', metadata=metadata) + + return data.to_dict() + + @activity.defn(name='write_metrics') + def write_metrics(self, input_data: dict[str, Any]) -> None: + """ + Write metrics to the database. + input_data: + metadata: dict[str, Any] + """ + metadata = input_data['metadata'] + tag_values = DataFrame(input_data['tag_values']) + + self.info(f'Writing metrics for {metadata["model_name"]}', metadata=metadata) + + metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels( + pod_id=self.pod_id, + model_name=metadata['model_name'], + workflow_name=metadata['workflow_name'], + ).inc() + + # Register metrics + for _, row in tag_values.iterrows(): + value = row['value'] + if value is not None: + metrics.TAG_CHANGES_MONITOR.labels( + pod_id=self.pod_id, + model_name=metadata['model_name'], + workflow_name=metadata['workflow_name'], + tag_name=row['variable'], + ).set(row['value']) + + self.info(f'Metrics written for {metadata["model_name"]}', metadata=metadata) diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py new file mode 100644 index 0000000..28f78f6 --- /dev/null +++ b/scouter/activities/mongodb.py @@ -0,0 +1,154 @@ +from datetime import UTC + +from sientia_do.observability.metrics_controller import MetricsController +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + import traceback + from datetime import datetime + from typing import Any + + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.observability.logger import Logger + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.repository.mongodb_repository_sync import MongoDBRepository + from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ + + +class MongoDB(SientiaMonitoring): + """ + MongoDB operations for data retrieval and storage. + + This class provides MongoDB connectivity and operations for the Scouter system. + It handles: + - Connection management with automatic reconnection + - Data retrieval with timestamp-based filtering + - Document cleaning and preprocessing + - Error handling and notification integration + + The class implements Temporal activities for MongoDB operations, enabling + distributed data processing with fault tolerance and monitoring. + """ + + def __init__( + self, + connection_string: str, + database_name: str, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ): + """ + Initialize MongoDB connection and services. + + Args: + connection_string (str): MongoDB connection URI string + database_name (str): Name of the target database + logger (Logger): Logger instance for operation logging + notification_handler (NotificationHandler): Handler for system notifications + + Raises: + ConnectionError: If MongoDB connection fails + """ + + self.mongodb_repository = MongoDBRepository( + connection_string=connection_string, + database_name=database_name, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + SientiaMonitoring.__init__( + self, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + def close(self): + """ + Close the MongoDB connection. + """ + self.mongodb_repository.close() + SientiaMonitoring.shutdown(self) + + def __del__(self): + """ + Destructor to ensure MongoDB client is closed. + + This destructor ensures that MongoDB connections are properly closed + when the object is garbage collected, preventing resource leaks. + """ + self.close() + + @activity.defn(name='load_latest_data') + def load_latest_data(self, input_data: dict[str, Any]) -> list[dict[str, Any]]: + """ + Load the latest data from MongoDB collection since a specified timestamp. + + This activity retrieves data from a MongoDB collection, optionally + filtering by timestamp to enable incremental data processing. It + handles connection management and provides comprehensive error reporting. + + Args: + input_data (dict[str, Any]): Activity input parameters. + Required fields: + - metadata (dict[str, Any]): Workflow execution metadata + - collection_name (str): Name of the MongoDB collection + - last_data_timestamp (str | None): Last processed timestamp for filtering + + Returns: + dict[str, Any]: Retrieved data, or empty dict if no data found + + Raises: + Exception: If MongoDB operation fails + """ + metadata = input_data['metadata'] + collection_name = input_data['collection_name'] + last_data_timestamp = input_data['last_data_timestamp'] + + self.info(f'Loading data from MongoDB: {input_data}', metadata=metadata) + + try: + if last_data_timestamp is None: + data_filter = {} + else: + data_filter = { + 'inserted_at': { + '$gt': datetime.strptime(last_data_timestamp, DATETIME_FORMAT_MS_WITH_TZ) + } + } + + self.debug(f'Data filter: {data_filter}', metadata=metadata) + + data = self.mongodb_repository.find( + collection_name=collection_name, + filters=data_filter, + metadata=metadata, + ) + + self.debug(f'Collected: {data}', metadata=metadata) + + for item in data: + item['inserted_at'] = ( + item['inserted_at'].replace(tzinfo=UTC).strftime(DATETIME_FORMAT_MS_WITH_TZ) + ) + + self.info(f'Loaded {len(data)} documents from MongoDB', metadata=metadata) + + self.debug(f'Loaded data: {data}', metadata=metadata) + + return data + except Exception as e: + trace = traceback.format_exc() + self.send_notification( + metadata=metadata, + notification_id='MONGO_LOAD_ERROR', + message=f'Error loading data from MongoDB: {e}', + block='load_latest_data', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + raise e diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py new file mode 100644 index 0000000..bd52de7 --- /dev/null +++ b/scouter/activities/redis.py @@ -0,0 +1,311 @@ +from sientia_do.observability.metrics_controller import MetricsController +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + import traceback + from collections.abc import Hashable + from typing import Any + + from pandas import DataFrame + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.observability.logger import Logger + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.repository.redis_repository_sync import RedisRepository + from sientia_do.temporal.constants import DATETIME_FORMAT, now + + +class Redis(SientiaMonitoring): + """ + Redis operations for data caching and temporary storage. + + This class extends the base Redis functionality to provide specialized + operations for the Scouter system, including: + - Data timestamp management for incremental processing + - Temporary data storage with configurable TTL + - Data grouping and holding for batch processing + - Error handling and notification integration + + The class implements Temporal activities for Redis operations, enabling + distributed data processing with fault tolerance and monitoring. + """ + + def __init__( + self, + host: str, + port: int, + username: str, + password: str, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ): + """ + Initialize Redis connection and services. + + Args: + host (str): Redis server hostname or IP address + port (int): Redis server port number + username (str): Redis authentication username + password (str): Redis authentication password + logger (Logger): Logger instance for operation logging + notification_handler (NotificationHandler): Handler for system notifications + """ + SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) + self.redis_repository = RedisRepository( + host=host, + port=port, + username=username, + password=password, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + def close(self): + """ + Close the Redis connection. + """ + self.redis_repository.close() + SientiaMonitoring.shutdown(self) + + @activity.defn(name='get_last_data_timestamp') + def get_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None: + """ + Retrieve the last processed data timestamp from Redis. + + This activity retrieves the timestamp of the last successfully processed + data point for a specific workflow and schedule combination. It's used + for incremental data processing to avoid reprocessing the same data. + + Args: + input_data (dict[str, Any]): Activity input parameters. + Required fields: + - metadata (dict[str, Any]): Workflow execution metadata + - workflow_name (str): Name of the workflow + - schedule_name (str): Name of the data collection schedule + + Returns: + str | None: Last processed timestamp string, or None if no previous data exists + + Raises: + Exception: If Redis operation fails + """ + metadata = input_data['metadata'] + key = f'last_data_timestamp:{input_data["workflow_name"]}:{input_data["schedule_name"]}' + + self.info(f'Getting last data timestamp for {key}', metadata=metadata) + + try: + data_hold = self.redis_repository.get(key, metadata=metadata) + except Exception as e: + self.send_notification( + metadata=metadata, + notification_id='REDIS_GET_ERROR', + message=f'Error getting last data timestamp: {e}', + block='get_last_data_timestamp', + level=NotificationLevel.ERROR, + attachment_content=traceback.format_exc(), + ) + raise e + + self.info(f'Last collected timestamp: {data_hold}', metadata=metadata) + + if not data_hold: + return None + + return data_hold + + @activity.defn(name='put_last_data_timestamp') + def put_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None: + """ + Store the last processed data timestamp in Redis. + + This activity stores the timestamp of the most recent data point that + has been successfully processed. The timestamp is used for incremental + data loading in subsequent workflow executions. + + Args: + input_data (dict[str, Any]): Activity input parameters. + Required fields: + - metadata (dict[str, Any]): Workflow execution metadata + - data (dict[str, Any]): Processed data to extract timestamp from + - workflow_name (str): Name of the workflow + - schedule_name (str): Name of the data collection schedule + + Returns: + str | None: The timestamp that was stored, or None if no data was processed + + Raises: + Exception: If Redis operation fails + """ + metadata = input_data['metadata'] + key = f'last_data_timestamp:{input_data["workflow_name"]}:{input_data["schedule_name"]}' + + self.info(f'Putting last data timestamp for {key}', metadata=metadata) + + data = DataFrame(input_data['data']) + + if data.empty: + self.warning('No data to insert', metadata=metadata) + return None + + last_data_timestamp = data['inserted_at'].max() + + self.info(f'Last collected timestamp to insert: {last_data_timestamp}', metadata=metadata) + + try: + self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5, metadata=metadata) + except Exception as e: + self.send_notification( + metadata=metadata, + notification_id='REDIS_SET_ERROR', + message=f'Error setting last data timestamp: {e}', + block='put_last_data_timestamp', + level=NotificationLevel.ERROR, + attachment_content=traceback.format_exc(), + ) + raise e + + return last_data_timestamp + + @activity.defn(name='group_and_hold_data') + def group_and_hold_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]: + """ + Group data by tags and store temporarily in Redis with TTL. + + This activity organizes processed data by tag names and stores it in Redis + with a configurable retention period. The data is grouped to enable + efficient batch processing and export operations. + + Args: + input_data (dict[str, Any]): Activity input parameters. + Required fields: + - metadata (dict[str, Any]): Workflow execution metadata + - schedule_name (str): Name of the data collection schedule + - workflow_name (str): Name of the workflow + - data (dict[str, Any]): Data to group and store + - model_id (str): Unique model identifier + - model_tags (dict[str, Any]): Tag configuration + - retention_time (int): Data retention period in seconds + + Returns: + dict[str, Any]: Grouped data organized by tag names + + Raises: + Exception: If Redis operation fails + """ + metadata = input_data['metadata'] + + self.debug('Grouping and holding data...', metadata=metadata) + data = DataFrame(input_data['data']) + model_tags = input_data['model_tags'] + retention_time = input_data['retention_time'] + fill_missing_tags = input_data['fill_missing_tags'] + + key = f'held_data_{input_data["workflow_name"]}_{input_data["schedule_name"]}' + + self.info(f'Getting held data for {key}', metadata=metadata) + + try: + data_hold = self.redis_repository.get(key, metadata=metadata) + except Exception as e: + self.send_notification( + metadata=metadata, + notification_id='REDIS_GET_ERROR', + message=f'Error getting held data: {e}', + block='group_and_hold_data', + level=NotificationLevel.ERROR, + attachment_content=traceback.format_exc(), + ) + raise e + + if not data_hold: + data_hold = {} + if data.empty: + self.warning('No data to export', metadata=metadata) + return data_hold + + self.info(f'Grouping and holding data for {len(data)} rows') + + try: + # Remove possibly removed tags + tags = list(model_tags.keys()) + tags.append('timestamp') + self.debug(f'Tags to keep: {tags}', metadata=metadata) + data_hold = {tag: content for tag, content in data_hold.items() if tag in tags} + self.debug(f'Data hold after removing removed tags: {data_hold}', metadata=metadata) + + for _, row in data.iterrows(): + value = row['value'] + + data_hold[row['name']] = value + + if fill_missing_tags: + self.debug('Filling missing tags in data package', metadata=metadata) + missing_tags = [tag for tag in tags if tag not in list(data_hold.keys())] + + for tag in missing_tags: + data_hold[tag] = None + + data_hold['timestamp'] = ( + data['timestamp'].max() if not data.empty else data_hold['timestamp'] + ) + + self.redis_repository.set(key, data_hold, ttl=retention_time, metadata=metadata) + + data_hold_df = DataFrame(data_hold, index=[0]) + data_hold_melted = data_hold_df.melt( + id_vars='timestamp', var_name='variable', value_name='value' + ) + data_hold_melted['model_id'] = input_data['model_id'] + + data_hold_melted.reset_index(drop=True, inplace=True) + except Exception as e: + self.send_notification( + metadata=metadata, + notification_id='REDIS_SET_ERROR', + message=f'Error setting held data: {e}', + block='group_and_hold_data', + level=NotificationLevel.ERROR, + attachment_content=traceback.format_exc(), + ) + raise e + + self.info(f'Data held and melted has {len(data_hold_melted)} rows') + + self.debug(f'Data held and melted:\n {data_hold_melted.to_string()}', metadata=metadata) + + return data_hold_melted.to_dict() + + @activity.defn(name='store_data_package') + def store_data_package(self, input_data: dict[str, Any]): + """ + Stores the data package in redis. It's a debug feature and must be toggled on. + input_data: + metadata: The metadata of the workflow. + workflow_name: The name of the workflow. + schedule_name: The name of the schedule. + held_data: The final scouter output. + data: The data used to collect the data. + """ + metadata = input_data['metadata'] + key = f'data_package_{input_data["workflow_name"]}_{input_data["schedule_name"]}_{now().strftime(DATETIME_FORMAT)}' + + data = DataFrame(input_data['data']) + held_data = DataFrame(input_data['held_data']) + + cache = {'data': data.to_dict(), 'held_data': held_data.to_dict()} + + try: + self.redis_repository.set(key, cache, ttl=120, metadata=metadata) + except Exception as e: + self.send_notification( + metadata=metadata, + notification_id='REDIS_SET_ERROR', + message=f'Error setting data package: {e}', + block='store_data_package', + level=NotificationLevel.ERROR, + attachment_content=traceback.format_exc(), + ) + raise e diff --git a/scouter/metrics.py b/scouter/metrics.py new file mode 100644 index 0000000..b25fa46 --- /dev/null +++ b/scouter/metrics.py @@ -0,0 +1,25 @@ +from prometheus_client import Counter, Gauge + +# Application health and status metrics +APP_UP = Gauge( + 'app_up', + 'Indicates if the application is running (1) or shutting down (0)', + ['pod_id'], +) + +# Core labels for consistent metric labeling +CORE_LABELS = ['pod_id', 'model_name', 'workflow_name'] + +# Data processing metrics +LABORIOUS_DATA_WRITTEN_COUNT = Counter( + 'scouter_laborious_data_written_count', + 'Number of writings to the database table laborious_data', + CORE_LABELS, +) + +# Tag monitoring metrics +TAG_CHANGES_MONITOR = Gauge( + 'scouter_tag_changes_monitor', + 'Current value change of each tag', + [*CORE_LABELS, 'tag_name'], +) diff --git a/scouter/utils/__init__.py b/scouter/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scouter/utils/connectors_config.py b/scouter/utils/connectors_config.py new file mode 100644 index 0000000..64737df --- /dev/null +++ b/scouter/utils/connectors_config.py @@ -0,0 +1,19 @@ +from os import getenv +from typing import Any + + +def build_kafka_config() -> dict[str, Any]: + """ + Build Kafka configuration from environment variables. + + Returns: + dict[str, Any]: Kafka configuration dictionary with keys: + - bootstrap_servers: Kafka broker addresses (default: localhost:9092) + - polling_time: Consumer polling interval in milliseconds (default: 1000) + - group_id: Consumer group identifier (default: scouter-group) + """ + return { + 'bootstrap_servers': getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092'), + 'polling_time': int(getenv('KAFKA_POLLING_TIME', '1000')), + 'group_id': 'scouter-group', + } diff --git a/scouter/utils/quality/filters.py b/scouter/utils/quality/filters.py new file mode 100644 index 0000000..5c466d7 --- /dev/null +++ b/scouter/utils/quality/filters.py @@ -0,0 +1,81 @@ +from typing import Any + +import numpy as np +from pandas import DataFrame + + +def check_data_range(value: float | int | None, val_range: list) -> bool: + """ + Check if a value falls outside the specified range. + + This function validates if a numeric value is within the acceptable range + defined by the minimum and maximum bounds. It handles edge cases including + None values and NaN values. + + Args: + value (float | int | None): The numeric value to validate + val_range (list): List containing [min_value, max_value] bounds + + Returns: + bool: True if value is outside the range, False if within range + + Note: + None and NaN values are considered out of range (return True) + """ + if value is None or np.isnan(value): + return True + + bottom = val_range[0] + up = val_range[-1] + + return value < bottom or value > up + + +def out_of_bounds_filter(df: DataFrame, model_tags: dict[str, Any]) -> DataFrame: + """ + Filter DataFrame rows where values are outside configured ranges. + + This function applies range validation to each row in the DataFrame based + on tag-specific configuration. Rows with values outside the configured + ranges are filtered out. + + Args: + df (DataFrame): DataFrame containing sensor data with 'name' and 'value' columns + model_tags (dict[str, Any]): Tag configuration containing data_range for each tag. + If a tag doesn't have data_range, it's considered to have infinite bounds. + + Returns: + DataFrame: Filtered DataFrame with out-of-bounds values removed + + Note: + Tags without data_range configuration are treated as having infinite bounds + """ + return df[ + df.apply( + lambda x: check_data_range( + x['value'], model_tags[x['name']].get('data_range', (-np.inf, np.inf)) + ), + axis=1, + ) + ] + + +def null_values_filter(df: DataFrame, _model_tags: dict[str, Any]) -> DataFrame: + """ + Filter DataFrame rows containing null values. + + This function removes rows where the 'value' column contains null values. + It's used for data quality filtering to ensure only complete data records + are processed. + + Args: + df (DataFrame): DataFrame containing sensor data with 'value' column + _model_tags (dict[str, Any]): Tag configuration (unused in this filter) + + Returns: + DataFrame: Filtered DataFrame with null values removed + + Note: + The _model_tags parameter is included for interface consistency but not used + """ + return df[df['value'].isnull()] diff --git a/scouter/worker/__init__.py b/scouter/worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py new file mode 100644 index 0000000..8ef61a3 --- /dev/null +++ b/scouter/worker/worker.py @@ -0,0 +1,187 @@ +from temporalio import client, workflow +from temporalio.runtime import PrometheusConfig, Runtime, TelemetryConfig + +with workflow.unsafe.imports_passed_through(): + import asyncio + import os + import sys + + from prometheus_client import start_http_server + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.observability.logger import get_logger + from sientia_do.temporal.worker.prepare_worker import prepare_worker + from sientia_do.utils.connectors_config import ( + build_api_config, + build_mongodb_config, + build_postgres_config, + build_redis_config, + ) + + from scouter import metrics + from scouter.activities.activities import Activities + from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter + from scouter.workflow.scouter import Scouter + from scouter.workflow.sub_workflows.core_scouter import CoreScouter + +# Environment configuration +POD_ID = os.getenv('HOSTNAME', 'localhost') +SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091')) + + +async def main(): + """ + Main entry point for the Scouter Temporal worker. + + This function initializes and starts all required services: + - Prometheus metrics server + - Notification handler for MongoDB + - Activity implementations for data processing + - Temporal client and workers + - Multiple task queues for different workflow types + + The worker supports two main task queues: + - scouter-queue: Main data processing workflows + - fake_data-queue: Test data generation workflows + + Returns: + None + + Raises: + SystemExit: If worker initialization or execution fails + """ + host = os.getenv('TEMPORAL_HOST', 'localhost:7233') + + logger = get_logger(__name__) + + metadata = { + 'pod_id': POD_ID, + 'model_name': '-', + 'model_id': '-', + 'workflow_name': '-', + 'schedule_name': '-', + } + + logger.custom_info(f'Starting Worker with pod_id: {POD_ID}', metadata) + + logger.custom_info('Starting prometheus client...', metadata) + start_prometheus_server() + + logger.custom_info('Starting Notification Handler...', metadata) + + mongo_config = build_mongodb_config() + notification_handler = NotificationHandler( + connection_string=mongo_config['connection_string'], + database=mongo_config['database_name'], + logger=logger, + project_name=os.getenv('PROJECT_NAME', 'scouter'), + ) + + logger.custom_info('Starting Activities...', metadata) + + activities = Activities( + logger=logger, + notification_handler=notification_handler, + postgres_config=build_postgres_config(), + redis_config=build_redis_config(), + mongodb_config=build_mongodb_config(), + api_config=build_api_config(), + ) + + logger.custom_info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata) + + new_runtime = Runtime( + telemetry=TelemetryConfig( + metrics=PrometheusConfig(bind_address=f'0.0.0.0:{SDK_METRICS_PORT}') + ) + ) + + logger.custom_info('Starting Temporal Client...', metadata) + + temporal_client = await client.Client.connect( + target_host=host, namespace=os.getenv('TEMPORAL_NAMESPACE', 'scouter'), runtime=new_runtime + ) + + logger.custom_info('Starting Workers...', metadata) + + workers = [ + prepare_worker( + temporal_client=temporal_client, + main_workflow=Scouter, + other_workflows=[CoreScouter], + activities=[ + activities.load_latest_data, + activities.get_last_data_timestamp, + activities.put_last_data_timestamp, + activities.data_quality_gate, + activities.aggregate_data, + activities.group_and_hold_data, + activities.export_data_to_postgres, + activities.write_metrics, + activities.store_data_package, + ], + logger=logger, + ), + prepare_worker( + temporal_client=temporal_client, + main_workflow=PIWebAPIScouter, + other_workflows=[CoreScouter], + activities=[ + activities.get_tag_values, + activities.data_quality_gate, + activities.aggregate_data, + activities.group_and_hold_data, + activities.export_data_to_postgres, + activities.write_metrics, + activities.store_data_package, + ], + logger=logger, + ), + ] + + handlers = [] + for w in workers: + handlers.append(w.run()) + + logger.custom_info('Workers started successfully', metadata) + + try: + await asyncio.gather(*handlers) + + except BaseException: # NOSONAR + logger.custom_error('An unhandled exception occurred: %s', metadata=metadata) + finally: + if notification_handler: + notification_handler.shutdown() + if activities: + activities.shutdown() + # Exit with a non-zero status code to indicate failure to Kubernetes + metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN + sys.exit(1) + + +def start_prometheus_server(): + """ + Start the Prometheus metrics HTTP server. + + This function initializes the Prometheus metrics server on the configured + port and sets the application health status. It's essential for + monitoring and observability of the Scouter system. + + Returns: + None + + Raises: + SystemExit: If metrics server fails to start + """ + try: + port = int(os.getenv('HTTP_METRICS_PORT', 9090)) + start_http_server(port) + print(f'Prometheus server started on port {port}.') + metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP + except Exception as e: + print(f'Failed to start Prometheus server: {e}') + os._exit(1) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/scouter/workflow/__init__.py b/scouter/workflow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scouter/workflow/pi_web_api_scouter.py b/scouter/workflow/pi_web_api_scouter.py new file mode 100644 index 0000000..d9d6d96 --- /dev/null +++ b/scouter/workflow/pi_web_api_scouter.py @@ -0,0 +1,108 @@ +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + from datetime import timedelta + from typing import Any + + from sientia_do.temporal.policies import retry_policy + + from scouter.activities.activities import Activities + + +@workflow.defn(name='pi_web_api_scouter') +class PIWebAPIScouter: + """ + PI Web API Scouter workflow that orchestrates data ingestion from PI systems. + + This workflow serves as the entry point for PI Web API data processing pipelines. + Unlike the standard Scouter that loads from MongoDB, this workflow directly queries + PI Web API endpoints to retrieve tag values and processes them for downstream use. + + The workflow implements a direct API ingestion pattern with: + - Real-time data retrieval from PI Web API + - Configurable time periods and data point limits + - Error handling and retry policies + - Child workflow orchestration for data processing + - Integration with CoreScouter for standardized processing + """ + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> None: + """ + Execute the PI Web API Scouter workflow. + + This method orchestrates the complete data ingestion process from PI Web API: + 1. Retrieves tag values from PI Web API using configured WebIds + 2. Validates and normalizes the retrieved data (timestamps are normalized) + 3. Delegates data processing to the CoreScouter workflow + + If no data is retrieved from the PI Web API, the workflow exits early without + invoking the CoreScouter workflow. + + Args: + input_data (dict[str, Any]): Configuration and parameters for the workflow execution. + Required fields: + - schedule_name (str): Unique identifier for the data collection schedule + - model_name (str): Name of the data model being processed + - model_id (str): Unique identifier for the data model + - pi_web_api_query (dict[str, Any]): PI Web API query configuration containing: + - endpoint (str): PI Web API endpoint path (e.g., '/streamsets/recorded') + - period (str): Time period configuration (e.g., '*-1d', '*-1h') + - api_timeout (int): Request timeout in seconds for PI Web API calls + - max_count (int, optional): Maximum data points per tag. Defaults to 1 + - trigger_laborious (bool): Flag to enable intensive data processing + - filters (dict[str, str]): Data quality filters configuration + - schema (str): Target database schema for data export + - table_name (str): Target table name for data export + - retention_time (int): Data retention period in Redis (seconds) + - model_tags (dict[str, Any]): Tag-specific configuration mapping tag names + to WebIds and processing rules, including: + - webid (str): PI Web API WebId for the tag + - data_range: [min, max] values for data validation + - aggr_func: Aggregation method (avg, mdn, max, min, lts) + - frequency: Data collection frequency in milliseconds + - topics: List of Kafka topics for data routing + + Returns: + None: This workflow doesn't return data, it orchestrates data processing + + Raises: + WorkflowExecutionError: If workflow execution fails + ActivityExecutionError: If any activity fails after retry attempts + PIMSRequestError: If PI Web API request fails + """ + + input_data['workflow_name'] = 'pi_web_api_scouter' + + metadata = { + 'metadata': { + 'model_id': input_data['model_id'], + 'model_name': input_data['model_name'], + 'schedule_name': input_data['schedule_name'], + 'workflow_name': input_data['workflow_name'], + } + } + + pi_web_api_query = input_data['pi_web_api_query'] + + data = await workflow.execute_local_activity_method( + Activities.get_tag_values, + { + **metadata, + 'endpoint': pi_web_api_query['endpoint'], + 'web_ids': input_data['model_tags'], + 'period': pi_web_api_query['period'], + 'max_count': pi_web_api_query.get('max_count', 1), + 'api_timeout': pi_web_api_query['api_timeout'], + }, + start_to_close_timeout=timedelta(seconds=60), + retry_policy=retry_policy, + ) + + if not data: + return + + input_data['data'] = data + input_data['metadata'] = metadata + + await workflow.execute_child_workflow('subworkflow.core_scouter', input_data) diff --git a/scouter/workflow/scouter.py b/scouter/workflow/scouter.py new file mode 100644 index 0000000..3ca3bbe --- /dev/null +++ b/scouter/workflow/scouter.py @@ -0,0 +1,120 @@ +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + from datetime import timedelta + from typing import Any + + from sientia_do.temporal.policies import retry_policy + + from scouter.activities.activities import Activities + + +@workflow.defn(name='scouter') +class Scouter: + """ + Main Scouter workflow that orchestrates data ingestion and processing. + + This workflow serves as the entry point for data processing pipelines. It loads + data from MongoDB collections, manages data timestamps for incremental processing, + and delegates the actual data processing to the CoreScouter workflow. + + The workflow implements a robust data ingestion pattern with: + - Incremental data loading based on last processed timestamp + - Automatic timestamp management for data continuity + - Error handling and retry policies + - Child workflow orchestration for data processing + """ + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> None: + """ + Execute the main Scouter workflow. + + This method orchestrates the complete data ingestion process: + 1. Retrieves the last processed timestamp from Redis + 2. Loads new data from MongoDB since the last timestamp using collection name + format: `raw_{schedule_name}` + 3. Updates the last processed timestamp with the most recent data point + 4. Delegates data processing to the CoreScouter workflow + + If no new data is found in MongoDB, the workflow exits early without updating + the timestamp or invoking the CoreScouter workflow. + + Args: + input_data (dict[str, Any]): Configuration and parameters for the workflow execution. + Required fields: + - topic (str): The Kafka topic name for data source identification + - schedule_name (str): Unique identifier for the data collection schedule + - model_name (str): Name of the data model being processed + - model_id (str): Unique identifier for the data model + - trigger_laborious (bool): Flag to enable intensive data processing + - filters (dict[str, str]): Data quality filters configuration + - schema (str): Target database schema for data export + - table_name (str): Target table name for data export + - retention_time (int): Data retention period in Redis (seconds) + - model_tags (dict[str, Any]): Tag-specific configuration including: + - data_range: [min, max] values for data validation + - aggr_function: Aggregation method (avg, mdn, max, min, lts) + - frequency: Data collection frequency in milliseconds + - topics: List of Kafka topics for data routing + + Returns: + None: This workflow doesn't return data, it orchestrates data processing + + Raises: + WorkflowExecutionError: If workflow execution fails + ActivityExecutionError: If any activity fails after retry attempts + """ + + input_data['workflow_name'] = 'scouter' + + metadata = { + 'metadata': { + 'model_id': input_data['model_id'], + 'model_name': input_data['model_name'], + 'schedule_name': input_data['schedule_name'], + 'workflow_name': input_data['workflow_name'], + } + } + + last_data_timestamp = await workflow.execute_local_activity_method( + Activities.get_last_data_timestamp, + { + **metadata, + 'workflow_name': input_data['workflow_name'], + 'schedule_name': input_data['schedule_name'], + }, + start_to_close_timeout=timedelta(seconds=60), + retry_policy=retry_policy, + ) + + data = await workflow.execute_local_activity_method( + Activities.load_latest_data, + { + **metadata, + 'collection_name': f'raw_{input_data["schedule_name"]}', + 'last_data_timestamp': last_data_timestamp, + }, + start_to_close_timeout=timedelta(seconds=60), + retry_policy=retry_policy, + ) + + if not data: + return + + await workflow.execute_activity_method( + Activities.put_last_data_timestamp, + { + **metadata, + 'data': data, + 'workflow_name': input_data['workflow_name'], + 'schedule_name': input_data['schedule_name'], + }, + start_to_close_timeout=timedelta(seconds=60), + retry_policy=retry_policy, + ) + + input_data['data'] = data + input_data['metadata'] = metadata + + await workflow.execute_child_workflow('subworkflow.core_scouter', input_data) diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py new file mode 100644 index 0000000..1daa235 --- /dev/null +++ b/scouter/workflow/sub_workflows/core_scouter.py @@ -0,0 +1,154 @@ +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + from datetime import timedelta + from typing import Any + + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ + from sientia_do.temporal.policies import retry_policy + + from scouter.activities.activities import Activities + + +@workflow.defn(name='subworkflow.core_scouter') +class CoreScouter: + """ + Core data processing workflow that handles data quality, aggregation, and export. + + This workflow implements the core data processing pipeline for industrial data: + - Data quality validation and filtering + - Time-series data aggregation using configurable functions + - Data grouping and temporary storage in Redis + - Asynchronous export to PostgreSQL for persistent storage + - Metrics collection and monitoring + + The workflow is designed for high-throughput data processing with configurable + quality gates and aggregation strategies. It is typically invoked as a child + workflow by parent workflows such as Scouter or PIWebAPIScouter. + """ + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> None: + """ + Execute the core data processing workflow. + + This method processes industrial time-series data through a series of stages: + 1. Data Quality Gate: Applies configurable filters for data validation + 2. Data Aggregation: Groups and aggregates data using specified functions + 3. Data Grouping: Organizes data by tags and applies retention policies + 4. Data Export: Persists processed data to PostgreSQL with timestamp conversion + 5. Metrics Collection: Records processing metrics for monitoring + + The workflow implements early exit conditions: + - If held_data is empty after grouping, the workflow exits without exporting + - If data export results in zero or negative affected_rows, the workflow exits + without writing metrics or storing debug packages + + Args: + input_data (dict[str, Any]): Complete workflow configuration and data. + Required fields: + - metadata (dict[str, Any]): Workflow execution metadata + - workflow_name (str): Name of the parent workflow + - schedule_name (str): Data collection schedule identifier + - model_name (str): Data model name + - model_id (str): Unique model identifier + - data (dict[str, Any]): Raw time-series data to process + - trigger_laborious (bool): Enable intensive processing mode + - filters (dict[str, str]): Data quality filter configurations + - schema (str): Target database schema + - table_name (str): Target database table + - retention_time (int): Redis data retention period (seconds) + - model_tags (dict[str, Any]): Tag-specific processing rules + - fill_missing_tags (bool): Enable filling of missing tag values + - debug_data_package (bool, optional): Store data packages for debugging. + When True, stores both raw and processed data in MongoDB for debugging + + Returns: + None: This workflow processes data but doesn't return results + + Raises: + WorkflowExecutionError: If workflow execution fails + ActivityExecutionError: If any activity fails after retry attempts + """ + + metadata = input_data['metadata'] + + filtered_data = await workflow.execute_local_activity_method( + Activities.data_quality_gate, + { + **metadata, + 'filters': input_data['filters'], + 'data': input_data['data'], + 'model_tags': input_data['model_tags'], + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60), + ) + + grouped_data = await workflow.execute_local_activity_method( + Activities.aggregate_data, + {**metadata, 'data': filtered_data, 'model_tags': input_data['model_tags']}, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60), + ) + + held_data = await workflow.execute_local_activity_method( + Activities.group_and_hold_data, + { + **metadata, + 'schedule_name': input_data['schedule_name'], + 'workflow_name': input_data['workflow_name'], + 'data': grouped_data, + 'model_id': input_data['model_id'], + 'model_tags': input_data['model_tags'], + 'retention_time': input_data['retention_time'], + 'fill_missing_tags': input_data['fill_missing_tags'], + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60), + ) + + if held_data == {}: + return + + data_exported = await workflow.execute_activity_method( + Activities.export_data_to_postgres, + { + **metadata, + 'schema': input_data['schema'], + 'table_name': input_data['table_name'], + 'data': held_data, + 'on_conflict': 'ignore', + 'unique_columns': ['model_id', 'timestamp', 'variable'], + 'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ}, + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60), + ) + + if data_exported.get('affected_rows', 0) <= 0: + return + + await workflow.execute_activity_method( + Activities.write_metrics, + { + **metadata, + 'tag_values': held_data, + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60), + ) + + if input_data.get('debug_data_package', False): + await workflow.execute_activity_method( + Activities.store_data_package, + { + **metadata, + 'data': input_data['data'], + 'held_data': held_data, + 'workflow_name': input_data['workflow_name'], + 'schedule_name': input_data['schedule_name'], + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60), + ) diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..e16b501 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,11 @@ +sonar.projectKey=Aignosi_sientia-dataops-scouter_temporal_82f6f501-b2d4-45ec-b2bc-d8fdc1a8a06d +sonar.projectName=sientia-dataops-scouter_temporal +sonar.sources=scouter +sonar.tests=tests +sonar.qualitygate.wait=true +sonar.qualitygate.timeout=300 +sonar.python.coverage.reportPaths=coverage.xml +sonar.coverage.exclusions=scouter/worker/worker.py +sonar.python.xunit.reportPath=pytest.xml +sonar.python.version=3.11 +sonar.projectVersion=1.0.0 diff --git a/tests.ipynb b/tests.ipynb new file mode 100644 index 0000000..24e9fb3 --- /dev/null +++ b/tests.ipynb @@ -0,0 +1,2433 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "9d16b24a", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import timedelta\n", + "from typing import Any\n", + "from temporalio import client\n", + "from temporalio.client import WorkflowHandle\n", + "\n", + "\n", + "async def start_workflow_advanced(\n", + " temporal_client: client.Client,\n", + " workflow_name: str,\n", + " workflow_input: dict[str, Any],\n", + " workflow_id: str,\n", + " task_queue: str,\n", + " execution_timeout: timedelta | None = None,\n", + " run_timeout: timedelta | None = None,\n", + " task_timeout: timedelta | None = None,\n", + ") -> WorkflowHandle:\n", + " handle = await temporal_client.start_workflow(\n", + " workflow=workflow_name,\n", + " arg=workflow_input,\n", + " id=workflow_id or f\"{workflow_name}-{id(workflow_input)}\",\n", + " task_queue=task_queue,\n", + " execution_timeout=execution_timeout,\n", + " run_timeout=run_timeout,\n", + " task_timeout=task_timeout,\n", + " )\n", + " \n", + " return handle" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e344fb0", + "metadata": {}, + "outputs": [], + "source": [ + "input_data = {\n", + " \"debug_data_package\": False,\n", + " \"execution_timeout_seconds\": 300,\n", + " \"fill_missing_tags\": False,\n", + " \"filters\": {\n", + " \"NULL_VALUES_FILTER\": {\n", + " \"policy\": \"DISCARD\"\n", + " },\n", + " \"OUT_OF_BOUNDS_FILTER\": {\n", + " \"policy\": \"DISCARD\"\n", + " }\n", + " },\n", + " \"frequency\": \"30s\",\n", + " \"max_retry_policy\": 1,\n", + " \"model_config\": {\n", + " \"predict_flavor\": \"sklearn\",\n", + " \"retention_minutes\": 0,\n", + " \"target\": \"CI-W3A05F1\",\n", + " \"transform_flavor\": \"sklearn\"\n", + " },\n", + " \"model_id\": \"10\",\n", + " \"model_name\": \"Pi Web API Test Model\",\n", + " \"model_tags\": {\n", + " \"CI-W3W03S1\": {\n", + " \"aggr_func\": \"avg\",\n", + " \"data_range\": [\n", + " -100000,\n", + " 100000\n", + " ],\n", + " \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujATFUAAAUElIQVZDXENJLVczVzAzUzE\"\n", + " },\n", + " \"CI-W3A05F1\": {\n", + " \"aggr_func\": \"lts\",\n", + " \"data_range\": [\n", + " -100000,\n", + " 100000\n", + " ],\n", + " \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE\"\n", + " }\n", + " },\n", + " \"pi_web_api_query\": {\n", + " \"endpoint\": \"/streamsets/recorded\",\n", + " \"period\": \"*-1d\",\n", + " \"max_count\": 1,\n", + " \"api_timeout\": 5\n", + " },\n", + " \"offset\": \"0m\",\n", + " \"retention_time\": 3600,\n", + " \"schedule_name\": \"pi-web-api-scouter-test\",\n", + " \"schema\": \"sientia_data\",\n", + " \"table_name\": \"laborious_data\",\n", + " \"task_timeout_seconds\": 300,\n", + " \"trigger_laborious\": False,\n", + " \"updated_at\": \"2025-08-13 18:35:01.600000+0000\",\n", + " \"workflow_type\": \"pi_web_api_scouter\"\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9350bff3", + "metadata": {}, + "outputs": [], + "source": [ + "from temporalio import client\n", + "\n", + "temporal_client = await client.Client.connect(\n", + " target_host=\"localhost:7233\",\n", + " namespace=\"scouter\"\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45712d7a", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "import datetime\n", + "\n", + "now = datetime.datetime.now()\n", + "\n", + "handle = await start_workflow_advanced(\n", + " temporal_client=temporal_client,\n", + " workflow_name='pi_web_api_scouter',\n", + " workflow_input=input_data,\n", + " workflow_id='test_workflow_id_' + now.strftime('%Y%m%d%H%M%S'),\n", + " task_queue='pi-web-api-scouter-queue',\n", + " execution_timeout=timedelta(seconds=30),\n", + " run_timeout=timedelta(seconds=30),\n", + " task_timeout=timedelta(seconds=30),\n", + ")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d065d0de", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "def camel_to_kebab(text: str) -> str:\n", + " \"\"\"Convert camelCase or PascalCase to kebab-case.\"\"\"\n", + " text = re.sub('(.)([A-Z][a-z]+)', r'\\1-\\2', text)\n", + " text = re.sub('([a-z0-9])([A-Z])', r'\\1-\\2', text)\n", + " return text.lower()\n", + "\n", + "print(camel_to_kebab('PiWebApiScouter'))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72af4236", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from time import sleep\n", + "\n", + "# Obter web id das seguintes tags:\n", + "TAG_NAMES = [\n", + " \"CI-W3A05F1\",\n", + " \"CI-W3W03S1\",\n", + " \"CI-W3W03I1\",\n", + " \"CI-W3K01T1\",\n", + " \"CI-W3W01A3\",\n", + " \"CI-W3W01A2\",\n", + " \"CI-W3W01A1\",\n", + " \"CI-J3P01T1A\",\n", + " \"CI-W3A50T1\",\n", + " \"CI-W3A55T1\",\n", + " \"CI-W3A55P1\",\n", + " \"CI-W3V33P1\",\n", + " \"CI-W3E01F1\",\n", + " \"CI-W3A50A3\",\n", + " \"CI-W3A50A2\",\n", + " \"CI-W3A50A1\",\n", + " \"CI-W3A50P1\",\n", + " \"CI-W3W01P1\",\n", + " \"CI-W3A71P1\",\n", + " \"CI-W3W01P2\",\n", + " \"CI-W3A71P2\",\n", + " \"CI-W3A71P3\",\n", + " \"CI-J3J01S1\",\n", + " \"CI-W3P17S1\",\n", + " \"CI-J3P03S1\",\n", + " \"CI-W3K01S1\",\n", + " \"CI-W3K14P1\",\n", + " \"CI-W3K01T4\",\n", + " \"CI-W3K01T2\",\n", + " \"CI-W3A65_SO3\",\n", + " \"CI-W3A65_Cl\",\n", + " \"CI-W3_C3S\",\n", + " \"CI-W3_MS\",\n", + " \"CI-W3_MA\",\n", + " \"CI-W3_PL\",\n", + " \"CI-W3_CAO\",\n", + " \"CI-W3V04P3\",\n", + " \"CI-W3V04P1\",\n", + " \"CI-W3W01G1\",\n", + " \"CI-W3V21F1\",\n", + " \"CI-W3V21P1\",\n", + " \"CI-W3V30F1\",\n", + " \"CI-W3V33P1\",\n", + " \"CI-W3W01A1_AI\",\n", + " \"CI-W3W01A2_AI\"\n", + "]\n", + "url = 'https://pivision.votorantimcimentos.com/piwebapi/dataservers/F1DS-7fYgsRTtUOa7V9NIwSujAUElIQVZD/points?namefilter={tag}'\n", + "\n", + "headers = {\n", + " 'Content-Type': 'application/json',\n", + " 'Accept': 'application/json',\n", + " 'X-Requested-With': 'piwebapistreams', # Header recomendado pelo PI Web API\n", + 'Authorization': "Basic " + __import__('os').environ.get('PI_WEB_API_BASIC_AUTH', '') + "}\n", + "\n", + "web_ids = {}\n", + "\n", + "for tag in TAG_NAMES:\n", + " response = requests.get(url.replace('{tag}', tag), headers=headers).json()\n", + " web_ids[tag] = {\n", + " 'webid': response['Items'][0]['WebId'],\n", + " 'aggr_func': 'lts',\n", + " 'data_range': [-100000, 100000],\n", + " }\n", + " sleep(0.5)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55793801", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "json.dump(web_ids, open('web_ids.json', 'w'), indent=4)\n", + "web_ids" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a83b3cb4", + "metadata": {}, + "outputs": [], + "source": [ + "from scouter.activities.api import API\n", + "from unittest.mock import MagicMock, AsyncMock\n", + "from pandas import DataFrame, concat\n", + "import json\n", + "from time import sleep\n", + "\n", + "web_ids = json.load(open('web_ids.json'))\n", + "\n", + "api = API(\n", + " base_url='https://pivision.votorantimcimentos.com/piwebapi',\n", + " auth_type='basic',\n", + auth_token=__import__('os').environ.get('PI_WEB_API_BASIC_AUTH', ''), + " logger=MagicMock(),\n", + " notification_handler=AsyncMock(),\n", + " metrics_controller=AsyncMock(),\n", + ")\n", + "\n", + "start_time = 1800\n", + "pace = 30\n", + "\n", + "data = DataFrame()\n", + "\n", + "for i in range(start_time, 0, -pace):\n", + " j = i - pace\n", + " print(f'Getting data for chunk -{i} to -{j} days')\n", + " try:\n", + " chunk = DataFrame(api.get_tag_values(\n", + " input_data={\n", + " 'endpoint': '/streamsets/recorded',\n", + " 'web_ids': web_ids,\n", + " 'period': f'*-{i}d',\n", + " 'end_time': f'*-{j}d'.replace('-0d', ''),\n", + " 'max_count': None,\n", + " 'api_timeout': 5,\n", + " 'metadata': {}\n", + " }\n", + " ))\n", + " except Exception as e:\n", + " chunk = DataFrame(api.get_tag_values(\n", + " input_data={\n", + " 'endpoint': '/streamsets/recorded',\n", + " 'web_ids': web_ids,\n", + " 'period': f'*-{i}d',\n", + " 'end_time': f'*-{j}d'.replace('-0d', ''),\n", + " 'max_count': None,\n", + " 'api_timeout': 5,\n", + " 'metadata': {}\n", + " }\n", + " ))\n", + " sleep(1)\n", + "\n", + " data = concat([data, chunk])\n", + "\n", + "base_data = data\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e2ef295", + "metadata": {}, + "outputs": [], + "source": [ + "trunc_spec = 's'\n", + "\n", + "base_data['truncated_timestamp'] = base_data['timestamp'].dt.floor(trunc_spec)\n", + "\n", + "# drop timestamp NaT\n", + "base_data = base_data[base_data['timestamp'].notna()]\n", + "base_data.sort_values(by='truncated_timestamp', inplace=True)\n", + "\n", + "display(base_data)\n", + "\n", + "base_data.drop_duplicates(subset=['truncated_timestamp', 'name'], keep='last', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae7b5889", + "metadata": {}, + "outputs": [], + "source": [ + "data = base_data.pivot(index='truncated_timestamp', columns='name', values='value')\n", + "\n", + "data.sort_index(inplace=True)\n", + "\n", + "# replace Nan with upper row value\n", + "data.ffill(inplace=True)\n", + "#data.bfill(inplace=True)\n", + "data.dropna(inplace=True)\n", + "\n", + "data['timestamp'] = data.index\n", + "\n", + "data.reset_index(drop=True, inplace=True)\n", + "\n", + "display(data)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4782290f", + "metadata": {}, + "outputs": [], + "source": [ + "to_insert_data = data.melt(id_vars=['timestamp'], var_name='variable', value_name='value')\n", + "to_insert_data['model_id'] = '111'\n", + "to_insert_data.drop_duplicates(subset=['timestamp', 'variable'], keep='last', inplace=True)\n", + "to_insert_data = to_insert_data.sort_values(by='timestamp', ascending=False)\n", + "to_insert_data.to_csv('data.csv', index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "a36d48d1", + "metadata": {}, + "outputs": [], + "source": [ + "from sientia_do.temporal.activities.postgres import Postgres\n", + "from unittest.mock import MagicMock, AsyncMock\n", + "\n", + "postgres_interface = Postgres(\n", + " host='localhost',\n", + " port=5432,\n", + " dbname='sientia',\n", + " user='sientia',\n", + " password='sientia',\n", + " min_connections=1,\n", + " max_connections=200,\n", + " logger=MagicMock(),\n", + " notification_handler=AsyncMock(),\n", + " metrics_controller=AsyncMock(),\n", + ")\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "713ecb2e", + "metadata": {}, + "outputs": [], + "source": [ + "from pandas import read_csv\n", + "from time import sleep\n", + "to_insert_data = read_csv('data.csv')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f269b8e7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 0 to 100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 100000 to 200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 200000 to 300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 300000 to 400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 400000 to 500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 500000 to 600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 600000 to 700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 700000 to 800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 800000 to 900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 900000 to 1000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1000000 to 1100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1100000 to 1200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1200000 to 1300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1300000 to 1400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1400000 to 1500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1500000 to 1600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1600000 to 1700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1700000 to 1800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1800000 to 1900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1900000 to 2000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2000000 to 2100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2100000 to 2200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2200000 to 2300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2300000 to 2400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2400000 to 2500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2500000 to 2600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2600000 to 2700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2700000 to 2800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2800000 to 2900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2900000 to 3000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3000000 to 3100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3100000 to 3200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3200000 to 3300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3300000 to 3400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3400000 to 3500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3500000 to 3600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3600000 to 3700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3700000 to 3800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3800000 to 3900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3900000 to 4000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4000000 to 4100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4100000 to 4200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4200000 to 4300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4300000 to 4400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4400000 to 4500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4500000 to 4600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4600000 to 4700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4700000 to 4800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4800000 to 4900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4900000 to 5000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5000000 to 5100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5100000 to 5200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5200000 to 5300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5300000 to 5400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5400000 to 5500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5500000 to 5600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5600000 to 5700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5700000 to 5800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5800000 to 5900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5900000 to 6000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6000000 to 6100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6100000 to 6200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6200000 to 6300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6300000 to 6400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6400000 to 6500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6500000 to 6600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6600000 to 6700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6700000 to 6800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6800000 to 6900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6900000 to 7000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7000000 to 7100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7100000 to 7200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7200000 to 7300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7300000 to 7400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7400000 to 7500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7500000 to 7600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7600000 to 7700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7700000 to 7800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7800000 to 7900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7900000 to 8000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8000000 to 8100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8100000 to 8200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8200000 to 8300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8300000 to 8400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8400000 to 8500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8500000 to 8600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8600000 to 8700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8700000 to 8800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8800000 to 8900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8900000 to 9000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9000000 to 9100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9100000 to 9200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9200000 to 9300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9300000 to 9400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9400000 to 9500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9500000 to 9600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9600000 to 9700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9700000 to 9800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9800000 to 9900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9900000 to 10000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10000000 to 10100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10100000 to 10200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10200000 to 10300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10300000 to 10400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10400000 to 10500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10500000 to 10600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10600000 to 10700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10700000 to 10800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10800000 to 10900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10900000 to 11000000\n" + ] + } + ], + "source": [ + "\n", + "chunk_start = 10900000\n", + "chunk_end = 20000000\n", + "\n", + "chunk_size = 100000\n", + "\n", + "chunk_pace = chunk_start\n", + "while chunk_pace < chunk_end:\n", + " print(f'Inserting chunk {chunk_pace} to {chunk_pace+chunk_size}')\n", + " chunk = to_insert_data.iloc[chunk_pace:chunk_pace+chunk_size]\n", + " await postgres_interface.export_data_to_postgres(\n", + " input_data={\n", + " 'data': chunk,\n", + " 'table_name': 'laborious_data',\n", + " 'schema': 'sientia_data',\n", + " 'unique_columns': ['timestamp', 'variable', 'model_id'],\n", + " 'on_conflict': 'ignore',\n", + " 'metadata': {}\n", + " }\n", + " )\n", + " sleep(10)\n", + " chunk_pace += chunk_size" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e0e3d5a", + "metadata": {}, + "outputs": [], + "source": [ + "item = web_ids['CI-W3A05F1']['webid']\n", + "\n", + "requests.get(\n", + " f'https://pivision.votorantimcimentos.com/piwebapi/streamsets/recorded',\n", + " params={\n", + " 'webid': item,\n", + " 'startTime': '*-1d',\n", + " 'endtime': '*',\n", + " \"maxCount\": 1,\n", + " },\n", + " headers=headers\n", + ").json()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8c425e2", + "metadata": {}, + "outputs": [], + "source": [ + "from pandas import read_csv, to_datetime\n", + "\n", + "df = read_csv('/home/grezewave/Downloads/laborious_data_202512220821.csv')\n", + "data = df.pivot(index='timestamp', columns='variable', values='value')\n", + "data['timestamp'] = data.index\n", + "\n", + "#Remove tz from timestamp\n", + "data['timestamp'] = to_datetime(data['timestamp'])\n", + "data['timestamp'] = data['timestamp'].dt.tz_localize(None)\n", + "\n", + "# Back to string and add \"\"\n", + "data['timestamp'] = data['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')\n", + "data.reset_index(drop=True, inplace=True)\n", + "data.to_csv('VC-model-data.csv', index=False)\n", + "\n", + "display(data)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "2bd5569d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
model_idvariablevaluetimestampcreated_at
0111CI-W3V30F18.5148512026-01-20 19:41:23+00002026-01-20 23:28:18+0000
1111CI-W3A71P3-2.3086902026-01-20 19:41:23+00002026-01-20 23:28:18+0000
2111CI-J3P01T1A228.9000002026-01-20 19:41:23+00002026-01-20 23:28:18+0000
3111CI-W3W03I175.3674242026-01-20 19:41:23+00002026-01-20 23:28:18+0000
4111CI-J3J01S187.9990202026-01-20 19:41:23+00002026-01-20 23:28:18+0000
..................
9993055111CI-W3V30F16.8461082025-01-21 11:35:08+00002026-01-21 00:18:50+0000
9993056111CI-W3K14P138.0967702025-01-21 11:35:08+00002026-01-21 00:18:50+0000
9993057111CI-W3A50T1364.2500002025-01-21 11:35:08+00002026-01-21 00:18:50+0000
9993058111CI-W3A55T1871.6823002025-01-21 11:35:08+00002026-01-21 00:18:50+0000
9993059111CI-W3W01P1-2.3568292025-01-21 11:35:08+00002026-01-21 00:18:50+0000
\n", + "

9993060 rows Γ— 5 columns

\n", + "
" + ], + "text/plain": [ + " model_id variable value timestamp \\\n", + "0 111 CI-W3V30F1 8.514851 2026-01-20 19:41:23+0000 \n", + "1 111 CI-W3A71P3 -2.308690 2026-01-20 19:41:23+0000 \n", + "2 111 CI-J3P01T1A 228.900000 2026-01-20 19:41:23+0000 \n", + "3 111 CI-W3W03I1 75.367424 2026-01-20 19:41:23+0000 \n", + "4 111 CI-J3J01S1 87.999020 2026-01-20 19:41:23+0000 \n", + "... ... ... ... ... \n", + "9993055 111 CI-W3V30F1 6.846108 2025-01-21 11:35:08+0000 \n", + "9993056 111 CI-W3K14P1 38.096770 2025-01-21 11:35:08+0000 \n", + "9993057 111 CI-W3A50T1 364.250000 2025-01-21 11:35:08+0000 \n", + "9993058 111 CI-W3A55T1 871.682300 2025-01-21 11:35:08+0000 \n", + "9993059 111 CI-W3W01P1 -2.356829 2025-01-21 11:35:08+0000 \n", + "\n", + " created_at \n", + "0 2026-01-20 23:28:18+0000 \n", + "1 2026-01-20 23:28:18+0000 \n", + "2 2026-01-20 23:28:18+0000 \n", + "3 2026-01-20 23:28:18+0000 \n", + "4 2026-01-20 23:28:18+0000 \n", + "... ... \n", + "9993055 2026-01-21 00:18:50+0000 \n", + "9993056 2026-01-21 00:18:50+0000 \n", + "9993057 2026-01-21 00:18:50+0000 \n", + "9993058 2026-01-21 00:18:50+0000 \n", + "9993059 2026-01-21 00:18:50+0000 \n", + "\n", + "[9993060 rows x 5 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from pandas import read_parquet\n", + "\n", + "df = read_parquet('~/Downloads/data_2026-01-21_11-32-10.parquet')\n", + "\n", + "display(df)\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/activities/__init__.py b/tests/activities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/activities/test_activities.py b/tests/activities/test_activities.py new file mode 100644 index 0000000..087655b --- /dev/null +++ b/tests/activities/test_activities.py @@ -0,0 +1,194 @@ +import inspect +from unittest.mock import ANY, MagicMock, patch + +from sientia_do.temporal.activities.postgres_sync import Postgres + +from scouter.activities.activities import Activities +from scouter.activities.api import API +from scouter.activities.gates import Gates +from scouter.activities.mongodb import MongoDB +from scouter.activities.redis import Redis + + +@patch('scouter.activities.activities.MongoDB.__init__') +@patch('scouter.activities.activities.Postgres.__init__') +@patch('scouter.activities.activities.Redis.__init__') +@patch('scouter.activities.activities.Gates.__init__') +@patch('scouter.activities.activities.API.__init__') +@patch('scouter.activities.activities.MetricsController') +def test___init__( + mock_metrics_controller, + mock_api_init, + mock_gates_init, + mock_redis_init, + mock_postgres_init, + mock_mongodb_init, +): + postgres_config = { + 'host': 'localhost', + 'port': 5432, + 'user': 'postgres', + 'password': 'postgres', + 'dbname': 'postgres', + 'min_connections': 1, + 'max_connections': 10, + } + + redis_config = {'host': 'localhost', 'port': 6379, 'username': 'redis', 'password': 'redis'} + + mongodb_config = { + 'connection_string': 'mongodb://localhost:27017', + 'database_name': 'test_database', + } + + api_config = { + 'base_url': 'https://api.example.com', + 'auth_type': 'bearer', + 'auth_token': 'test_token', + } + + logger = MagicMock() + notification_handler = MagicMock() + + activities = Activities( + postgres_config=postgres_config, + redis_config=redis_config, + mongodb_config=mongodb_config, + api_config=api_config, + logger=logger, + notification_handler=notification_handler, + ) + + assert isinstance(activities, Activities) + assert isinstance(activities, Postgres) + assert isinstance(activities, Redis) + assert isinstance(activities, MongoDB) + assert isinstance(activities, Gates) + assert isinstance(activities, API) + + mock_postgres_init.assert_called_once_with( + ANY, + host=postgres_config['host'], + port=postgres_config['port'], + user=postgres_config['user'], + password=postgres_config['password'], + dbname=postgres_config['dbname'], + min_connections=postgres_config['min_connections'], + max_connections=postgres_config['max_connections'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=mock_metrics_controller.return_value, + ) + + mock_redis_init.assert_called_once_with( + ANY, + host=redis_config['host'], + port=redis_config['port'], + username=redis_config['username'], + password=redis_config['password'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=mock_metrics_controller.return_value, + ) + + mock_mongodb_init.assert_called_once_with( + ANY, + connection_string=mongodb_config['connection_string'], + database_name=mongodb_config['database_name'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=mock_metrics_controller.return_value, + ) + + mock_gates_init.assert_called_once_with( + ANY, + logger=logger, + notification_handler=notification_handler, + metrics_controller=mock_metrics_controller.return_value, + ) + + mock_api_init.assert_called_once_with( + ANY, + base_url=api_config['base_url'], + auth_type=api_config['auth_type'], + auth_token=api_config['auth_token'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=mock_metrics_controller.return_value, + ) + + +@patch('scouter.activities.activities.Postgres.__init__') +@patch('scouter.activities.activities.Redis.__init__') +@patch('scouter.activities.activities.Gates.__init__') +@patch('scouter.activities.activities.MongoDB.__init__') +@patch('scouter.activities.activities.API.__init__') +@patch('scouter.activities.activities.Postgres.close') +@patch('scouter.activities.activities.MongoDB.close') +@patch('scouter.activities.activities.Redis.close') +@patch('scouter.activities.activities.Gates.close') +@patch('scouter.activities.activities.API.close') +def test_shutdown( + mock_api_close, + mock_gates_close, + mock_redis_close, + mock_mongodb_close, + mock_postgres_close, + _mock_api_init, + _mock_mongodb_init, + _mock_gates_init, + _mock_redis_init, + _mock_postgres_init, +): + postgres_config = { + 'host': 'localhost', + 'port': 5432, + 'user': 'postgres', + 'password': 'postgres', + 'dbname': 'postgres', + 'min_connections': 1, + 'max_connections': 10, + } + + redis_config = {'host': 'localhost', 'port': 6379, 'username': 'redis', 'password': 'redis'} + + mongodb_config = { + 'connection_string': 'mongodb://localhost:27017', + 'database_name': 'test_database', + } + + api_config = { + 'base_url': 'https://api.example.com', + 'auth_type': 'bearer', + 'auth_token': 'test_token', + } + + logger = MagicMock() + notification_handler = MagicMock() + + activities = Activities( + postgres_config=postgres_config, + redis_config=redis_config, + mongodb_config=mongodb_config, + api_config=api_config, + logger=logger, + notification_handler=notification_handler, + ) + + activities.shutdown() + + mock_postgres_close.assert_called() + mock_mongodb_close.assert_called() + mock_redis_close.assert_called() + mock_gates_close.assert_called() + mock_api_close.assert_called() + + +def test_activity_methods_are_sync(): + """Every @activity.defn method on Activities must be a synchronous def.""" + for cls in Activities.__mro__: + for name, member in vars(cls).items(): + if getattr(member, '__temporal_activity_definition', None) is not None: + assert not inspect.iscoroutinefunction(member), ( + f'{cls.__name__}.{name} must not be async' + ) diff --git a/tests/activities/test_api.py b/tests/activities/test_api.py new file mode 100644 index 0000000..3c12209 --- /dev/null +++ b/tests/activities/test_api.py @@ -0,0 +1,340 @@ +from unittest.mock import ANY, MagicMock, Mock, patch + +import pandas as pd +import pytest +from sientia_do.notifications.models import NotificationLevel + +from scouter.activities.api import API + + +@pytest.fixture +@patch('scouter.activities.api.PIWebAPIClient') +def api_activity(mock_pi_web_api_client): + """Fixture to create an API activity instance with mocked dependencies.""" + logger = MagicMock() + notification_handler = MagicMock() + metrics_controller = MagicMock() + + activity = API( + base_url='https://pi.example.com', + auth_type='basic', + auth_token='test_token', + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + activity.logger = logger + activity.notification_handler = notification_handler + activity.metrics_controller = metrics_controller + activity.pod_id = 'test_pod_id' + + return activity + + +metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'pi_web_api_scouter', + } +} + + +@patch('scouter.activities.api.PIWebAPIClient') +def test_api_initialization(mock_pi_web_api_client): + """Test API activity initialization.""" + logger = MagicMock() + notification_handler = MagicMock() + metrics_controller = MagicMock() + + activity = API( + base_url='https://pi.example.com', + auth_type='basic', + auth_token='test_token', + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + mock_pi_web_api_client.assert_called_once_with( + base_url='https://pi.example.com', + auth_config={ + 'type': 'basic', + 'token': 'test_token', + }, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + headers_config={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'x-requested-with': 'piwebapistreams', + 'User-Agent': 'Aig-Scouter-Agent/1.0', + }, + ) + + assert activity.pi_web_api_client is not None + + +@patch('scouter.activities.api.SientiaMonitoring') +def test_close(mock_sientia_monitoring, api_activity): + """Test close method.""" + api_activity.close() + api_activity.pi_web_api_client.close.assert_called_once() + mock_sientia_monitoring.shutdown.assert_called_once() + + +def test_get_tag_values_success(api_activity): + """Test get_tag_values with successful data retrieval.""" + # Setup test data + test_data = { + **metadata, + 'endpoint': '/streamsets/recorded', + 'web_ids': { + 'tag1': { + 'webid': 'webid1', + 'aggr_function': 'avg', + 'data_range': [0, 100], + }, + 'tag2': { + 'webid': 'webid2', + 'aggr_function': 'avg', + 'data_range': [0, 100], + }, + 'tag3': { + 'webid': 'webid3', + 'aggr_function': 'avg', + 'data_range': [0, 100], + }, + }, + 'period': '*-1d', + 'max_count': 10, + 'api_timeout': 30, + } + + # Mock DataFrame response + mock_df = pd.DataFrame( + { + 'timestamp': [ + '2023-01-01 12:00:00+0000', + '2023-01-01 12:01:00+0000', + '2023-01-01 12:02:00+0000', + ], + 'name': ['tag1', 'tag2', 'tag3'], + 'value': [10.5, 20.3, 30.7], + 'tag': ['webid1', 'webid2', 'webid3'], + } + ) + + mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp']) + + api_activity.pi_web_api_client.get_latest_values_df = Mock(return_value=mock_df) + + # Execute + result = api_activity.get_tag_values(test_data) + + # Verify + api_activity.pi_web_api_client.get_latest_values_df.assert_called_once_with( + endpoint='/streamsets/recorded', + web_ids={ + 'tag1': {'webid': 'webid1', 'aggr_function': 'avg', 'data_range': [0, 100]}, + 'tag2': {'webid': 'webid2', 'aggr_function': 'avg', 'data_range': [0, 100]}, + 'tag3': {'webid': 'webid3', 'aggr_function': 'avg', 'data_range': [0, 100]}, + }, + start_time='*-1d', + end_time='*', + max_count=10, + metadata=metadata['metadata'], + request_timeout=30, + ) + + assert len(result) == 3 + assert result[0]['name'] == 'tag1' + assert result[0]['value'] == pytest.approx(10.5) + assert result[1]['name'] == 'tag2' + assert result[2]['name'] == 'tag3' + assert result[0]['timestamp'] == '2023-01-01 12:02:00+0000' + assert result[1]['timestamp'] == '2023-01-01 12:02:00+0000' + assert result[2]['timestamp'] == '2023-01-01 12:02:00+0000' + + +def test_get_tag_values_with_default_max_count(api_activity): + """Test get_tag_values with default max_count value.""" + # Setup test data without max_count + test_data = { + **metadata, + 'endpoint': '/streamsets/recorded', + 'web_ids': { + 'tag1': { + 'webid': 'webid1', + 'aggr_function': 'avg', + 'data_range': [0, 100], + } + }, + 'period': '*-1h', + 'api_timeout': 15, + } + + mock_df = pd.DataFrame( + { + 'timestamp': ['2023-01-01 12:00:00+0000'], + 'name': ['tag1'], + 'value': [42.0], + 'tag': ['webid1'], + } + ) + + mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp']) + + api_activity.pi_web_api_client.get_latest_values_df = Mock(return_value=mock_df) + + # Execute + result = api_activity.get_tag_values(test_data) + + # Verify default max_count is 1 + api_activity.pi_web_api_client.get_latest_values_df.assert_called_once_with( + endpoint='/streamsets/recorded', + web_ids={'tag1': {'webid': 'webid1', 'aggr_function': 'avg', 'data_range': [0, 100]}}, + start_time='*-1h', + end_time='*', + max_count=1, + metadata=metadata['metadata'], + request_timeout=15, + ) + + assert len(result) == 1 + + +def test_get_tag_values_with_none_webids(api_activity): + """Test get_tag_values with some None WebIds.""" + # Setup test data with None values + test_data = { + **metadata, + 'endpoint': '/streamsets/recorded', + 'web_ids': { + 'tag1': { + 'webid': 'webid1', + 'aggr_function': 'avg', + 'data_range': [0, 100], + }, + 'tag2': None, + 'tag3': { + 'webid': 'webid3', + 'aggr_function': 'max', + 'data_range': [0, 200], + }, + }, + 'period': '*-1h', + 'max_count': 5, + 'api_timeout': 20, + } + + mock_df = pd.DataFrame( + { + 'timestamp': ['2023-01-01 12:00:00+0000', '2023-01-01 12:01:00+0000'], + 'name': ['tag1', 'tag3'], + 'value': [10.5, 30.7], + 'tag': ['webid1', 'webid3'], + } + ) + + mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp']) + + api_activity.pi_web_api_client.get_latest_values_df = Mock(return_value=mock_df) + + # Execute + result = api_activity.get_tag_values(test_data) + + # Verify - should only query non-None WebIds + assert len(result) == 2 + assert all(r['name'] in ['tag1', 'tag3'] for r in result) + + +def test_get_tag_values_api_error(api_activity): + """Test get_tag_values when PI Web API client raises an error and sends notification.""" + # Setup test data + test_data = { + **metadata, + 'endpoint': '/streamsets/recorded', + 'web_ids': { + 'tag1': { + 'webid': 'webid1', + 'aggr_function': 'avg', + 'data_range': [0, 100], + } + }, + 'period': '*-1d', + 'max_count': 1, + 'api_timeout': 30, + } + + # Mock API error + api_activity.pi_web_api_client.get_latest_values_df = Mock( + side_effect=Exception('PI Web API connection error') + ) + api_activity.send_notification = MagicMock() + + # Execute and verify exception is raised + with pytest.raises(Exception) as exc_info: + api_activity.get_tag_values(test_data) + + assert str(exc_info.value) == 'PI Web API connection error' + + # Verify notification was sent + api_activity.send_notification.assert_called_once_with( + metadata=metadata['metadata'], + notification_id='PI_WEB_API_REQUEST_ERROR', + message='Error getting tag values from PI Web API: PI Web API connection error', + block='get_tag_values', + level=NotificationLevel.ERROR, + attachment_content=ANY, + ) + + +def test_get_tag_values_with_nan_values(api_activity): + """Test get_tag_values handling NaN values in the DataFrame.""" + # Setup test data + test_data = { + **metadata, + 'endpoint': '/streamsets/recorded', + 'web_ids': { + 'tag1': { + 'webid': 'webid1', + 'aggr_function': 'avg', + 'data_range': [0, 100], + }, + 'tag2': { + 'webid': 'webid2', + 'aggr_function': 'avg', + 'data_range': [0, 100], + }, + }, + 'period': '*-1d', + 'max_count': 1, + 'api_timeout': 30, + } + + # Mock DataFrame with NaN values + mock_df = pd.DataFrame( + { + 'timestamp': ['2023-01-01 12:00:00+0000', '2023-01-01 12:00:00+0000'], + 'name': ['tag1', 'tag2'], + 'value': [10.0, float('nan')], + 'tag': ['webid1', 'webid2'], + } + ) + + mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp']) + + api_activity.pi_web_api_client.get_latest_values_df = Mock(return_value=mock_df) + + # Execute + result = api_activity.get_tag_values(test_data) + + # Verify + assert len(result) == 2 + assert result[0]['value'] == pytest.approx(10.0) + # NaN should be preserved in the result + assert pd.isna(result[1]['value']) diff --git a/tests/activities/test_gates.py b/tests/activities/test_gates.py new file mode 100644 index 0000000..84e513b --- /dev/null +++ b/tests/activities/test_gates.py @@ -0,0 +1,409 @@ +from typing import Any +from unittest.mock import ANY, MagicMock, Mock, call, patch + +import numpy as np +import pandas as pd +import pytest +from sientia_do.notifications.models import NotificationLevel + +from scouter.activities.gates import Gates + + +@pytest.fixture +def gates_fixture(): + """Fixture to create a Gates instance with mocked dependencies.""" + logger = Mock() + notification_handler = MagicMock() + metrics_controller = MagicMock() + gates = Gates( + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + gates.send_notification = MagicMock() + + gates.logger = logger + gates.notification_handler = notification_handler + gates.metrics_controller = metrics_controller + gates.pod_id = 'localhost' + return gates + + +metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'scouter', + } +} + + +@patch('scouter.activities.gates.SientiaMonitoring') +def test_close(mock_sientia_monitoring, gates_fixture): + """Test close method.""" + gates_fixture.close() + mock_sientia_monitoring.shutdown.assert_called_once() + + +def test_data_quality_gate_with_null_values_filter_discard(gates_fixture): + """Test data_quality_gate with NULL_VALUES_FILTER and DISCARD policy.""" + # Setup test data + input_data = { + 'filters': {'NULL_VALUES_FILTER': {'policy': 'DISCARD'}}, + 'data': { + 'name': ['tag1', 'tag2', 'tag3'], + 'tag': ['tag1', 'tag2', 'tag3'], + 'value': [1.0, None, 3.0], + 'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03'], + }, + 'model_tags': { + 'tag1': {'data_range': [0, 100]}, + 'tag2': {'data_range': [0, 100]}, + 'tag3': {'data_range': [0, 100]}, + }, + **metadata, + } + + # Execute + result = gates_fixture.data_quality_gate(input_data) + + # Verify + assert len(result['tag']) == 2 + assert 'tag2' not in result['tag'] + gates_fixture.send_notification.assert_called_once() + + +def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture): + """Test data_quality_gate with OUT_OF_BOUNDS_FILTER and KEEP policy.""" + # Setup test data with out of bounds values + input_data = { + 'filters': {'OUT_OF_BOUNDS_FILTER': {'policy': 'KEEP'}}, + 'data': { + 'name': ['tag1', 'tag2', 'tag3'], + 'tag': ['tag1', 'tag2', 'tag3'], + 'value': [1.0, 200.0, 3.0], + 'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03'], + }, + 'model_tags': { + 'tag1': {'data_range': [0, 100]}, + 'tag2': {'data_range': [0, 100]}, + 'tag3': {'data_range': [0, 100]}, + }, + **metadata, + } + + # Mock the out_of_bounds_filter to return rows with out of bounds values + with patch( + 'scouter.activities.gates.quality_gate_filters', + {'OUT_OF_BOUNDS_FILTER': lambda df: df[df['tag'] == 'tag2']}, + ): + # Execute + result = gates_fixture.data_quality_gate(input_data) + + # Verify data is kept but notification is sent + assert len(result['tag']) == 3 # All rows kept + gates_fixture.send_notification.assert_called_once() + + +def test_data_quality_gate_with_multiple_filters(gates_fixture): + """Test data_quality_gate with multiple filters.""" + # Setup test data + input_data = { + 'filters': { + 'NULL_VALUES_FILTER': {'policy': 'DISCARD'}, + 'OUT_OF_BOUNDS_FILTER': {'policy': 'DISCARD'}, + }, + 'data': { + 'tag': ['tag1', 'tag2', 'tag3', 'tag4'], + 'name': ['tag1', 'tag2', 'tag3', 'tag4'], + 'value': [1.0, None, 300.0, 4.0], + 'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03', '2023-01-04'], + }, + 'model_tags': { + 'tag1': {'data_range': [0, 100]}, + 'tag2': {'data_range': [0, 100]}, + 'tag3': {'data_range': [0, 100]}, + 'tag4': {'data_range': [0, 100]}, + }, + **metadata, + } + + result = gates_fixture.data_quality_gate(input_data) + + # Verify only tag1 and tag4 remain (tag2 has null, tag3 is out of bounds) + assert result == { + 'tag': {0: 'tag1', 3: 'tag4'}, + 'name': {0: 'tag1', 3: 'tag4'}, + 'value': {0: 1.0, 3: 4.0}, + 'timestamp': {0: '2023-01-01', 3: '2023-01-04'}, + } + # Should be called twice (once for each filter) + assert gates_fixture.send_notification.call_count == 2 + + +def test_data_quality_gate_with_unknown_filter(gates_fixture): + """Test data_quality_gate with an unknown filter.""" + # Setup test data with unknown filter + gates_fixture.warning = MagicMock() + input_data = { + 'filters': {'UNKNOWN_FILTER': {'policy': 'DISCARD'}}, + 'data': {'tag': ['tag1'], 'name': ['tag1'], 'value': [1.0], 'timestamp': ['2023-01-01']}, + 'model_tags': {'tag1': {'data_range': [0, 100]}}, + **metadata, + } + + # Execute + result = gates_fixture.data_quality_gate(input_data) + + # Verify data is unchanged and warning is logged + assert len(result['tag']) == 1 + gates_fixture.warning.assert_called_once_with( + 'Filter UNKNOWN_FILTER not found', metadata=metadata['metadata'] + ) + + +def test_data_quality_gate_with_filter_error(gates_fixture): + """Test data_quality_gate when a filter raises an exception.""" + # Setup test data + input_data = { + 'filters': {'NULL_VALUES_FILTER': {'policy': 'DISCARD'}}, + 'data': {'tag': ['tag1'], 'name': ['tag1'], 'value': [1.0], 'timestamp': ['2023-01-01']}, + 'model_tags': {'tag1': {'data_range': [0, 100]}}, + **metadata, + } + + # Mock the filter to raise an exception + def failing_filter(_, _model_tags): + raise ValueError('Filter error') + + with patch( + 'scouter.activities.gates.quality_gate_filters', {'NULL_VALUES_FILTER': failing_filter} + ): + # Execute + result = gates_fixture.data_quality_gate(input_data) + + # Verify error notification is sent and data is unchanged + assert len(result['tag']) == 1 + gates_fixture.send_notification.assert_called_once() + call_args = gates_fixture.send_notification.call_args[1] + assert call_args['notification_id'] == 'DATA_QUALITY_GATE_ISSUES' + assert call_args['level'] == NotificationLevel.ERROR + assert 'Filter error' in call_args['message'] + + +def test_data_quality_gate_with_empty_data(gates_fixture): + """Test data_quality_gate with empty input data.""" + # Setup empty input data + input_data = { + 'filters': {'NULL_VALUES_FILTER': {'policy': 'DISCARD'}}, + 'data': {'tag': [], 'name': [], 'value': [], 'timestamp': []}, + 'model_tags': {}, + **metadata, + } + + # Execute + result = gates_fixture.data_quality_gate(input_data) + + # Verify empty result and no notifications + assert len(result['tag']) == 0 + gates_fixture.send_notification.assert_not_called() + + +def test_data_quality_gate_with_no_filters(gates_fixture): + """Test data_quality_gate with no filters specified.""" + # Setup test data with no filters + input_data = { + 'filters': {}, + 'data': {'tag': ['tag1'], 'name': ['tag1'], 'value': [1.0], 'timestamp': ['2023-01-01']}, + 'model_tags': {'tag1': {'data_range': [0, 100]}}, + **metadata, + } + + # Execute + result = gates_fixture.data_quality_gate(input_data) + + # Verify data is unchanged and no notifications + assert len(result['tag']) == 1 + gates_fixture.send_notification.assert_not_called() + + +@pytest.mark.parametrize( + 'group_data, aggr_function, expected_result', + [ + # Single value case + (pd.DataFrame({'value': [10.0]}), 'avg', 10.0), + # Multiple values with different aggregation functions + (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'avg', 2.5), + (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'mdn', 2.5), + (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'max', 4.0), + (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'min', 1.0), + (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'lts', 4.0), + # With NaN values + (pd.DataFrame({'value': [1.0, np.nan, 3.0, 4.0]}), 'avg', 2.6666666666666665), + # Empty group after dropping NaN + (pd.DataFrame({'value': [np.nan, np.nan]}), 'avg', None), + # Invalid aggregation function + (pd.DataFrame({'value': [1.0, 2.0]}), 'invalid', 'continue'), + (pd.DataFrame({'value': [10.0]}), 'invalid', 'continue'), + ], +) +def test_apply_aggregation(gates_fixture, group_data, aggr_function, expected_result): + """Test apply_aggregation method with various scenarios.""" + result = gates_fixture.apply_aggregation(group_data, aggr_function, metadata) + assert result == expected_result + + # Check notification was sent for invalid function + if aggr_function == 'invalid': + gates_fixture.send_notification.assert_called_once() + else: + gates_fixture.send_notification.assert_not_called() + + +def test_aggregate_data(gates_fixture): + """Test aggregate_data method with multiple groups and aggregation functions.""" + input_data = { + 'data': [ + {'tag': 'tag1', 'name': 'name1', 'value': 1.0, 'timestamp': '2023-01-01'}, + {'tag': 'tag1', 'name': 'name1', 'value': 2.0, 'timestamp': '2023-01-02'}, + {'tag': 'tag1', 'name': 'name1', 'value': 3.0, 'timestamp': '2023-01-03'}, + {'tag': 'tag2', 'name': 'name2', 'value': 4.0, 'timestamp': '2023-01-01'}, + {'tag': 'tag2', 'name': 'name2', 'value': 5.0, 'timestamp': '2023-01-02'}, + {'tag': 'tag2', 'name': 'name2', 'value': 6.0, 'timestamp': '2023-01-03'}, + {'tag': 'tag1', 'name': 'name1', 'value': None, 'timestamp': '2023-01-04'}, + ], + 'model_tags': { + 'name1': {'aggr_func': 'avg'}, + 'name2': {'aggr_func': 'max'}, + }, + **metadata, + } + + # Expected result + expected_result = { + 'tag': {0: 'tag1', 1: 'tag2'}, + 'name': {0: 'name1', 1: 'name2'}, + 'value': {0: 2.0, 1: 6.0}, + 'timestamp': {0: '2023-01-04', 1: '2023-01-03'}, + 'aggregation_function': {0: 'avg', 1: 'max'}, + } + + # Execute + result = gates_fixture.aggregate_data(input_data) + + # Verify + assert result == expected_result + gates_fixture.send_notification.assert_not_called() + + +def test_aggregate_data_with_continue(gates_fixture): + gates_fixture.apply_aggregation = MagicMock(return_value='continue') + + input_data = { + 'data': [ + {'tag': 'tag1', 'name': 'name1', 'value': 1.0, 'timestamp': '2023-01-01'}, + {'tag': 'tag1', 'name': 'name1', 'value': 2.0, 'timestamp': '2023-01-02'}, + {'tag': 'tag1', 'name': 'name1', 'value': 3.0, 'timestamp': '2023-01-03'}, + {'tag': 'tag2', 'name': 'name2', 'value': 4.0, 'timestamp': '2023-01-01'}, + {'tag': 'tag2', 'name': 'name2', 'value': 5.0, 'timestamp': '2023-01-02'}, + {'tag': 'tag2', 'name': 'name2', 'value': 6.0, 'timestamp': '2023-01-03'}, + {'tag': 'tag1', 'name': 'name1', 'value': None, 'timestamp': '2023-01-04'}, + ], + 'model_tags': { + 'name1': {'aggr_function': 'avg'}, + 'name2': {'aggr_function': 'max'}, + }, + **metadata, + } + + # Expected result + expected_result: dict[str, Any] = {} + + # Execute + result = gates_fixture.aggregate_data(input_data) + + # Verify + assert result == expected_result + gates_fixture.send_notification.assert_not_called() + + +def test_aggregate_data_raise_exception(gates_fixture): + gates_fixture.apply_aggregation = MagicMock(side_effect=Exception('Test exception')) + + input_data = { + 'data': [ + {'tag': 'tag1', 'name': 'name1', 'value': 1.0, 'timestamp': '2023-01-01'}, + {'tag': 'tag1', 'name': 'name1', 'value': 2.0, 'timestamp': '2023-01-02'}, + {'tag': 'tag1', 'name': 'name1', 'value': 3.0, 'timestamp': '2023-01-03'}, + {'tag': 'tag2', 'name': 'name2', 'value': 4.0, 'timestamp': '2023-01-01'}, + {'tag': 'tag2', 'name': 'name2', 'value': 5.0, 'timestamp': '2023-01-02'}, + {'tag': 'tag2', 'name': 'name2', 'value': 6.0, 'timestamp': '2023-01-03'}, + {'tag': 'tag1', 'name': 'name1', 'value': None, 'timestamp': '2023-01-04'}, + ], + 'model_tags': { + 'name1': {'aggr_function': 'avg'}, + 'name2': {'aggr_function': 'max'}, + }, + **metadata, + } + + try: + gates_fixture.aggregate_data(input_data) + except Exception as e: + assert str(e) == 'Test exception' + gates_fixture.send_notification.assert_called_once_with( + metadata=metadata['metadata'], + notification_id='AGGREGATION_ISSUES', + message='Error aggregating data: Test exception', + block='aggregate_data', + level=NotificationLevel.ERROR, + attachment_content=ANY, + ) + else: + raise AssertionError('Exception not raised') + + +@patch('scouter.activities.gates.metrics') +def test_write_metrics(mock_metrics, gates_fixture): + """Test write_metrics method.""" + input_data = { + 'metadata': metadata['metadata'], + 'tag_values': { + 'variable': ['tag1', 'tag2', 'tag3'], + 'value': [1.0, 2.0, None], + }, + } + gates_fixture.write_metrics(input_data) + mock_metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels.assert_called_once_with( + pod_id=gates_fixture.pod_id, + model_name=metadata['metadata']['model_name'], + workflow_name=metadata['metadata']['workflow_name'], + ) + mock_metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels.return_value.inc.assert_called_once() + + # Only tags with None values should be registered + mock_metrics.TAG_CHANGES_MONITOR.labels.return_value.set.assert_has_calls( + [ + call(1.0), + call(2.0), + ], + any_order=True, + ) + + mock_metrics.TAG_CHANGES_MONITOR.labels.assert_has_calls( + [ + call( + pod_id=gates_fixture.pod_id, + model_name=metadata['metadata']['model_name'], + workflow_name=metadata['metadata']['workflow_name'], + tag_name='tag1', + ), + call( + pod_id=gates_fixture.pod_id, + model_name=metadata['metadata']['model_name'], + workflow_name=metadata['metadata']['workflow_name'], + tag_name='tag2', + ), + ], + any_order=True, + ) diff --git a/tests/activities/test_mongo.py b/tests/activities/test_mongo.py new file mode 100644 index 0000000..8f72530 --- /dev/null +++ b/tests/activities/test_mongo.py @@ -0,0 +1,174 @@ +from datetime import datetime +from unittest.mock import ANY, MagicMock, Mock, patch + +from pytest import fixture +from sientia_do.notifications.models import NotificationLevel +from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ + +from scouter.activities.mongodb import MongoDB + + +@patch('scouter.activities.mongodb.MongoDBRepository') +def test_mongodb___init__(mock_mongodb_repository): + """Test MongoDB __init__""" + + logger = MagicMock() + notification_handler = MagicMock() + metrics_controller = MagicMock() + mongo = MongoDB( + connection_string='mongodb://localhost:27017', + database_name='test_db', + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + mock_mongodb_repository.assert_called_once_with( + connection_string='mongodb://localhost:27017', + database_name='test_db', + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + assert mongo.mongodb_repository is not None + + +@fixture +@patch('scouter.activities.mongodb.MongoDBRepository') +def mongodb_activity(mock_mongodb_repository): + """Test MongoDB activity""" + + mongo = MongoDB( + connection_string='mongodb://localhost:27017', + database_name='test_db', + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=MagicMock(), + ) + return mongo + + +@patch('scouter.activities.mongodb.SientiaMonitoring') +def test_close(mock_sientia_monitoring, mongodb_activity): + """Test close""" + mongodb_activity.close() + + mongodb_activity.mongodb_repository.close.assert_called_once() + mock_sientia_monitoring.shutdown.assert_called_once() + + +def test_del(mongodb_activity): + mongodb_activity.close = MagicMock() + + mongodb_activity.__del__() + + mongodb_activity.close.assert_called_once() + + +def test_load_latest_data_none_last_data_timestamp(mongodb_activity): + """Test load_latest_data""" + + mongodb_activity.mongodb_repository.find = Mock( + return_value=[ + { + 'name': 'test1', + 'value': 1, + 'inserted_at': datetime.strptime( + '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ + ), + } + ] + ) + + result = mongodb_activity.load_latest_data( + { + 'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, + 'collection_name': 'test_collection', + 'last_data_timestamp': None, + } + ) + + mongodb_activity.mongodb_repository.find.assert_called_once_with( + collection_name='test_collection', + filters={}, + metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, + ) + + assert result == [ + { + 'name': 'test1', + 'value': 1, + 'inserted_at': '2023-01-01 12:00:00.000000+0000', + } + ] + + +def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): + """Test load_latest_data""" + + mongodb_activity.mongodb_repository.find = Mock( + return_value=[ + { + 'name': 'test1', + 'value': 1, + 'inserted_at': datetime.strptime( + '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ + ), + } + ] + ) + + result = mongodb_activity.load_latest_data( + { + 'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, + 'collection_name': 'test_collection', + 'last_data_timestamp': '2023-01-01 12:00:00.000000+0000', + } + ) + + mongodb_activity.mongodb_repository.find.assert_called_once_with( + collection_name='test_collection', + filters={ + 'inserted_at': { + '$gt': datetime.strptime( + '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ + ) + } + }, + metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, + ) + + assert result == [ + { + 'name': 'test1', + 'value': 1, + 'inserted_at': '2023-01-01 12:00:00.000000+0000', + } + ] + + +def test_load_latest_data_error(mongodb_activity): + """Test load_latest_data""" + mongodb_activity.mongodb_repository.find.side_effect = Exception('test') + mongodb_activity.send_notification = MagicMock() + + try: + mongodb_activity.load_latest_data( + { + 'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, + 'collection_name': 'test_collection', + 'last_data_timestamp': '2023-01-01 12:00:00.000000+0000', + } + ) + except Exception as e: + assert str(e) == 'test' + + mongodb_activity.send_notification.assert_called_once_with( + metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, + notification_id='MONGO_LOAD_ERROR', + message='Error loading data from MongoDB: test', + block='load_latest_data', + level=NotificationLevel.ERROR, + attachment_content=ANY, + ) diff --git a/tests/activities/test_redis.py b/tests/activities/test_redis.py new file mode 100644 index 0000000..947811d --- /dev/null +++ b/tests/activities/test_redis.py @@ -0,0 +1,525 @@ +from datetime import datetime +from unittest.mock import ANY, MagicMock, Mock, patch + +import numpy as np +import pytest +from pandas import DataFrame +from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler +from sientia_do.notifications.models import NotificationLevel + +from scouter.activities.redis import Redis + + +@pytest.fixture +@patch('scouter.activities.redis.RedisRepository') +def redis_activity(mock_redis_repository): + logger = MagicMock() + notification_handler = MagicMock(spec=NotificationHandler) + metrics_controller = MagicMock() + activity = Redis( + host='localhost', + port=6379, + logger=logger, + notification_handler=notification_handler, + username='test', + password='test', + metrics_controller=metrics_controller, + ) + + activity.redis_client = MagicMock() + activity.logger = logger + activity.notification_handler = notification_handler + activity.pod_id = 'test_pod_id' + return activity + + +metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'scouter', + } +} + + +@patch('scouter.activities.redis.SientiaMonitoring') +def test_close(mock_sientia_monitoring, redis_activity): + """Test close method.""" + redis_activity.close() + redis_activity.redis_repository.close.assert_called_once() + mock_sientia_monitoring.shutdown.assert_called_once() + + +@patch('scouter.activities.redis.RedisRepository') +def test_redis_initialization(mock_redis_repository): + """Test Redis activity initialization""" + logger = MagicMock() + notification_handler = MagicMock(spec=NotificationHandler) + metrics_controller = MagicMock() + activity = Redis( + host='localhost', + port=6379, + logger=logger, + notification_handler=notification_handler, + username='test', + password='test', + metrics_controller=metrics_controller, + ) + mock_redis_repository.assert_called_once_with( + host='localhost', + port=6379, + username='test', + password='test', + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + assert activity.redis_repository is not None + + +def test_get_last_data_timestamp_none(redis_activity): + """Test get_last_data_timestamp""" + test_data = { + **metadata, + 'workflow_name': 'test_pipeline', + 'schedule_name': 'test_schedule', + } + + redis_activity.redis_repository.get = Mock(return_value=None) + + result = redis_activity.get_last_data_timestamp(test_data) + + redis_activity.redis_repository.get.assert_called_once_with( + 'last_data_timestamp:test_pipeline:test_schedule', + metadata=metadata['metadata'], + ) + + assert result is None + + +def test_get_last_data_timestamp_not_none(redis_activity): + """Test get_last_data_timestamp""" + test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'} + + redis_activity.redis_repository.get = Mock(return_value='2023-01-01 12:00:00') + + result = redis_activity.get_last_data_timestamp(test_data) + + redis_activity.redis_repository.get.assert_called_once_with( + 'last_data_timestamp:test_pipeline:test_schedule', + metadata=metadata['metadata'], + ) + + assert result == '2023-01-01 12:00:00' + + +def test_get_last_data_timestamp_error(redis_activity): + """Test get_last_data_timestamp error""" + test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'} + + redis_activity.send_notification = Mock() + redis_activity.redis_repository.get.side_effect = Exception('test') + + try: + redis_activity.get_last_data_timestamp(test_data) + + except Exception as e: + assert str(e) == 'test' + + redis_activity.send_notification.assert_called_once_with( + metadata=metadata['metadata'], + notification_id='REDIS_GET_ERROR', + message='Error getting last data timestamp: test', + block='get_last_data_timestamp', + level=NotificationLevel.ERROR, + attachment_content=ANY, + ) + + else: + raise AssertionError('Expected exception') + + +def test_put_last_data_timestamp_empty_dataframe(redis_activity): + """Test put_last_data_timestamp with empty dataframe""" + test_data = { + **metadata, + 'workflow_name': 'test_pipeline', + 'schedule_name': 'test_schedule', + 'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'), + } + + redis_activity.redis_repository.set = MagicMock() + + result = redis_activity.put_last_data_timestamp(test_data) + + assert result is None + + redis_activity.redis_repository.set.assert_not_called() + + +def test_put_last_data_timestamp_not_empty_dataframe(redis_activity): + """Test put_last_data_timestamp with not empty dataframe""" + + data = DataFrame( + { + 'name': ['sensor1', 'sensor2'], + 'value': [25.5, 30.0], + 'inserted_at': ['2023-01-01 12:00:00', '2023-01-01 12:00:01'], + } + ) + test_data = { + **metadata, + 'workflow_name': 'test_pipeline', + 'schedule_name': 'test_schedule', + 'data': data.to_dict('records'), + } + + redis_activity.redis_repository.set = Mock() + + result = redis_activity.put_last_data_timestamp(test_data) + + assert result == '2023-01-01 12:00:01' + + redis_activity.redis_repository.set.assert_called_once_with( + 'last_data_timestamp:test_pipeline:test_schedule', + '2023-01-01 12:00:01', + ttl=18000, + metadata=metadata['metadata'], + ) + + +def test_put_last_data_timestamp_error(redis_activity): + """Test put_last_data_timestamp error""" + test_data = { + **metadata, + 'workflow_name': 'test_pipeline', + 'schedule_name': 'test_schedule', + 'data': DataFrame( + { + 'name': ['sensor1', 'sensor2'], + 'value': [25.5, 30.0], + 'inserted_at': ['2023-01-01 12:00:00'] * 2, + } + ).to_dict('records'), + } + + redis_activity.send_notification = Mock() + redis_activity.redis_repository.set = Mock(side_effect=Exception('test')) + + try: + redis_activity.put_last_data_timestamp(test_data) + + except Exception as e: + assert str(e) == 'test' + + redis_activity.send_notification.assert_called_once_with( + metadata=metadata['metadata'], + notification_id='REDIS_SET_ERROR', + message='Error setting last data timestamp: test', + block='put_last_data_timestamp', + level=NotificationLevel.ERROR, + attachment_content=ANY, + ) + + else: + raise AssertionError('Expected exception') + + +def test_group_and_hold_data_new_key(redis_activity): + """Test group_and_hold_data with a new key""" + # Setup + test_data = { + **metadata, + 'workflow_name': 'test_pipeline', + 'schedule_name': 'test_schedule', + 'retention_time': 3600, + 'model_id': 1, + 'data': DataFrame( + { + 'name': ['sensor1', 'sensor2'], + 'value': [25.5, 30.0], + 'timestamp': ['2023-01-01 12:00:00'] * 2, + } + ).to_dict('records'), + 'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'}, + 'fill_missing_tags': False, + } + + # Mock get to return None for new key + redis_activity.redis_repository.get = Mock(return_value=None) + redis_activity.redis_repository.set = Mock() + + # Call the method + result = redis_activity.group_and_hold_data(test_data) + + # Verify the result + expected_result = { + 'timestamp': {0: '2023-01-01 12:00:00', 1: '2023-01-01 12:00:00'}, + 'variable': {0: 'sensor1', 1: 'sensor2'}, + 'value': {0: 25.5, 1: 30.0}, + 'model_id': {0: 1, 1: 1}, + } + assert result == expected_result + + # Verify set was called with correct arguments + redis_activity.redis_repository.set.assert_called_once_with( + 'held_data_test_pipeline_test_schedule', + {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}, + ttl=3600, + metadata=metadata['metadata'], + ) + + +def test_group_and_hold_data_update_existing_fill_missing(redis_activity): + """Test updating existing data with group_and_hold_data""" + # Setup initial data in Redis + existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'} + + # New data to update with + test_data = { + **metadata, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'retention_time': 3600, + 'model_id': 1, + 'data': DataFrame( + { + 'name': ['sensor1', 'sensor3'], + 'value': [25.5, 42.0], + 'timestamp': ['2023-01-01 12:00:00'] * 2, + } + ).to_dict('records'), + 'model_tags': { + 'sensor1': 'sensor1', + 'sensor2': 'sensor2', + 'sensor3': 'sensor3', + 'sensor4': 'sensor4', + }, + 'fill_missing_tags': True, + } + + # Mock get to return existing data + redis_activity.redis_repository.get = Mock(return_value=existing_data) + redis_activity.redis_repository.set = Mock() + + # Call the method + result = redis_activity.group_and_hold_data(test_data) + + # Verify the result + expected_result = { + 'timestamp': { + 0: '2023-01-01 12:00:00', + 1: '2023-01-01 12:00:00', + 2: '2023-01-01 12:00:00', + 3: '2023-01-01 12:00:00', + }, + 'variable': {0: 'sensor1', 1: 'sensor2', 2: 'sensor3', 3: 'sensor4'}, + 'value': {0: 25.5, 1: 28.0, 2: 42.0, 3: None}, + 'model_id': {0: 1, 1: 1, 2: 1, 3: 1}, + } + assert result == expected_result + + # Verify set was called with correct arguments + redis_activity.redis_repository.set.assert_called_once_with( + 'held_data_test_workflow_test_schedule', + { + 'sensor1': 25.5, + 'sensor2': 28.0, + 'sensor3': 42.0, + 'sensor4': None, + 'timestamp': '2023-01-01 12:00:00', + }, + ttl=3600, + metadata=metadata['metadata'], + ) + + +def test_group_and_hold_data_with_none_values(redis_activity): + """Test handling of None values in group_and_hold_data""" + # Setup test data with None values + test_data = { + **metadata, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'retention_time': 3600, + 'model_id': 1, + 'data': DataFrame( + { + 'name': ['sensor1', 'sensor2'], + 'value': [None, 30.0], + 'timestamp': [datetime(2023, 1, 1, 12, 0, 0)] * 2, + } + ).to_dict('records'), + 'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'}, + 'fill_missing_tags': False, + } + + # Mock get to return None for new key + redis_activity.redis_repository.get = Mock(return_value=None) + redis_activity.redis_repository.set = Mock() + + # Call the method + result = redis_activity.group_and_hold_data(test_data) + + # Verify None was converted to np.nan and values are as expected + assert np.isnan(result['value'][0]) + assert result['value'][1] == pytest.approx(30.0) + + +def test_group_and_hold_data_empty_dataframe(redis_activity): + """Test group_and_hold_data with empty DataFrame""" + # Setup test with empty data + test_data = { + **metadata, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'retention_time': 3600, + 'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'), + 'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'}, + 'fill_missing_tags': False, + } + + redis_activity.redis_repository.get = Mock(return_value=None) + + # Call the method + result = redis_activity.group_and_hold_data(test_data) + + assert result == {} + + +def test_group_and_hold_data_error_get(redis_activity): + """Test group_and_hold_data error""" + test_data = { + **metadata, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'retention_time': 3600, + 'model_id': 1, + 'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'), + 'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'}, + 'fill_missing_tags': False, + } + + redis_activity.redis_repository.get = Mock(side_effect=Exception('test')) + redis_activity.send_notification = Mock() + + try: + redis_activity.group_and_hold_data(test_data) + + except Exception as e: + assert str(e) == 'test' + + redis_activity.send_notification.assert_called_once_with( + metadata=metadata['metadata'], + notification_id='REDIS_GET_ERROR', + message='Error getting held data: test', + block='group_and_hold_data', + level=NotificationLevel.ERROR, + attachment_content=ANY, + ) + + else: + raise AssertionError('Expected exception') + + +def test_group_and_hold_data_error_set(redis_activity): + """Test group_and_hold_data error""" + test_data = { + **metadata, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'retention_time': 3600, + 'model_id': 1, + 'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'), + 'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'}, + 'fill_missing_tags': False, + } + + existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'} + + # Mock get to return existing data + redis_activity.redis_repository.get = Mock(return_value=existing_data) + redis_activity.redis_repository.set = Mock(side_effect=Exception('test')) + redis_activity.send_notification = Mock() + + try: + redis_activity.group_and_hold_data(test_data) + + except Exception as e: + assert str(e) == 'test' + + +def test_store_data_package(redis_activity): + """Test store_data_package""" + redis_activity.redis_repository.set = Mock() + + test_data = { + **metadata, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'held_data': DataFrame( + { + 'name': ['sensor1', 'sensor2'], + 'value': [25.5, 30.0], + 'timestamp': ['2023-01-01 12:00:00'] * 2, + } + ).to_dict(), + 'data': DataFrame( + { + 'name': ['sensor1', 'sensor2'], + 'value': [25.5, 30.0], + 'timestamp': ['2023-01-01 12:00:00'] * 2, + } + ).to_dict(), + 'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'}, + } + + redis_activity.store_data_package(test_data) + + redis_activity.redis_repository.set.assert_called_once_with( + ANY, + {'data': test_data['data'], 'held_data': test_data['held_data']}, + ttl=120, + metadata=metadata['metadata'], + ) + + +def test_store_data_package_error(redis_activity): + """Test store_data_package error""" + redis_activity.redis_repository.set = Mock(side_effect=ValueError('test')) + redis_activity.send_notification = Mock() + + test_data = { + **metadata, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'held_data': DataFrame( + { + 'name': ['sensor1', 'sensor2'], + 'value': [25.5, 30.0], + 'timestamp': ['2023-01-01 12:00:00'] * 2, + } + ).to_dict(), + 'data': DataFrame( + { + 'name': ['sensor1', 'sensor2'], + 'value': [25.5, 30.0], + 'timestamp': ['2023-01-01 12:00:00'] * 2, + } + ).to_dict(), + 'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'}, + } + + with pytest.raises(ValueError): + redis_activity.store_data_package(test_data) + + redis_activity.send_notification.assert_called_once_with( + metadata=metadata['metadata'], + notification_id='REDIS_SET_ERROR', + message='Error setting data package: test', + block='store_data_package', + level=NotificationLevel.ERROR, + attachment_content=ANY, + ) diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..be7b4f3 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,32 @@ +# tests/unit/test_metrics.py + +from prometheus_client import Counter, Gauge + +import scouter.metrics as metrics + +# --- Test Functions for Each Metric (Corrected for v0.22.0 _name behavior) --- + + +def test_scouter_laborious_data_written_count(): + """Verify the definition of LABORIOUS_DATA_WRITTEN_COUNT.""" + assert metrics.LABORIOUS_DATA_WRITTEN_COUNT is not None + assert isinstance(metrics.LABORIOUS_DATA_WRITTEN_COUNT, Counter) + assert metrics.LABORIOUS_DATA_WRITTEN_COUNT._name == 'scouter_laborious_data_written_count' + assert set(metrics.LABORIOUS_DATA_WRITTEN_COUNT._labelnames) == { + 'pod_id', + 'model_name', + 'workflow_name', + } + + +def test_scouter_tag_changes_monitor(): + """Verify the definition of TAG_CHANGES_MONITOR.""" + assert metrics.TAG_CHANGES_MONITOR is not None + assert isinstance(metrics.TAG_CHANGES_MONITOR, Gauge) + assert metrics.TAG_CHANGES_MONITOR._name == 'scouter_tag_changes_monitor' + assert set(metrics.TAG_CHANGES_MONITOR._labelnames) == { + 'pod_id', + 'model_name', + 'workflow_name', + 'tag_name', + } diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/utils/quality/test_filters.py b/tests/utils/quality/test_filters.py new file mode 100644 index 0000000..ca3079e --- /dev/null +++ b/tests/utils/quality/test_filters.py @@ -0,0 +1,139 @@ +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal + +from scouter.utils.quality.filters import check_data_range, null_values_filter, out_of_bounds_filter + +# Fixtures + + +@pytest.fixture +def sample_dataframe(): + """Fixture providing a sample DataFrame for testing.""" + return pd.DataFrame( + { + 'tag': ['temp', 'temp', 'pressure', 'pressure', 'humidity', 'wind_speed'], + 'name': ['temp', 'temp', 'pressure', 'pressure', 'humidity', 'wind_speed'], + 'value': [25, 35, 95, 105, 60, None], + 'timestamp': pd.date_range(start='2023-01-01', periods=6), + } + ) + + +@pytest.fixture +def nodes_data_range(): + """Fixture providing data ranges for different tags.""" + return { + 'temp': {'data_range': [10, 30]}, + 'pressure': {'data_range': [90, 100]}, + 'humidity': {'data_range': [40, 80]}, + 'wind_speed': {'data_range': [0, 50]}, + } + + +# Parameterized test data +CHECK_DATA_RANGE_CASES = [ + # (value, val_range, expected) + # Values within range + (5, [0, 10], False), + (0, [0, 10], False), # Edge case: value equals lower bound + (10, [0, 10], False), # Edge case: value equals upper bound + # Values outside range + (-1, [0, 10], True), + (11, [0, 10], True), + # Single value range + (5, [5, 5], False), + (4, [5, 5], True), + # Empty or None value + (None, [0, 10], True), + (np.nan, [0, 10], True), +] + +# Tests for check_data_range + + +@pytest.mark.parametrize('value,val_range,expected', CHECK_DATA_RANGE_CASES) +def test_check_data_range(value, val_range, expected): + """Test the check_data_range function with various input scenarios.""" + result = check_data_range(value, val_range) + if isinstance(value, float) and np.isnan(value): + assert result is True + else: + assert result == expected + + +# Tests for out_of_bounds_filter + + +def test_out_of_bounds_filter(sample_dataframe, nodes_data_range): + """Test filtering out-of-bounds values from a DataFrame.""" + # Expected result: rows where value is outside the defined range + expected_data = { + 'tag': ['temp', 'pressure', 'wind_speed'], + 'name': ['temp', 'pressure', 'wind_speed'], + 'value': [35, 105, None], + 'timestamp': [ + pd.Timestamp('2023-01-02'), + pd.Timestamp('2023-01-04'), + pd.Timestamp('2023-01-06'), + ], + } + expected_df = pd.DataFrame(expected_data) + + result = out_of_bounds_filter(sample_dataframe, nodes_data_range) + result = result.reset_index(drop=True) + expected_df = expected_df.reset_index(drop=True) + + assert_frame_equal(result, expected_df) + + +def test_out_of_bounds_filter_empty_df(nodes_data_range): + """Test with an empty DataFrame.""" + df = pd.DataFrame(columns=['tag', 'name', 'value', 'timestamp']) + result = out_of_bounds_filter(df, nodes_data_range) + assert result.empty + assert list(result.columns) == ['tag', 'name', 'value', 'timestamp'] + + +# Tests for null_values_filter + + +def test_null_values_filter(sample_dataframe, nodes_data_range): + """Test filtering null values from a DataFrame.""" + expected_data = { + 'tag': ['wind_speed'], + 'name': ['wind_speed'], + 'value': [np.nan], + 'timestamp': [pd.Timestamp('2023-01-06')], + } + expected_df = pd.DataFrame(expected_data) + + result = null_values_filter(sample_dataframe, nodes_data_range) + result = result.reset_index(drop=True) + expected_df = expected_df.reset_index(drop=True) + + assert_frame_equal(result, expected_df, check_dtype=False) + + +def test_null_values_filter_no_nulls(nodes_data_range): + """Test with a DataFrame containing no null values.""" + df = pd.DataFrame( + { + 'tag': ['temp', 'pressure'], + 'name': ['temp', 'pressure'], + 'value': [25, 100], + 'timestamp': pd.date_range(start='2023-01-01', periods=2), + } + ) + result = null_values_filter(df, nodes_data_range) + assert result.empty + assert list(result.columns) == ['tag', 'name', 'value', 'timestamp'] + + +def test_null_values_filter_empty_df(nodes_data_range): + """Test with an empty DataFrame.""" + df = pd.DataFrame(columns=['tag', 'name', 'value', 'timestamp']) + result = null_values_filter(df, nodes_data_range) + assert result.empty + assert list(result.columns) == ['tag', 'name', 'value', 'timestamp'] diff --git a/tests/utils/test_connectors_config.py b/tests/utils/test_connectors_config.py new file mode 100644 index 0000000..fe24347 --- /dev/null +++ b/tests/utils/test_connectors_config.py @@ -0,0 +1,42 @@ +import os +from unittest.mock import patch + +import pytest + +from scouter.utils.connectors_config import ( + build_kafka_config, +) + + +@pytest.fixture +def mock_env_vars(): + with patch.dict(os.environ, {}, clear=True): + yield + + +@pytest.mark.usefixtures('mock_env_vars') +def test_build_kafka_config_defaults(): + """Test that build_kafka_config returns default values when no env vars are set""" + config = build_kafka_config() + + assert config == { + 'bootstrap_servers': 'localhost:9092', + 'polling_time': 1000, + 'group_id': 'scouter-group', + } + + +@pytest.mark.usefixtures('mock_env_vars') +def test_build_kafka_config_with_env_vars(): + """Test that build_kafka_config uses env vars when set""" + with patch.dict( + os.environ, + {'KAFKA_BOOTSTRAP_SERVERS': 'kafka.example.com:9092', 'KAFKA_POLLING_TIME': '5000'}, + ): + config = build_kafka_config() + + assert config == { + 'bootstrap_servers': 'kafka.example.com:9092', + 'polling_time': 5000, + 'group_id': 'scouter-group', + } diff --git a/tests/worker/__init__.py b/tests/worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/workflow/__init__.py b/tests/workflow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/workflow/sub_workflows/test_core_scouter.py b/tests/workflow/sub_workflows/test_core_scouter.py new file mode 100644 index 0000000..387118d --- /dev/null +++ b/tests/workflow/sub_workflows/test_core_scouter.py @@ -0,0 +1,403 @@ +from unittest.mock import ANY, AsyncMock, call, patch + +import pytest +from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ + +from scouter.activities.activities import Activities +from scouter.workflow.sub_workflows.core_scouter import CoreScouter + + +@pytest.fixture +def core_scouter(): + return CoreScouter() + + +@pytest.mark.asyncio +@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock) +async def test_core_scouter_workflow_success(mock_workflow, core_scouter): + mock_workflow.execute_local_activity_method.side_effect = [ + 'filtered_data', + 'grouped_data', + 'held_data', + ] + mock_workflow.execute_activity_method.side_effect = [ + {'affected_rows': 10}, # export_data_to_postgres + None, # write_metrics + None, # store_data_package + ] + await core_scouter.run( + input_data={ + 'metadata': { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow', + } + }, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'model_name': 'test_model', + 'model_id': 'test_model_id', + 'data': 'test_data', + 'trigger_laborious': False, + 'filters': {'test_filter': 'test_value'}, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'retention_time': 3600, + 'model_tags': {}, + 'debug_data_package': True, + 'fill_missing_tags': False, + } + ) + + expected_metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow', + } + } + + mock_workflow.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.data_quality_gate, + { + **expected_metadata, + 'filters': {'test_filter': 'test_value'}, + 'data': 'test_data', + 'model_tags': {}, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + mock_workflow.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.aggregate_data, + {**expected_metadata, 'data': 'filtered_data', 'model_tags': {}}, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + mock_workflow.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.group_and_hold_data, + { + **expected_metadata, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'data': 'grouped_data', + 'model_id': 'test_model_id', + 'retention_time': 3600, + 'model_tags': {}, + 'fill_missing_tags': False, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + mock_workflow.execute_activity_method.assert_has_calls( + [ + call( + Activities.export_data_to_postgres, + { + **expected_metadata, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'data': 'held_data', + 'on_conflict': 'ignore', + 'unique_columns': ['model_id', 'timestamp', 'variable'], + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT_WITH_TZ, + }, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + mock_workflow.execute_activity_method.assert_has_calls( + [ + call( + Activities.write_metrics, + { + **expected_metadata, + 'tag_values': 'held_data', + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + mock_workflow.execute_activity_method.assert_has_calls( + [ + call( + Activities.store_data_package, + { + **expected_metadata, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'held_data': 'held_data', + 'data': 'test_data', + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + +@pytest.mark.asyncio +@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock) +async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter): + mock_workflow.execute_local_activity_method.return_value = {} + await core_scouter.run( + input_data={ + 'metadata': { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow', + } + }, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'model_name': 'test_model', + 'model_id': 'test_model_id', + 'data': 'test_data', + 'trigger_laborious': False, + 'filters': {'test_filter': 'test_value'}, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'retention_time': 3600, + 'model_tags': {}, + 'fill_missing_tags': False, + } + ) + + expected_metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow', + } + } + + mock_workflow.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.data_quality_gate, + { + **expected_metadata, + 'filters': {'test_filter': 'test_value'}, + 'data': 'test_data', + 'model_tags': {}, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + mock_workflow.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.aggregate_data, + {**expected_metadata, 'data': {}, 'model_tags': {}}, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + mock_workflow.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.group_and_hold_data, + { + **expected_metadata, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'data': {}, + 'model_id': 'test_model_id', + 'retention_time': 3600, + 'model_tags': {}, + 'fill_missing_tags': False, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + assert mock_workflow.execute_local_activity_method.call_count == 3 + mock_workflow.execute_activity_method.assert_not_called() + + +@pytest.mark.asyncio +@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock) +async def test_core_scouter_workflow_with_zero_affected_rows(mock_workflow, core_scouter): + """ + Test that workflow stops after export when no rows are affected + """ + mock_workflow.execute_local_activity_method.side_effect = [ + 'filtered_data', + 'grouped_data', + 'held_data', + ] + mock_workflow.execute_activity_method.return_value = {'affected_rows': 0} + + await core_scouter.run( + input_data={ + 'metadata': { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow', + } + }, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'model_name': 'test_model', + 'model_id': 'test_model_id', + 'data': 'test_data', + 'trigger_laborious': False, + 'filters': {'test_filter': 'test_value'}, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'retention_time': 3600, + 'model_tags': {}, + 'debug_data_package': True, + 'fill_missing_tags': False, + } + ) + + expected_metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow', + } + } + + mock_workflow.execute_activity_method.assert_called_once_with( + Activities.export_data_to_postgres, + { + **expected_metadata, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'data': 'held_data', + 'on_conflict': 'ignore', + 'unique_columns': ['model_id', 'timestamp', 'variable'], + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT_WITH_TZ, + }, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + + +@pytest.mark.asyncio +@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock) +async def test_core_scouter_workflow_without_debug_data_package(mock_workflow, core_scouter): + """ + Test that store_data_package is not called when debug_data_package is False + """ + mock_workflow.execute_local_activity_method.side_effect = [ + 'filtered_data', + 'grouped_data', + 'held_data', + ] + mock_workflow.execute_activity_method.side_effect = [ + {'affected_rows': 5}, + None, + ] + + await core_scouter.run( + input_data={ + 'metadata': { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow', + } + }, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'model_name': 'test_model', + 'model_id': 'test_model_id', + 'data': 'test_data', + 'trigger_laborious': False, + 'filters': {'test_filter': 'test_value'}, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'retention_time': 3600, + 'model_tags': {}, + 'debug_data_package': False, + 'fill_missing_tags': False, + } + ) + + expected_metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow', + } + } + + mock_workflow.execute_activity_method.assert_has_calls( + [ + call( + Activities.export_data_to_postgres, + { + **expected_metadata, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'data': 'held_data', + 'on_conflict': 'ignore', + 'unique_columns': ['model_id', 'timestamp', 'variable'], + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT_WITH_TZ, + }, + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + mock_workflow.execute_activity_method.assert_has_calls( + [ + call( + Activities.write_metrics, + { + **expected_metadata, + 'tag_values': 'held_data', + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + assert mock_workflow.execute_activity_method.call_count == 2 diff --git a/tests/workflow/test_pi_web_api_scouter.py b/tests/workflow/test_pi_web_api_scouter.py new file mode 100644 index 0000000..c9fa59a --- /dev/null +++ b/tests/workflow/test_pi_web_api_scouter.py @@ -0,0 +1,133 @@ +from unittest.mock import ANY, AsyncMock, patch + +from pytest import fixture, mark + +from scouter.activities.activities import Activities +from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter + + +@fixture +def pi_web_api_scouter(): + return PIWebAPIScouter() + + +@mark.asyncio +@patch('scouter.workflow.pi_web_api_scouter.workflow', new_callable=AsyncMock) +async def test_pi_web_api_scouter_workflow(mock_workflow, pi_web_api_scouter): + mock_workflow.execute_local_activity_method.return_value = 'test_data' + await pi_web_api_scouter.run( + input_data={ + 'model_name': 'test_model', + 'model_id': 'test_model_id', + 'schedule_name': 'test_schedule', + 'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'}, + 'trigger_laborious': True, + 'filters': {'quality': 'good'}, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'retention_time': 3600, + 'pi_web_api_query': { + 'endpoint': '/streamsets/recorded', + 'period': '*-1d', + 'max_count': 10, + 'api_timeout': 30, + }, + } + ) + + expected_metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'pi_web_api_scouter', + } + } + + mock_workflow.execute_local_activity_method.assert_called_once_with( + Activities.get_tag_values, + { + **expected_metadata, + 'endpoint': '/streamsets/recorded', + 'web_ids': {'tag1': 'webid1', 'tag2': 'webid2'}, + 'period': '*-1d', + 'api_timeout': 30, + 'max_count': 10, + }, + start_to_close_timeout=ANY, + retry_policy=ANY, + ) + + mock_workflow.execute_child_workflow.assert_called_once_with( + 'subworkflow.core_scouter', + { + 'model_name': 'test_model', + 'model_id': 'test_model_id', + 'schedule_name': 'test_schedule', + 'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'}, + 'trigger_laborious': True, + 'filters': {'quality': 'good'}, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'retention_time': 3600, + 'workflow_name': 'pi_web_api_scouter', + 'data': 'test_data', + 'metadata': expected_metadata, + 'pi_web_api_query': { + 'endpoint': '/streamsets/recorded', + 'period': '*-1d', + 'max_count': 10, + 'api_timeout': 30, + }, + }, + ) + + +@mark.asyncio +@patch('scouter.workflow.pi_web_api_scouter.workflow', new_callable=AsyncMock) +async def test_pi_web_api_scouter_workflow_empty(mock_workflow, pi_web_api_scouter): + mock_workflow.execute_local_activity_method.return_value = [] + await pi_web_api_scouter.run( + input_data={ + 'model_name': 'test_model', + 'model_id': 'test_model_id', + 'schedule_name': 'test_schedule', + 'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'}, + 'trigger_laborious': True, + 'filters': {'quality': 'good'}, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'retention_time': 3600, + 'pi_web_api_query': { + 'endpoint': '/streamsets/recorded', + 'period': '*-1d', + 'max_count': 1, + 'api_timeout': 30, + }, + } + ) + + expected_metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'pi_web_api_scouter', + } + } + + mock_workflow.execute_local_activity_method.assert_called_once_with( + Activities.get_tag_values, + { + **expected_metadata, + 'web_ids': {'tag1': 'webid1', 'tag2': 'webid2'}, + 'period': '*-1d', + 'api_timeout': 30, + 'max_count': 1, + 'endpoint': '/streamsets/recorded', + }, + start_to_close_timeout=ANY, + retry_policy=ANY, + ) + + mock_workflow.execute_child_workflow.assert_not_called() diff --git a/tests/workflow/test_scouter.py b/tests/workflow/test_scouter.py new file mode 100644 index 0000000..ba47683 --- /dev/null +++ b/tests/workflow/test_scouter.py @@ -0,0 +1,129 @@ +from unittest.mock import ANY, AsyncMock, call, patch + +from pytest import fixture, mark + +from scouter.activities.activities import Activities +from scouter.workflow.scouter import Scouter + + +@fixture +def scouter(): + return Scouter() + + +@mark.asyncio +@patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock) +async def test_scouter_workflow(mock_workflow, scouter): + mock_workflow.execute_local_activity_method.side_effect = [ + 'test_last_data_timestamp', + 'test_data', + ] + await scouter.run( + input_data={ + 'topic': 'test_topic', + 'schedule_name': 'test_schedule', + 'model_name': 'test_model', + 'model_id': 'test_model_id', + } + ) + + expected_metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'scouter', + } + } + + mock_workflow.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.get_last_data_timestamp, + {**expected_metadata, 'workflow_name': 'scouter', 'schedule_name': 'test_schedule'}, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + mock_workflow.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.load_latest_data, + { + **expected_metadata, + 'collection_name': 'raw_test_schedule', + 'last_data_timestamp': 'test_last_data_timestamp', + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + mock_workflow.execute_activity_method.assert_called_once_with( + Activities.put_last_data_timestamp, + { + **expected_metadata, + 'data': 'test_data', + 'workflow_name': 'scouter', + 'schedule_name': 'test_schedule', + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + + mock_workflow.execute_child_workflow.assert_called_once_with( + 'subworkflow.core_scouter', + { + 'metadata': expected_metadata, + 'topic': 'test_topic', + 'data': 'test_data', + 'workflow_name': 'scouter', + 'schedule_name': 'test_schedule', + 'model_name': 'test_model', + 'model_id': 'test_model_id', + }, + ) + + +@mark.asyncio +@patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock) +async def test_scouter_workflow_empty(mock_workflow, scouter): + mock_workflow.execute_local_activity_method.side_effect = ['test_last_data_timestamp', {}] + await scouter.run( + input_data={ + 'topic': 'test_topic', + 'schedule_name': 'test_schedule', + 'model_name': 'test_model', + 'model_id': 'test_model_id', + } + ) + + expected_metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'scouter', + } + } + + mock_workflow.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.load_latest_data, + { + **expected_metadata, + 'collection_name': 'raw_test_schedule', + 'last_data_timestamp': 'test_last_data_timestamp', + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + mock_workflow.execute_activity_method.assert_not_called() + mock_workflow.execute_child_workflow.assert_not_called()