From 34dbc886f34d155e7d249a8dc5b88fb8ca6f3bd7 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 25 May 2026 12:58:03 -0300 Subject: [PATCH] SIENTIAPDE-1646 Update project configuration and dependencies - Added .mypy_cache and .cursor to .gitignore. - Changed asyncio_default_fixture_loop_scope and asyncio_default_test_loop_scope to "session" in pyproject.toml. - Updated e2e testing dependencies in requirements-dev.txt, replacing fakeredis and mongomock with pytest-httpserver. - Updated requirements.txt to use sientia_do instead of a specific git commit. - Modified sonar-project.properties to remove a file from coverage exclusions. - Enhanced E2E test fixtures in e2e/conftest.py for better container management. - Cleaned up e2e test files related to CoreScouter and PIWebAPIScouter workflows. --- .gitignore | 5 +- e2e/README.md | 37 ++ e2e/conftest.py | 555 ++++++++---------- e2e/db_schema.sql | 16 + e2e/fixtures/fake_mongodb_repository.py | 241 -------- e2e/fixtures/fake_redis_repository.py | 170 ------ e2e/helpers.py | 404 +++++++++++++ e2e/pi_web_api_test_server.py | 124 ++++ .../core_scouter_aggregation_avg.json | 24 + .../core_scouter_aggregation_lts.json | 24 + .../core_scouter_aggregation_max.json | 24 + .../core_scouter_aggregation_mdn.json | 24 + .../core_scouter_aggregation_min.json | 24 + .../core_scouter_debug_data_package.json | 22 + .../core_scouter_empty_after_grouping.json | 24 + .../core_scouter_fill_missing_tags.json | 24 + .../core_scouter_happy_path.json | 28 + .../core_scouter_invalid_aggregation.json | 23 + ...re_scouter_null_values_filter_discard.json | 23 + .../core_scouter_null_values_filter_warn.json | 23 + ..._scouter_out_of_bounds_filter_discard.json | 23 + .../core_scouter_postgres_export_failure.json | 21 + .../pi_web_api_scouter_connection_error.json | 23 + ...pi_web_api_scouter_debug_data_package.json | 29 + .../pi_web_api_scouter_empty_response.json | 23 + .../pi_web_api_scouter_happy_path.json | 30 + .../pi_web_api_scouter_invalid_endpoint.json | 23 + .../pi_web_api_scouter_multiple_tags.json | 53 ++ .../pi_web_api_scouter_timeout.json | 23 + e2e/scenario_inputs/scouter_empty_mongo.json | 23 + e2e/scenario_inputs/scouter_happy_path.json | 38 ++ .../scouter_incremental_load.json | 43 ++ .../scouter_no_redis_timestamp_first_run.json | 31 + e2e/scenarios.md | 459 +++------------ e2e/test_core_scouter_early_exit.py | 96 --- e2e/test_core_scouter_errors.py | 90 --- e2e/test_core_scouter_success.py | 462 --------------- e2e/test_harness_smoke.py | 51 ++ e2e/test_pi_web_api_scouter_early_exit.py | 182 ------ e2e/test_pi_web_api_scouter_errors.py | 206 ------- e2e/test_pi_web_api_scouter_main_workflow.py | 222 +++++++ e2e/test_pi_web_api_scouter_success.py | 298 ---------- e2e/test_scouter_main_workflow.py | 154 +++++ e2e/test_subworkflow_core_scouter.py | 360 ++++++++++++ pyproject.toml | 4 +- requirements-dev.txt | 5 +- requirements-local.txt | 8 + requirements.txt | 2 +- scouter/activities/activities.py | 2 +- scouter/activities/api.py | 8 +- scouter/activities/gates.py | 50 +- scouter/activities/mongodb.py | 8 +- scouter/activities/redis.py | 32 +- scouter/worker/prepare_worker.py | 74 --- scouter/worker/worker.py | 22 +- scouter/worker/worker_parameters.md | 113 ---- sonar-project.properties | 2 +- tests.ipynb | 4 +- tests/activities/test_activities.py | 13 +- tests/activities/test_api.py | 41 +- tests/activities/test_gates.py | 85 ++- tests/activities/test_mongo.py | 33 +- tests/activities/test_redis.py | 122 ++-- tests/utils/quality/test_filters.py | 2 +- values.yaml | 8 +- 65 files changed, 2591 insertions(+), 2849 deletions(-) create mode 100644 e2e/README.md create mode 100644 e2e/db_schema.sql delete mode 100644 e2e/fixtures/fake_mongodb_repository.py delete mode 100644 e2e/fixtures/fake_redis_repository.py create mode 100644 e2e/helpers.py create mode 100644 e2e/pi_web_api_test_server.py create mode 100644 e2e/scenario_inputs/core_scouter_aggregation_avg.json create mode 100644 e2e/scenario_inputs/core_scouter_aggregation_lts.json create mode 100644 e2e/scenario_inputs/core_scouter_aggregation_max.json create mode 100644 e2e/scenario_inputs/core_scouter_aggregation_mdn.json create mode 100644 e2e/scenario_inputs/core_scouter_aggregation_min.json create mode 100644 e2e/scenario_inputs/core_scouter_debug_data_package.json create mode 100644 e2e/scenario_inputs/core_scouter_empty_after_grouping.json create mode 100644 e2e/scenario_inputs/core_scouter_fill_missing_tags.json create mode 100644 e2e/scenario_inputs/core_scouter_happy_path.json create mode 100644 e2e/scenario_inputs/core_scouter_invalid_aggregation.json create mode 100644 e2e/scenario_inputs/core_scouter_null_values_filter_discard.json create mode 100644 e2e/scenario_inputs/core_scouter_null_values_filter_warn.json create mode 100644 e2e/scenario_inputs/core_scouter_out_of_bounds_filter_discard.json create mode 100644 e2e/scenario_inputs/core_scouter_postgres_export_failure.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_connection_error.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_debug_data_package.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_empty_response.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_happy_path.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_invalid_endpoint.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_multiple_tags.json create mode 100644 e2e/scenario_inputs/pi_web_api_scouter_timeout.json create mode 100644 e2e/scenario_inputs/scouter_empty_mongo.json create mode 100644 e2e/scenario_inputs/scouter_happy_path.json create mode 100644 e2e/scenario_inputs/scouter_incremental_load.json create mode 100644 e2e/scenario_inputs/scouter_no_redis_timestamp_first_run.json delete mode 100644 e2e/test_core_scouter_early_exit.py delete mode 100644 e2e/test_core_scouter_errors.py delete mode 100644 e2e/test_core_scouter_success.py create mode 100644 e2e/test_harness_smoke.py delete mode 100644 e2e/test_pi_web_api_scouter_early_exit.py delete mode 100644 e2e/test_pi_web_api_scouter_errors.py create mode 100644 e2e/test_pi_web_api_scouter_main_workflow.py delete mode 100644 e2e/test_pi_web_api_scouter_success.py create mode 100644 e2e/test_scouter_main_workflow.py create mode 100644 e2e/test_subworkflow_core_scouter.py create mode 100644 requirements-local.txt delete mode 100644 scouter/worker/prepare_worker.py delete mode 100644 scouter/worker/worker_parameters.md diff --git a/.gitignore b/.gitignore index b9198c7..7cebf07 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,7 @@ git_log .env .ruff_cache/ -.mypy_cache/ \ No newline at end of file +.mypy_cache/ + +.cursor +openspec \ No newline at end of file diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000..05f8b33 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,37 @@ +# Scouter end-to-end tests + +Production-faithful E2E tests for `Scouter`, `PIWebAPIScouter`, and `CoreScouter` against real backing services. + +## Requirements + +- Docker (for testcontainers: MongoDB, Redis, PostgreSQL) +- Python 3.11+ with dev dependencies: `pip install -r requirements-dev.txt` + +## Run locally + +```bash +pytest e2e/ --override-ini testpaths=e2e -m e2e -v +``` + +Stop on first failure: + +```bash +pytest e2e/ --override-ini testpaths=e2e -m e2e -x +``` + +## Coverage (separate from unit tests) + +```bash +COVERAGE_FILE=.coverage.e2e pytest e2e/ --override-ini testpaths=e2e -m e2e --cov=scouter --cov-branch +coverage combine .coverage .coverage.e2e && coverage report +``` + +## Scenario catalog + +See [scenarios.md](scenarios.md) for numbered scenarios and how they map to `test_scenario_*` functions. Section `## 0` of that file lists the harness smoke tests in `test_harness_smoke.py` (infra liveness checks, not business scenarios). + +## Production code is not mocked + +E2E uses real testcontainers, an in-process PI Web API HTTP server, `WorkflowEnvironment.start_local()`, and production `Activities` wiring. The only stand-ins are `mock_logger` and the optional `notification_inserts` spy. If a scenario fails due to a production defect, it is marked `xfail(strict=True)` and tracked in `openspec/changes/standardize-and-complete-e2e-tests/notes.md` when applicable. + +Unit tests under `tests/` remain Docker-free and run with the default `pytest` invocation. diff --git a/e2e/conftest.py b/e2e/conftest.py index 34042fa..45b68f9 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -1,121 +1,177 @@ """ -Pytest configuration and fixtures for E2E tests. +Pytest configuration and fixtures for production-faithful Scouter E2E tests. """ -from typing import Any -from unittest.mock import AsyncMock, MagicMock, patch +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import MagicMock -import pandas as pd import pytest import pytest_asyncio -from pandas import DataFrame +from pymongo import MongoClient +from redis import Redis from sqlalchemy import create_engine, text -from sqlalchemy.orm import sessionmaker +from testcontainers.mongodb import MongoDbContainer from testcontainers.postgres import PostgresContainer +from testcontainers.redis import RedisContainer from temporalio.testing import WorkflowEnvironment -from temporalio.worker import Worker +from temporalio.worker import PollerBehaviorAutoscaling, Worker +from e2e.helpers import SCOUTER_TASK_QUEUE, postgres_connection_parts +from e2e.pi_web_api_test_server import PIWebAPITestServer from scouter.activities.activities import Activities from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter +from scouter.workflow.scouter import Scouter from scouter.workflow.sub_workflows.core_scouter import CoreScouter from sientia_do.notifications.handlers import CoreNotificationHandler from sientia_do.observability.logger import Logger -from sientia_do.observability.metrics_controller import MetricsController -from e2e.fixtures.fake_mongodb_repository import FakeMongoDBRepository -from e2e.fixtures.fake_redis_repository import FakeRedisRepository +E2E_DATABASE = 'scouter_e2e_test' +E2E_NOTIFICATION_COLLECTION = 'notification_queue' +DB_SCHEMA_PATH = Path(__file__).resolve().parent / 'db_schema.sql' -# Test constants -TEST_MONGODB_CONNECTION_STRING = 'mongodb://localhost:27017' -TEST_DATABASE_NAME = 'test_db' +def _activity_list(activities: Activities) -> list: + """Return bound activity callables for the E2E worker.""" + return [ + activities.load_latest_data, + activities.get_last_data_timestamp, + activities.put_last_data_timestamp, + activities.get_tag_values, + activities.data_quality_gate, + activities.aggregate_data, + activities.group_and_hold_data, + activities.export_data_to_postgres, + activities.write_metrics, + activities.store_data_package, + ] -@pytest_asyncio.fixture(scope='session') +@pytest.fixture(scope='session') def postgres_container(): """ - Create a PostgreSQL container using testcontainers. - - This fixture creates a real PostgreSQL database in a Docker container - that will be used for all tests in the session. + Session-scoped PostgreSQL testcontainer. + + Return: + Running PostgresContainer instance """ - postgres = PostgresContainer('postgres:15') - postgres.start() - yield postgres - postgres.stop() + container = PostgresContainer('postgres:15') + container.start() + yield container + container.stop() -@pytest_asyncio.fixture +@pytest.fixture(scope='session') +def mongo_container(): + """ + Session-scoped MongoDB testcontainer. + + Return: + Running MongoDbContainer instance + """ + container = MongoDbContainer('mongo:7') + container.start() + yield container + container.stop() + + +@pytest.fixture(scope='session') +def redis_container(): + """ + Session-scoped Redis testcontainer. + + Return: + Running RedisContainer instance + """ + container = RedisContainer('redis:7') + container.start() + yield container + container.stop() + + +@pytest.fixture(scope='session') def postgres_engine(postgres_container): """ - Create SQLAlchemy engine for PostgreSQL test database. - - This fixture creates a connection to the PostgreSQL container - created by the postgres_container fixture. + SQLAlchemy engine bound to the Postgres testcontainer for the session. + + Return: + SQLAlchemy Engine """ - # Get connection URL from container - connection_string = postgres_container.get_connection_url() - engine = create_engine(connection_string) - + engine = create_engine(postgres_container.get_connection_url()) yield engine - engine.dispose() -def _create_schema_and_table(engine): +@pytest.fixture(scope='session') +def mongo_uri(mongo_container): """ - Helper function to create schema and table in the given engine. - - This is used by both the autouse fixture and test_activities to ensure - the schema exists before Activities tries to use it. - - Note: For tests, we create a non-partitioned table to avoid issues - with pandas to_sql recognizing partitioned tables. + MongoDB connection string for the testcontainer. + + Return: + Connection URI string """ - schema_name = 'sientia_data' - table_name = 'laborious_data' - - # Use begin() to ensure transaction is properly committed - with engine.begin() as conn: - # Create schema - conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {schema_name}")) - - # Create table WITHOUT partitioning (simpler for tests) - # Same structure as production, but without PARTITION BY RANGE - create_table_sql = f""" - CREATE TABLE IF NOT EXISTS {schema_name}.{table_name} ( - id SERIAL NOT NULL, - model_id int4 NOT NULL, - variable text NOT NULL, - value numeric NULL, - "timestamp" timestamptz NOT NULL, - created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL, - PRIMARY KEY (id, created_at) - ); - """ - - conn.execute(text(create_table_sql)) - # Transaction is automatically committed when exiting the 'with' block + return mongo_container.get_connection_url() -@pytest_asyncio.fixture(autouse=True) +@pytest.fixture(scope='session') +def redis_client(redis_container): + """ + Redis client connected to the testcontainer. + + Return: + redis.Redis client with decode_responses=True + """ + host = redis_container.get_container_host_ip() + port = int(redis_container.get_exposed_port(6379)) + client = Redis(host=host, port=port, decode_responses=True) + yield client + client.close() + + +@pytest.fixture(autouse=True) def setup_postgres_schema_and_table(postgres_engine): """ - Automatically create necessary schema and table before each test. - - This fixture runs automatically (autouse=True) and ensures - that the sientia_data schema and laborious_data table exist - with the correct structure before tests execute. - - Note: For tests, we use a non-partitioned table which is simpler and avoids issues - with pandas to_sql recognizing partitioned tables. + Apply db_schema.sql before each test so laborious_data is empty and current. """ - _create_schema_and_table(postgres_engine) + sql = DB_SCHEMA_PATH.read_text(encoding='utf-8') + with postgres_engine.begin() as conn: + conn.exec_driver_sql(sql) yield -@pytest_asyncio.fixture +@pytest.fixture(autouse=True) +def reset_mongo_collections(mongo_uri): + """ + Drop all collections in the E2E Mongo database between tests. + """ + client = MongoClient(mongo_uri) + try: + db = client[E2E_DATABASE] + for name in db.list_collection_names(): + db.drop_collection(name) + finally: + client.close() + yield + + +@pytest.fixture(autouse=True) +def reset_redis(redis_client): + """ + Flush the Redis testcontainer between tests. + """ + redis_client.flushdb() + yield + + +@pytest.fixture def mock_logger(): - """Mock logger for testing.""" + """ + Logger stand-in (only permitted MagicMock in the E2E harness). + + Return: + MagicMock with Logger spec + """ logger = MagicMock(spec=Logger) logger.info = MagicMock() logger.debug = MagicMock() @@ -125,238 +181,139 @@ def mock_logger(): return logger -@pytest_asyncio.fixture -def mock_mongo_client(): +@pytest.fixture +def notification_handler(mock_logger, mongo_uri): """ - Mock MongoDB client to avoid real connections. - - This fixture mocks the pymongo.MongoClient used by CoreNotificationHandler, - allowing us to use a real NotificationHandler instance without connecting to MongoDB. + Real CoreNotificationHandler backed by the Mongo testcontainer. + + Return: + CoreNotificationHandler instance """ - mock_client = MagicMock() - mock_db = MagicMock() - mock_collection = MagicMock() - - # Configure the mock chain: client[database] -> db[collection] -> collection - mock_client.__getitem__.return_value = mock_db - mock_db.__getitem__.return_value = mock_collection - - # Mock server_info() to avoid connection attempts - mock_client.server_info = MagicMock() - - # Mock insert_one for notifications - mock_collection.insert_one = MagicMock() - - return mock_client - - -@pytest_asyncio.fixture -def notification_handler(mock_logger, mock_mongo_client): - """ - Create a real NotificationHandler instance with mocked MongoDB client. - - This fixture creates a real CoreNotificationHandler instance but mocks - the underlying MongoDB connection to avoid real database connections. - """ - # Patch MongoClient where it's imported in the handlers module - with patch('sientia_do.notifications.handlers.MongoClient', return_value=mock_mongo_client): - handler = CoreNotificationHandler( - connection_string=TEST_MONGODB_CONNECTION_STRING, - database=TEST_DATABASE_NAME, - logger=mock_logger, - project_name='scouter', - ) - yield handler - handler.shutdown() - - -@pytest_asyncio.fixture -def metrics_controller(mock_logger): - """ - Create a real MetricsController instance. - - MetricsController doesn't require external services, so we can use - a real instance without mocking anything. - """ - controller = MetricsController(logger=mock_logger) - yield controller - # MetricsController might have cleanup, but it's optional - - -@pytest_asyncio.fixture -def mock_pi_web_api_client(): - """Mock PI Web API client.""" - mock_client = MagicMock() - - # Mock DataFrame response similar to real API - # The real PIWebAPIClient returns timestamp as datetime, so we need to match that - mock_df = DataFrame({ - 'timestamp': [ - '2024-01-01 12:00:00+0000', - '2024-01-01 12:01:00+0000', - '2024-01-01 12:02:00+0000', - ], - 'name': ['tag1', 'tag2', 'tag3'], - 'value': [10.5, 20.3, 30.7], - 'tag': ['webid1', 'webid2', 'webid3'], - }) - - # Convert timestamp to datetime (UTC, floored to seconds) to match real client behavior - mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp'], utc=True).dt.floor('s') - - mock_client.get_latest_values_df = AsyncMock(return_value=mock_df) - mock_client.close = MagicMock() - return mock_client - - -@pytest_asyncio.fixture(scope='function') -def test_activities( - postgres_engine, - postgres_container, - mock_logger, - notification_handler, - metrics_controller, - mock_pi_web_api_client, -): - """ - Create Activities instance with test dependencies. - - This fixture creates a real Activities instance with: - - PostgreSQL database (via testcontainers) - - FakeRedis instead of real Redis - - FakeMongoDB instead of real MongoDB - - Mocked PI Web API client - - Real NotificationHandler and MetricsController (with mocked underlying services) - """ - # Get connection details from container - connection_string = postgres_container.get_connection_url() - - # Parse connection string to get individual components - # Format: postgresql://testuser:testpass@localhost:5432/test - from urllib.parse import urlparse - parsed = urlparse(connection_string) - - # Ensure schema and table exist BEFORE creating Activities - # This ensures the schema exists when Activities initializes its engine - _create_schema_and_table(postgres_engine) - - # Create fake repositories that will be used from the start - fake_redis_repo = FakeRedisRepository( - host='localhost', - port=6379, - username='', - password='', + handler = CoreNotificationHandler( + connection_string=mongo_uri, + database=E2E_DATABASE, logger=mock_logger, - notification_handler=notification_handler, - metrics_controller=metrics_controller, + project_name='scouter-e2e', + notification_topic=E2E_NOTIFICATION_COLLECTION, ) - - fake_mongo_repo = FakeMongoDBRepository( - connection_string=TEST_MONGODB_CONNECTION_STRING, - database_name=TEST_DATABASE_NAME, - logger=mock_logger, - notification_handler=notification_handler, - metrics_controller=metrics_controller, - ) - - # Patch create_engine to return our postgres_engine instead of creating a new one - # This ensures Activities uses the same engine from the start - original_create_engine = create_engine - - def patched_create_engine(connection_string, *args, **kwargs): - # Check if this is the connection string that Activities would create - # Activities creates: postgresql://user:password@host:port/dbname - expected_conn_str = f"postgresql://{parsed.username or 'test'}:{parsed.password or 'test'}@localhost:{postgres_container.get_exposed_port(5432)}/{parsed.path.lstrip('/') if parsed.path else 'test'}" - - # If it matches our test container connection, return our engine - if connection_string == expected_conn_str: - return postgres_engine - # Otherwise, use the original create_engine - return original_create_engine(connection_string, *args, **kwargs) - - # Patch RedisRepository to return our fake repository from the start - def patched_redis_repository(*args, **kwargs): - return fake_redis_repo - - # Patch MongoDBRepository to return our fake repository from the start - def patched_mongodb_repository(*args, **kwargs): - return fake_mongo_repo - - # Patch PIWebAPIClient to return our mock from the start - def patched_pi_web_api_client(*args, **kwargs): - return mock_pi_web_api_client - - # Create Activities with test configurations - # All patches ensure it uses our test instances from the start - with patch('sientia_do.temporal.activities.postgres.create_engine', new=patched_create_engine), \ - patch('scouter.activities.redis.RedisRepository', new=patched_redis_repository), \ - patch('scouter.activities.mongodb.MongoDBRepository', new=patched_mongodb_repository), \ - patch('scouter.activities.api.PIWebAPIClient', new=patched_pi_web_api_client): - - activities = Activities( - postgres_config={ - 'host': 'localhost', # Container exposes to localhost - 'port': postgres_container.get_exposed_port(5432), - 'user': parsed.username or 'test', - 'password': parsed.password or 'test', - 'dbname': parsed.path.lstrip('/') if parsed.path else 'test', - 'min_connections': 1, - 'max_connections': 5, - }, - redis_config={ - 'host': 'localhost', - 'port': 6379, - 'username': '', - 'password': '', - }, - mongodb_config={ - 'connection_string': TEST_MONGODB_CONNECTION_STRING, - 'database_name': TEST_DATABASE_NAME, - }, - api_config={ - 'base_url': 'http://localhost:8080', - 'auth_type': 'bearer', - 'auth_token': 'test_token', - }, - logger=mock_logger, - notification_handler=notification_handler, - ) - - # Verify that Activities is using our instances from the start - assert activities.engine is postgres_engine, "Activities should use the same engine as postgres_engine" - assert activities.redis_repository is fake_redis_repo, "Activities should use the same fake Redis repository" - assert activities.mongodb_repository is fake_mongo_repo, "Activities should use the same fake MongoDB repository" - assert activities.pi_web_api_client is mock_pi_web_api_client, "Activities should use the same mock PI Web API client" - - yield activities - - # Cleanup - activities.shutdown() + yield handler + handler.shutdown() -@pytest_asyncio.fixture(scope='function') -async def temporal_test_env(): - """Create Temporal test environment.""" - env = await WorkflowEnvironment.start_time_skipping() - async with env: +@pytest.fixture +def notification_inserts(notification_handler): + """ + Spy wrapper around notification collection insert_one (still writes to Mongo). + + Return: + MagicMock wrapping insert_one + """ + collection = notification_handler.mongo_collection + spy = MagicMock(wraps=collection.insert_one) + collection.insert_one = spy + return spy + + +@pytest.fixture(scope='session') +def pi_web_api_server(): + """ + Session-scoped in-process PI Web API HTTP stub. + + Return: + Started PIWebAPITestServer instance + """ + server = PIWebAPITestServer() + server.start() + yield server + server.stop() + + +@pytest.fixture(autouse=True) +def cleanup_pi_web_api_server(pi_web_api_server): + """ + Reset PI Web API stub state between tests. + """ + pi_web_api_server.clear() + yield + + +@pytest_asyncio.fixture(scope='session') +async def temporal_env(): + """ + Session-scoped WorkflowEnvironment using the real local Temporal dev server. + + Return: + WorkflowEnvironment from start_local() + """ + async with await WorkflowEnvironment.start_local() as env: yield env -@pytest_asyncio.fixture(scope='function') -async def temporal_worker(temporal_test_env, test_activities): - """Create Temporal worker with test activities.""" +@pytest.fixture +def test_activities( + postgres_container, + mongo_uri, + redis_container, + pi_web_api_server, + mock_logger, + notification_handler, +): + """ + Production Activities wired to testcontainers and the PI Web API stub. + + Return: + Live Activities instance (no unittest.mock.patch) + """ + pg_parts = postgres_connection_parts(postgres_container.get_connection_url()) + redis_host = redis_container.get_container_host_ip() + redis_port = int(redis_container.get_exposed_port(6379)) + + activities = Activities( + postgres_config={ + **pg_parts, + 'min_connections': 1, + 'max_connections': 5, + }, + redis_config={ + 'host': redis_host, + 'port': redis_port, + 'username': '', + 'password': '', + }, + mongodb_config={ + 'connection_string': mongo_uri, + 'database_name': E2E_DATABASE, + }, + api_config={ + 'base_url': pi_web_api_server.base_url, + 'auth_type': 'bearer', + 'auth_token': 'e2e-test-token', + }, + logger=mock_logger, + notification_handler=notification_handler, + ) + yield activities + activities.shutdown() + + +@pytest_asyncio.fixture +async def temporal_worker(temporal_env, test_activities): + """ + Temporal worker registering all Scouter workflows and activities on the E2E queue. + + Return: + Running temporalio.worker.Worker + """ async with Worker( - temporal_test_env.client, - task_queue='test-queue', - workflows=[PIWebAPIScouter, CoreScouter], - activities=[ - test_activities.get_tag_values, - test_activities.data_quality_gate, - test_activities.aggregate_data, - test_activities.group_and_hold_data, - test_activities.export_data_to_postgres, - test_activities.write_metrics, - test_activities.store_data_package, - ], + temporal_env.client, + task_queue=SCOUTER_TASK_QUEUE, + workflows=[Scouter, PIWebAPIScouter, CoreScouter], + activities=_activity_list(test_activities), + activity_executor=ThreadPoolExecutor(max_workers=50, thread_name_prefix='e2e-activity'), + max_concurrent_workflow_tasks=50, + max_concurrent_activities=50, + max_concurrent_local_activities=50, + workflow_task_poller_behavior=PollerBehaviorAutoscaling(minimum=1, initial=2, maximum=10), + activity_task_poller_behavior=PollerBehaviorAutoscaling(minimum=1, initial=2, maximum=10), ) as worker: yield worker diff --git a/e2e/db_schema.sql b/e2e/db_schema.sql new file mode 100644 index 0000000..3e4111b --- /dev/null +++ b/e2e/db_schema.sql @@ -0,0 +1,16 @@ +-- Single source of truth for E2E Postgres DDL (non-partitioned mirror of production sientia_data.laborious_data). + +DROP TABLE IF EXISTS sientia_data.laborious_data; +DROP SCHEMA IF EXISTS sientia_data CASCADE; + +CREATE SCHEMA sientia_data; + +CREATE TABLE sientia_data.laborious_data ( + id SERIAL NOT NULL, + model_id int4 NOT NULL, + variable text NOT NULL, + value numeric NULL, + "timestamp" timestamptz NOT NULL, + created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL, + PRIMARY KEY (id, created_at) +); diff --git a/e2e/fixtures/fake_mongodb_repository.py b/e2e/fixtures/fake_mongodb_repository.py deleted file mode 100644 index 087332d..0000000 --- a/e2e/fixtures/fake_mongodb_repository.py +++ /dev/null @@ -1,241 +0,0 @@ -""" -Fake MongoDB Repository adapter for testing. - -This adapter implements the MongoDBRepository interface using mongomock -to provide an in-memory MongoDB server for testing. -""" - -import time -from typing import Any - -from mongomock import MongoClient -from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler -from sientia_do.observability.logger import Logger -from sientia_do.observability.metrics_controller import MetricsController -from sientia_do.observability.sientia_monitoring import SientiaMonitoring - - -def clear_mongo_id(docs: list) -> list: - """ - Remove MongoDB internal `_id` fields from nested structures. - - Same implementation as in the real MongoDBRepository. - - Args: - docs: The list of documents or nested structures to clean. - - Returns: - The cleaned documents with `_id` fields removed wherever present. - """ - for doc in docs: - if isinstance(doc, list): - clear_mongo_id(doc) - elif isinstance(doc, dict): - if '_id' in doc: - del doc['_id'] - for _key, value in doc.items(): - if isinstance(value, list): - clear_mongo_id(value) - elif isinstance(value, dict): - clear_mongo_id([value]) - return docs - - -class FakeMongoDBRepository(SientiaMonitoring): - """ - Fake MongoDB Repository that uses mongomock for testing. - - Implements the same interface as MongoDBRepository but uses - mongomock for in-memory MongoDB operations. - """ - - def __init__( - self, - connection_string: str, - database_name: str, - logger: Logger, - notification_handler: NotificationHandler, - metrics_controller: MetricsController, - ): - """ - Initialize fake MongoDB repository with mongomock. - - Args: - connection_string: MongoDB connection string (ignored in fake mode) - database_name: Target database name - logger: Logger instance - notification_handler: Notification handler - metrics_controller: Metrics controller - """ - SientiaMonitoring.__init__( - self, - logger=logger, - notification_handler=notification_handler, - metrics_controller=metrics_controller, - ) - - self.database_name = database_name - # Use mongomock instead of real MongoDB - self.mongo_client = MongoClient() - self.database = self.mongo_client[self.database_name] - logger.info('Fake MongoDB connection initialized') - - def close(self): - """ - Closes fake MongoDB connection and shuts down monitoring. - """ - try: - if self.mongo_client: - self.logger.info('Closing fake MongoDB connection...') - self.mongo_client.close() - self.logger.info('Fake MongoDB connection closed successfully') - except Exception as e: - self.logger.error(f'Failed to close fake MongoDB connection: {e}') - SientiaMonitoring.shutdown(self) - - def __del__(self): - """ - Destructor that closes the connection when destroying the instance. - """ - self.close() - - async def find( - self, - collection_name: str, - filters: dict[str, Any], - metadata: dict[str, Any], - ) -> list[dict[str, Any]]: - """ - Finds documents in a MongoDB collection based on provided filters. - - Args: - collection_name: Name of the collection to search - filters: Query filters to apply - metadata: Dictionary with additional metadata - - Return: - List of documents matching the filters (with `_id` removed) - """ - try: - start_time = time.time() - collection = self.database[collection_name] - documents = list(collection.find(filters, {'_id': 0})) - except Exception as e: - self.logger.error(f'Failed to find documents in fake MongoDB: {e}') - raise e - - return clear_mongo_id(documents) - - async def aggregate( - self, - collection_name: str, - pipeline: list[dict[str, Any]], - metadata: dict[str, Any], - ) -> list[dict[str, Any]]: - """ - Executes an aggregation on a MongoDB collection. - - Args: - collection_name: Name of the collection to aggregate - pipeline: MongoDB aggregation pipeline - metadata: Dictionary with additional metadata - - Return: - List of documents resulting from aggregation (with `_id` removed) - """ - try: - start_time = time.time() - collection = self.database[collection_name] - documents = list(collection.aggregate(pipeline)) - except Exception as e: - self.logger.error(f'Failed to aggregate documents in fake MongoDB: {e}') - raise e - - return clear_mongo_id(documents) - - async def update_many( - self, - collection_name: str, - filters: dict[str, Any], - update: dict[str, Any], - metadata: dict[str, Any], - ) -> None: - """ - Updates multiple documents in a MongoDB collection. - - Args: - collection_name: Collection name - filters: Filters to identify documents to update - update: Update operations to apply - metadata: Dictionary with additional metadata - """ - try: - collection = self.database[collection_name] - collection.update_many(filters, update) - except Exception as e: - self.logger.error(f'Failed to update documents in fake MongoDB: {e}') - raise e - - async def insert_many( - self, - collection_name: str, - documents: list[dict[str, Any]], - metadata: dict[str, Any], - ) -> None: - """ - Inserts multiple documents into a MongoDB collection. - - Args: - collection_name: Collection name - documents: List of documents to insert - metadata: Dictionary with additional metadata - """ - try: - collection = self.database[collection_name] - collection.insert_many(documents) - except Exception as e: - self.logger.error(f'Failed to insert documents in fake MongoDB: {e}') - raise e - - async def insert( - self, - collection_name: str, - document: dict[str, Any], - metadata: dict[str, Any], - ) -> None: - """ - Inserts a document into a MongoDB collection. - - Args: - collection_name: Collection name - document: Document to insert - metadata: Dictionary with additional metadata - """ - try: - collection = self.database[collection_name] - collection.insert_one(document) - except Exception as e: - self.logger.error(f'Failed to insert document in fake MongoDB: {e}') - raise e - - async def delete_many( - self, - collection_name: str, - filters: dict[str, Any], - metadata: dict[str, Any], - ) -> None: - """ - Removes multiple documents from a MongoDB collection. - - Args: - collection_name: Collection name - filters: Filters to identify documents to remove - metadata: Dictionary with additional metadata - """ - try: - collection = self.database[collection_name] - collection.delete_many(filters) - except Exception as e: - self.logger.error(f'Failed to delete documents in fake MongoDB: {e}') - raise e - diff --git a/e2e/fixtures/fake_redis_repository.py b/e2e/fixtures/fake_redis_repository.py deleted file mode 100644 index 9dbb762..0000000 --- a/e2e/fixtures/fake_redis_repository.py +++ /dev/null @@ -1,170 +0,0 @@ -""" -Fake Redis Repository adapter for testing. - -This adapter implements the RedisRepository interface using fakeredis -to provide an in-memory Redis server for testing. -""" - -import json -from typing import Any - -import fakeredis -from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler -from sientia_do.observability.logger import Logger -from sientia_do.observability.metrics_controller import MetricsController -from sientia_do.observability.sientia_monitoring import SientiaMonitoring - - -class FakeRedisRepository(SientiaMonitoring): - """ - Fake Redis Repository that uses fakeredis for testing. - - Implements the same interface as RedisRepository but uses - fakeredis for in-memory Redis operations. - """ - - def __init__( - self, - host: str, - port: int, - username: str, - password: str, - logger: Logger, - notification_handler: NotificationHandler, - metrics_controller: MetricsController, - ): - """ - Initialize fake Redis repository with fakeredis. - - Args: - host: Redis server address (ignored in fake mode) - port: Redis server port (ignored in fake mode) - username: Username (ignored in fake mode) - password: Password (ignored in fake mode) - logger: Logger instance - notification_handler: Notification handler - metrics_controller: Metrics controller - """ - SientiaMonitoring.__init__( - self, - logger=logger, - notification_handler=notification_handler, - metrics_controller=metrics_controller, - ) - # Create fake Redis server - self.redis_client = fakeredis.FakeStrictRedis( - decode_responses=True - ) - - def _get_redis(self): - """Get fakeredis connection.""" - return self.redis_client - - async def get(self, key: str, metadata: dict | None = None): - """ - Gets a value from fake Redis by key. - - Args: - key: Key of the value to retrieve - metadata: Optional dictionary with additional metadata - - Return: - Deserialized value from Redis or None if not found - """ - redis = self._get_redis() - try: - history = redis.get(key) - return json.loads(history) if history else None - except Exception as e: - self.error(f'Error getting data from fake redis: {e}', metadata or {}) - raise e - - async def set( - self, - key: str, - data: dict, - ttl: int = 600, - nx: bool = False, - metadata: dict | None = None, - ) -> bool: - """ - Sets a value in fake Redis with optional TTL. - - Args: - key: Key of the value to set - data: Dictionary with data to store - ttl: Time to live in seconds (default: 600) - nx: If True, only sets if key doesn't exist (default: False) - metadata: Optional dictionary with additional metadata - - Return: - True if value was set, False otherwise - """ - redis = self._get_redis() - try: - result = redis.set( - key, json.dumps(data), ex=ttl, nx=nx - ) - return bool(result) - except Exception as e: - self.error(f'Error setting data in fake redis: {e}', metadata or {}) - raise e - - async def delete(self, key: str, metadata: dict | None = None): - """ - Removes a key from fake Redis. - - Args: - key: Key to be removed - metadata: Optional dictionary with additional metadata - """ - redis = self._get_redis() - try: - redis.delete(key) - except Exception as e: - self.error(f'Error deleting data from fake redis: {e}', metadata or {}) - raise e - - async def expire(self, key: str, ttl: int, metadata: dict | None = None): - """ - Sets the time to live (TTL) of an existing Redis key. - - Args: - key: Key whose TTL will be set - ttl: Time to live in seconds - metadata: Optional dictionary with additional metadata - """ - redis = self._get_redis() - try: - redis.expire(key, ttl) - except Exception as e: - self.error(f'Error expiring data from fake redis: {e}', metadata or {}) - raise e - - async def keys(self, pattern: str, metadata: dict | None = None): - """ - Gets all keys matching the specified pattern. - - Args: - pattern: Search pattern for keys - metadata: Optional dictionary with additional metadata - - Return: - List of keys matching the pattern - """ - redis = self._get_redis() - try: - keys = redis.keys(pattern) - return keys - except Exception as e: - self.error(f'Error getting keys from fake redis: {e}', metadata or {}) - raise e - - def close(self): - """ - Closes fake Redis connection and shuts down monitoring. - """ - if self.redis_client: - self.redis_client.close() - SientiaMonitoring.shutdown(self) - diff --git a/e2e/helpers.py b/e2e/helpers.py new file mode 100644 index 0000000..97b8fbb --- /dev/null +++ b/e2e/helpers.py @@ -0,0 +1,404 @@ +""" +Shared helpers for Scouter end-to-end tests. +""" + +from __future__ import annotations + +import json +import re +import uuid +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from pymongo import MongoClient +from redis import Redis +from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Engine +from temporalio.client import Client + +SCENARIO_INPUTS_DIR = Path(__file__).resolve().parent / 'scenario_inputs' +SCOUTER_TASK_QUEUE = 'scouter-test-queue' + +_NOW_MARKER = re.compile(r'^@now(?:([+-])(\d+)([smhd]))?$') + + +def _resolve_timestamp_marker(value: str) -> datetime: + """ + Resolve @now relative timestamp markers to timezone-aware datetimes. + + Args: + - value: Marker string such as @now, @now-1h, or @now+30m + + Return: + Resolved datetime in UTC + """ + match = _NOW_MARKER.match(value.strip()) + if not match: + raise ValueError(f'Invalid timestamp marker: {value}') + + now = datetime.now(UTC) + if match.group(1) is None: + return now + + sign, amount, unit = match.group(1), int(match.group(2)), match.group(3) + delta_kwargs = {'seconds': 0, 'minutes': 0, 'hours': 0, 'days': 0} + if unit == 's': + delta_kwargs['seconds'] = amount + elif unit == 'm': + delta_kwargs['minutes'] = amount + elif unit == 'h': + delta_kwargs['hours'] = amount + elif unit == 'd': + delta_kwargs['days'] = amount + + delta = timedelta(**delta_kwargs) + return now - delta if sign == '-' else now + delta + + +def _resolve_payload(node: Any) -> Any: + """ + Recursively resolve @now markers inside JSON-loaded structures. + + Args: + - node: JSON node (dict, list, or scalar) + + Return: + Structure with markers replaced by datetimes or formatted strings + """ + if isinstance(node, dict): + return {key: _resolve_payload(value) for key, value in node.items()} + if isinstance(node, list): + return [_resolve_payload(item) for item in node] + if isinstance(node, str) and node.startswith('@now'): + resolved = _resolve_timestamp_marker(node) + return resolved.strftime('%Y-%m-%d %H:%M:%S.%f%z') + return node + + +def load_scenario_input(name: str, **overrides: Any) -> dict[str, Any]: + """ + Load a scenario JSON file and apply optional overrides. + + Args: + - name: Scenario slug with or without .json suffix + - overrides: Top-level keys merged into the loaded document + + Return: + Parsed scenario document with @now markers resolved + """ + slug = name.removesuffix('.json') + path = SCENARIO_INPUTS_DIR / f'{slug}.json' + with path.open(encoding='utf-8') as handle: + payload = json.load(handle) + resolved = _resolve_payload(payload) + if overrides: + resolved.update(overrides) + return resolved + + +def make_workflow_id(prefix: str) -> str: + """ + Build a unique Temporal workflow id for E2E runs. + + Args: + - prefix: Human-readable prefix for the workflow id + + Return: + Unique workflow id string + """ + return f'{prefix}-{uuid.uuid4().hex[:12]}' + + +async def start_and_await_workflow( + client: Client, + workflow_run: Any, + input_data: dict[str, Any], + workflow_id: str, + *, + task_queue: str = SCOUTER_TASK_QUEUE, + timeout: float = 300.0, +) -> Any: + """ + Start a workflow on the E2E task queue and await its result. + + Args: + - client: Temporal client from WorkflowEnvironment + - workflow_run: Workflow run method (e.g. Scouter.run) + - input_data: Workflow input payload + - workflow_id: Unique workflow id + - task_queue: Task queue name + - timeout: Maximum seconds to wait for completion + + Return: + Workflow result (None for Scouter-family workflows) + """ + return await client.execute_workflow( + workflow_run, + input_data, + id=workflow_id, + task_queue=task_queue, + execution_timeout=timedelta(seconds=timeout), + ) + + +def _coerce_mongo_document(document: dict[str, Any]) -> dict[str, Any]: + """ + Convert string timestamps in seed documents to BSON datetimes for Mongo filters. + + Args: + - document: Raw document dict from scenario JSON + + Return: + Document with inserted_at as datetime when present + """ + doc = dict(document) + inserted_at = doc.get('inserted_at') + if isinstance(inserted_at, str): + doc['inserted_at'] = datetime.strptime(inserted_at, DATETIME_FORMAT_MS_WITH_TZ).replace( + tzinfo=UTC + ) + return doc + + +def seed_raw_collection( + mongo_uri: str, + database: str, + schedule_name: str, + documents: list[dict[str, Any]], +) -> None: + """ + Insert raw Mongo documents into raw_. + + Args: + - mongo_uri: MongoDB connection string + - database: Database name + - schedule_name: Schedule slug used in collection name + - documents: Documents to insert + """ + collection_name = f'raw_{schedule_name}' + client = MongoClient(mongo_uri) + try: + collection = client[database][collection_name] + if documents: + collection.insert_many([_coerce_mongo_document(doc) for doc in documents]) + finally: + client.close() + + +def seed_last_data_timestamp( + redis_client: Redis, + workflow_name: str, + schedule_name: str, + value: str, +) -> None: + """ + Pre-seed last_data_timestamp Redis key the same way RedisRepository.set stores strings. + + Args: + - redis_client: Connected Redis client + - workflow_name: Workflow name segment in the key + - schedule_name: Schedule name segment in the key + - value: Timestamp string to store + """ + key = f'last_data_timestamp:{workflow_name}:{schedule_name}' + redis_client.set(key, json.dumps(value)) + + +def count_laborious_rows(engine: Engine, model_id: str | int | None = None) -> int: + """ + Count rows in sientia_data.laborious_data, optionally filtered by model_id. + + Args: + - engine: SQLAlchemy engine bound to the Postgres testcontainer + - model_id: Optional model id filter + + Return: + Row count + """ + query = 'SELECT COUNT(*) FROM sientia_data.laborious_data' + params: dict[str, Any] = {} + if model_id is not None: + query += ' WHERE model_id = :model_id' + params['model_id'] = int(model_id) + + with engine.connect() as conn: + return conn.execute(text(query), params).scalar() or 0 + + +def fetch_laborious_rows(engine: Engine, model_id: str | int | None = None) -> list[dict[str, Any]]: + """ + Fetch laborious_data rows as plain dicts. + + Args: + - engine: SQLAlchemy engine bound to the Postgres testcontainer + - model_id: Optional model id filter + + Return: + List of row dicts with variable and value keys + """ + query = 'SELECT model_id, variable, value, timestamp FROM sientia_data.laborious_data' + params: dict[str, Any] = {} + if model_id is not None: + query += ' WHERE model_id = :model_id' + params['model_id'] = int(model_id) + query += ' ORDER BY variable' + + with engine.connect() as conn: + rows = conn.execute(text(query), params).mappings().all() + return [dict(row) for row in rows] + + +def count_held_data_keys(redis_client: Redis) -> int: + """ + Count Redis keys matching held_data_*. + + Args: + - redis_client: Connected Redis client + + Return: + Number of matching keys + """ + return len(redis_client.keys('held_data_*')) + + +def count_notifications( + mongo_uri: str, + database: str, + *, + notification_id: str | None = None, + level: str | None = None, +) -> int: + """ + Count notification documents in the E2E notification collection. + + Args: + - mongo_uri: MongoDB connection string + - database: Database name + - notification_id: Optional notification_id filter + - level: Optional level filter (WARNING, ERROR, ...) + + Return: + Matching document count + """ + client = MongoClient(mongo_uri) + try: + collection = client[database]['notification_queue'] + query: dict[str, Any] = {} + if notification_id: + query['notification_id'] = notification_id + if level: + query['level'] = level + return collection.count_documents(query) + finally: + client.close() + + +def default_model_tags( + *, + names: list[str], + aggr: str = 'avg', + data_range: list[float] | None = None, + frequency: int = 60000, +) -> dict[str, dict[str, Any]]: + """ + Build a minimal model_tags map for E2E scenarios. + + Args: + - names: Tag names to include + - aggr: Aggregation function (aggr_func field) + - data_range: Optional [min, max] validation range + - frequency: Collection frequency in milliseconds + + Return: + model_tags dict keyed by tag name + """ + if data_range is None: + data_range = [0, 100] + return { + name: { + 'webid': f'webid_{name}', + 'aggr_func': aggr, + 'data_range': data_range, + 'frequency': frequency, + } + for name in names + } + + +def default_scouter_input( + *, + model_id: str = '1', + model_name: str = 'Test Model', + schedule_name: str = 'test-schedule', + workflow_name: str = 'scouter', + **overrides: Any, +) -> dict[str, Any]: + """ + Return a base workflow input dict for Scouter / CoreScouter E2E runs. + + Args: + - model_id: Model identifier + - model_name: Human-readable model name + - schedule_name: Schedule slug + - workflow_name: Parent workflow name + - overrides: Additional keys merged into the payload + + Return: + Workflow input dictionary + """ + payload: dict[str, Any] = { + 'topic': 'e2e-topic', + 'model_id': model_id, + 'model_name': model_name, + 'schedule_name': schedule_name, + 'workflow_name': workflow_name, + 'trigger_laborious': False, + 'filters': {}, + 'schema': 'sientia_data', + 'table_name': 'laborious_data', + 'retention_time': 3600, + 'fill_missing_tags': False, + 'model_tags': default_model_tags(names=['tag1']), + } + payload.update(overrides) + return payload + + +def apply_pi_web_api_server_config(server: Any, config: dict[str, Any] | None) -> None: + """ + Configure the in-process PI Web API stub from a scenario pi_web_api_server block. + + Args: + - server: PIWebAPITestServer instance + - config: Optional mode/rows/timeout_sleep_seconds dict from scenario JSON + """ + if not config: + return + server.set_mode( + config.get('mode', 'success'), + rows=config.get('rows'), + timeout_sleep_seconds=config.get('timeout_sleep_seconds', 60), + ) + + +def postgres_connection_parts(connection_url: str) -> dict[str, Any]: + """ + Parse a SQLAlchemy Postgres URL into Activities postgres_config fields. + + Args: + - connection_url: SQLAlchemy connection URL from testcontainers + + Return: + Dict with host, port, user, password, dbname keys + """ + parsed = urlparse(connection_url) + return { + 'host': parsed.hostname or 'localhost', + 'port': parsed.port or 5432, + 'user': parsed.username or 'test', + 'password': parsed.password or 'test', + 'dbname': (parsed.path or '/test').lstrip('/'), + } diff --git a/e2e/pi_web_api_test_server.py b/e2e/pi_web_api_test_server.py new file mode 100644 index 0000000..912c214 --- /dev/null +++ b/e2e/pi_web_api_test_server.py @@ -0,0 +1,124 @@ +""" +In-process HTTP server emulating PI Web API streamsets/recorded responses for E2E tests. +""" + +from __future__ import annotations + +import json +import time +from typing import Any, Literal + +from pytest_httpserver import HTTPServer +from werkzeug import Request +from werkzeug.wrappers import Response + +PIWebAPIMode = Literal['success', 'empty', 'error', 'timeout'] +STREAMSETS_RECORDED_PATH = '/streamsets/recorded' + + +class PIWebAPITestServer: + """ + Thread-backed PI Web API stub using pytest-httpserver (real HTTP for pycurl clients). + """ + + def __init__(self) -> None: + self._httpserver = HTTPServer(host='127.0.0.1', port=0) + self._mode: PIWebAPIMode = 'success' + self._rows: list[dict[str, Any]] = [] + self._timeout_sleep_seconds = 60 + self.requests: list[Request] = [] + + @property + def host(self) -> str: + return self._httpserver.host + + @property + def port(self) -> int: + return self._httpserver.port + + @property + def base_url(self) -> str: + return f'http://{self.host}:{self.port}' + + def start(self) -> None: + """Start the HTTP server and register the streamsets handler.""" + self._httpserver.start() + self._register_handler() + + def stop(self) -> None: + """Stop the HTTP server.""" + self._httpserver.stop() + + def clear(self) -> None: + """Clear recorded requests and reset mode to success with no rows.""" + self.requests.clear() + self._mode = 'success' + self._rows = [] + self._httpserver.clear() + self._register_handler() + + def set_mode( + self, + mode: PIWebAPIMode, + rows: list[dict[str, Any]] | None = None, + *, + timeout_sleep_seconds: int = 60, + ) -> None: + """ + Configure the next responses from the stub server. + + Args: + - mode: Response mode (success, empty, error, timeout) + - rows: Optional list of row dicts with keys name, webid, timestamp, value + - timeout_sleep_seconds: Sleep duration for timeout mode (must exceed client timeout) + """ + self._mode = mode + if rows is not None: + self._rows = rows + self._timeout_sleep_seconds = timeout_sleep_seconds + self._register_handler() + + def _register_handler(self) -> None: + self._httpserver.expect_request( + STREAMSETS_RECORDED_PATH, + method='GET', + ).respond_with_handler(self._handle_streamsets_recorded) + + def _handle_streamsets_recorded(self, request: Request): + self.requests.append(request) + + if self._mode == 'timeout': + time.sleep(self._timeout_sleep_seconds) + return self._json_response({'Items': []}, status=200) + + if self._mode == 'error': + return self._json_response({'error': 'internal'}, status=500) + + if self._mode == 'empty' or not self._rows: + return self._json_response({'Items': []}, status=200) + + items_by_name: dict[str, list[dict[str, Any]]] = {} + for row in self._rows: + name = row['name'] + items_by_name.setdefault(name, []).append( + { + 'Timestamp': row['timestamp'], + 'Value': row['value'], + 'Good': True, + 'Questionable': False, + } + ) + + items = [ + {'Name': name, 'Items': points} + for name, points in items_by_name.items() + ] + return self._json_response({'Items': items}, status=200) + + @staticmethod + def _json_response(payload: dict[str, Any], *, status: int) -> Response: + return Response( + json.dumps(payload), + status=status, + mimetype='application/json', + ) diff --git a/e2e/scenario_inputs/core_scouter_aggregation_avg.json b/e2e/scenario_inputs/core_scouter_aggregation_avg.json new file mode 100644 index 0000000..2863249 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_aggregation_avg.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "24", "model_name": "Core Avg", "schedule_name": "core-avg", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-avg", + "model_name": "Core Avg", + "model_id": "24", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_avg", "value": 10.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:01:00+0000", "name": "tag_avg", "value": 20.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:02:00+0000", "name": "tag_avg", "value": 30.0, "tag": "w1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_avg": {"webid": "w1", "aggr_func": "avg", "data_range": [0, 1000], "frequency": 60000} + } + }, + "expected_value": 20.0 +} diff --git a/e2e/scenario_inputs/core_scouter_aggregation_lts.json b/e2e/scenario_inputs/core_scouter_aggregation_lts.json new file mode 100644 index 0000000..95612c5 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_aggregation_lts.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "28", "model_name": "Core Lts", "schedule_name": "core-lts", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-lts", + "model_name": "Core Lts", + "model_id": "28", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_lts", "value": 100.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:01:00+0000", "name": "tag_lts", "value": 200.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:02:00+0000", "name": "tag_lts", "value": 300.0, "tag": "w1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_lts": {"webid": "w1", "aggr_func": "lts", "data_range": [0, 1000], "frequency": 60000} + } + }, + "expected_value": 300.0 +} diff --git a/e2e/scenario_inputs/core_scouter_aggregation_max.json b/e2e/scenario_inputs/core_scouter_aggregation_max.json new file mode 100644 index 0000000..e8ce960 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_aggregation_max.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "26", "model_name": "Core Max", "schedule_name": "core-max", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-max", + "model_name": "Core Max", + "model_id": "26", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_max", "value": 5.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:01:00+0000", "name": "tag_max", "value": 15.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:02:00+0000", "name": "tag_max", "value": 10.0, "tag": "w1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_max": {"webid": "w1", "aggr_func": "max", "data_range": [0, 1000], "frequency": 60000} + } + }, + "expected_value": 15.0 +} diff --git a/e2e/scenario_inputs/core_scouter_aggregation_mdn.json b/e2e/scenario_inputs/core_scouter_aggregation_mdn.json new file mode 100644 index 0000000..93e4fda --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_aggregation_mdn.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "25", "model_name": "Core Mdn", "schedule_name": "core-mdn", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-mdn", + "model_name": "Core Mdn", + "model_id": "25", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_mdn", "value": 1.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:01:00+0000", "name": "tag_mdn", "value": 9.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:02:00+0000", "name": "tag_mdn", "value": 5.0, "tag": "w1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_mdn": {"webid": "w1", "aggr_func": "mdn", "data_range": [0, 1000], "frequency": 60000} + } + }, + "expected_value": 5.0 +} diff --git a/e2e/scenario_inputs/core_scouter_aggregation_min.json b/e2e/scenario_inputs/core_scouter_aggregation_min.json new file mode 100644 index 0000000..3cfc8f4 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_aggregation_min.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "27", "model_name": "Core Min", "schedule_name": "core-min", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-min", + "model_name": "Core Min", + "model_id": "27", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_min", "value": 50.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:01:00+0000", "name": "tag_min", "value": 30.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:02:00+0000", "name": "tag_min", "value": 40.0, "tag": "w1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_min": {"webid": "w1", "aggr_func": "min", "data_range": [0, 1000], "frequency": 60000} + } + }, + "expected_value": 30.0 +} diff --git a/e2e/scenario_inputs/core_scouter_debug_data_package.json b/e2e/scenario_inputs/core_scouter_debug_data_package.json new file mode 100644 index 0000000..8f0eb92 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_debug_data_package.json @@ -0,0 +1,22 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "30", "model_name": "Core Debug", "schedule_name": "core-debug", "workflow_name": "subworkflow.core_scouter"}}, + "workflow_name": "subworkflow.core_scouter", + "schedule_name": "core-debug", + "model_name": "Core Debug", + "model_id": "30", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": 11.0, "tag": "webid1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "debug_data_package": true, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_empty_after_grouping.json b/e2e/scenario_inputs/core_scouter_empty_after_grouping.json new file mode 100644 index 0000000..236c269 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_empty_after_grouping.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "31", "model_name": "Core Empty Group", "schedule_name": "core-empty-group", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-empty-group", + "model_name": "Core Empty Group", + "model_id": "31", + "data": { + "timestamp": [], + "name": [], + "value": [], + "tag": [] + }, + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_fill_missing_tags.json b/e2e/scenario_inputs/core_scouter_fill_missing_tags.json new file mode 100644 index 0000000..677f5bd --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_fill_missing_tags.json @@ -0,0 +1,24 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "29", "model_name": "Core Fill Tags", "schedule_name": "core-fill", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-fill", + "model_name": "Core Fill Tags", + "model_id": "29", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": 10.0, "tag": "webid1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": true, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag2": {"webid": "webid2", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag3": {"webid": "webid3", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + }, + "expected_missing_tags": ["tag2", "tag3"] +} diff --git a/e2e/scenario_inputs/core_scouter_happy_path.json b/e2e/scenario_inputs/core_scouter_happy_path.json new file mode 100644 index 0000000..ad1e889 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_happy_path.json @@ -0,0 +1,28 @@ +{ + "workflow_input": { + "metadata": { + "metadata": { + "model_id": "20", + "model_name": "Core Happy", + "schedule_name": "core-happy", + "workflow_name": "pi_web_api_scouter" + } + }, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-happy", + "model_name": "Core Happy", + "model_id": "20", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": 10.5, "tag": "webid1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_invalid_aggregation.json b/e2e/scenario_inputs/core_scouter_invalid_aggregation.json new file mode 100644 index 0000000..b0c8a4a --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_invalid_aggregation.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "32", "model_name": "Core Bad Aggr", "schedule_name": "core-bad-aggr", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-bad-aggr", + "model_name": "Core Bad Aggr", + "model_id": "32", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_ok", "value": 10.0, "tag": "w1"}, + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag_bad", "value": 20.0, "tag": "w2"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_ok": {"webid": "w1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag_bad": {"webid": "w2", "aggr_func": "bogus", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_null_values_filter_discard.json b/e2e/scenario_inputs/core_scouter_null_values_filter_discard.json new file mode 100644 index 0000000..cfb74df --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_null_values_filter_discard.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "21", "model_name": "Core Null Discard", "schedule_name": "core-null-discard", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-null-discard", + "model_name": "Core Null Discard", + "model_id": "21", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": null, "tag": "webid1"}, + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag2", "value": 50.0, "tag": "webid2"} + ], + "trigger_laborious": false, + "filters": {"NULL_VALUES_FILTER": {"policy": "DISCARD"}}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag2": {"webid": "webid2", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_null_values_filter_warn.json b/e2e/scenario_inputs/core_scouter_null_values_filter_warn.json new file mode 100644 index 0000000..96cc560 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_null_values_filter_warn.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "22", "model_name": "Core Null Warn", "schedule_name": "core-null-warn", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-null-warn", + "model_name": "Core Null Warn", + "model_id": "22", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": null, "tag": "webid1"}, + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag2", "value": 50.0, "tag": "webid2"} + ], + "trigger_laborious": false, + "filters": {"NULL_VALUES_FILTER": {"policy": "WARN"}}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag2": {"webid": "webid2", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_out_of_bounds_filter_discard.json b/e2e/scenario_inputs/core_scouter_out_of_bounds_filter_discard.json new file mode 100644 index 0000000..a7225ad --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_out_of_bounds_filter_discard.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "23", "model_name": "Core OOB Discard", "schedule_name": "core-oob-discard", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-oob-discard", + "model_name": "Core OOB Discard", + "model_id": "23", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": 150.0, "tag": "webid1"}, + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag2", "value": 50.0, "tag": "webid2"} + ], + "trigger_laborious": false, + "filters": {"OUT_OF_BOUNDS_FILTER": {"policy": "DISCARD"}}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag2": {"webid": "webid2", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/core_scouter_postgres_export_failure.json b/e2e/scenario_inputs/core_scouter_postgres_export_failure.json new file mode 100644 index 0000000..982b976 --- /dev/null +++ b/e2e/scenario_inputs/core_scouter_postgres_export_failure.json @@ -0,0 +1,21 @@ +{ + "workflow_input": { + "metadata": {"metadata": {"model_id": "33", "model_name": "Core PG Fail", "schedule_name": "core-pg-fail", "workflow_name": "pi_web_api_scouter"}}, + "workflow_name": "pi_web_api_scouter", + "schedule_name": "core-pg-fail", + "model_name": "Core PG Fail", + "model_id": "33", + "data": [ + {"timestamp": "2024-01-01 12:00:00+0000", "name": "tag1", "value": 10.0, "tag": "webid1"} + ], + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + } + } +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_connection_error.json b/e2e/scenario_inputs/pi_web_api_scouter_connection_error.json new file mode 100644 index 0000000..29244c3 --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_connection_error.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "model_id": "14", + "model_name": "PI Error", + "schedule_name": "pi-error", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 1, + "api_timeout": 30 + } + }, + "pi_web_api_server": {"mode": "error"} +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_debug_data_package.json b/e2e/scenario_inputs/pi_web_api_scouter_debug_data_package.json new file mode 100644 index 0000000..d49ced1 --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_debug_data_package.json @@ -0,0 +1,29 @@ +{ + "workflow_input": { + "model_id": "12", + "model_name": "PI Debug Package", + "schedule_name": "pi-debug", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "debug_data_package": true, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 5, + "api_timeout": 30 + } + }, + "pi_web_api_server": { + "mode": "success", + "rows": [ + {"name": "tag1", "timestamp": "2024-06-01T12:00:00Z", "value": 7.5} + ] + } +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_empty_response.json b/e2e/scenario_inputs/pi_web_api_scouter_empty_response.json new file mode 100644 index 0000000..83591f2 --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_empty_response.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "model_id": "13", + "model_name": "PI Empty", + "schedule_name": "pi-empty", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 1, + "api_timeout": 30 + } + }, + "pi_web_api_server": {"mode": "empty"} +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_happy_path.json b/e2e/scenario_inputs/pi_web_api_scouter_happy_path.json new file mode 100644 index 0000000..7c4ae70 --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_happy_path.json @@ -0,0 +1,30 @@ +{ + "workflow_input": { + "model_id": "10", + "model_name": "PI Web API Happy", + "schedule_name": "pi-happy", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000}, + "tag2": {"webid": "webid2", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 10, + "api_timeout": 30 + } + }, + "pi_web_api_server": { + "mode": "success", + "rows": [ + {"name": "tag1", "timestamp": "2024-06-01T12:00:00Z", "value": 10.5}, + {"name": "tag2", "timestamp": "2024-06-01T12:00:00Z", "value": 20.3} + ] + } +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_invalid_endpoint.json b/e2e/scenario_inputs/pi_web_api_scouter_invalid_endpoint.json new file mode 100644 index 0000000..cc40d1b --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_invalid_endpoint.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "model_id": "16", + "model_name": "PI Invalid Endpoint", + "schedule_name": "pi-invalid", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/invalid/endpoint", + "period": "*-1d", + "max_count": 1, + "api_timeout": 30 + } + }, + "pi_web_api_server": {"mode": "success", "rows": []} +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_multiple_tags.json b/e2e/scenario_inputs/pi_web_api_scouter_multiple_tags.json new file mode 100644 index 0000000..c2c06a1 --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_multiple_tags.json @@ -0,0 +1,53 @@ +{ + "workflow_input": { + "model_id": "11", + "model_name": "PI Multi Tag", + "schedule_name": "pi-multi", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag_avg": {"webid": "w1", "aggr_func": "avg", "data_range": [0, 1000], "frequency": 60000}, + "tag_mdn": {"webid": "w2", "aggr_func": "mdn", "data_range": [0, 1000], "frequency": 60000}, + "tag_max": {"webid": "w3", "aggr_func": "max", "data_range": [0, 1000], "frequency": 60000}, + "tag_min": {"webid": "w4", "aggr_func": "min", "data_range": [0, 1000], "frequency": 60000}, + "tag_lts": {"webid": "w5", "aggr_func": "lts", "data_range": [0, 1000], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 3, + "api_timeout": 30 + } + }, + "pi_web_api_server": { + "mode": "success", + "rows": [ + {"name": "tag_avg", "timestamp": "2024-06-01T12:00:00Z", "value": 10}, + {"name": "tag_avg", "timestamp": "2024-06-01T12:01:00Z", "value": 20}, + {"name": "tag_avg", "timestamp": "2024-06-01T12:02:00Z", "value": 30}, + {"name": "tag_mdn", "timestamp": "2024-06-01T12:00:00Z", "value": 1}, + {"name": "tag_mdn", "timestamp": "2024-06-01T12:01:00Z", "value": 9}, + {"name": "tag_mdn", "timestamp": "2024-06-01T12:02:00Z", "value": 5}, + {"name": "tag_max", "timestamp": "2024-06-01T12:00:00Z", "value": 5}, + {"name": "tag_max", "timestamp": "2024-06-01T12:01:00Z", "value": 15}, + {"name": "tag_max", "timestamp": "2024-06-01T12:02:00Z", "value": 10}, + {"name": "tag_min", "timestamp": "2024-06-01T12:00:00Z", "value": 50}, + {"name": "tag_min", "timestamp": "2024-06-01T12:01:00Z", "value": 30}, + {"name": "tag_min", "timestamp": "2024-06-01T12:02:00Z", "value": 40}, + {"name": "tag_lts", "timestamp": "2024-06-01T12:00:00Z", "value": 100}, + {"name": "tag_lts", "timestamp": "2024-06-01T12:01:00Z", "value": 200}, + {"name": "tag_lts", "timestamp": "2024-06-01T12:02:00Z", "value": 300} + ] + }, + "expected_values": { + "tag_avg": 20.0, + "tag_mdn": 5.0, + "tag_max": 15.0, + "tag_min": 30.0, + "tag_lts": 300.0 + } +} diff --git a/e2e/scenario_inputs/pi_web_api_scouter_timeout.json b/e2e/scenario_inputs/pi_web_api_scouter_timeout.json new file mode 100644 index 0000000..5d70d8e --- /dev/null +++ b/e2e/scenario_inputs/pi_web_api_scouter_timeout.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "model_id": "15", + "model_name": "PI Timeout", + "schedule_name": "pi-timeout", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": {"webid": "webid1", "aggr_func": "avg", "data_range": [0, 100], "frequency": 60000} + }, + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 1, + "api_timeout": 2 + } + }, + "pi_web_api_server": {"mode": "timeout", "timeout_sleep_seconds": 5} +} diff --git a/e2e/scenario_inputs/scouter_empty_mongo.json b/e2e/scenario_inputs/scouter_empty_mongo.json new file mode 100644 index 0000000..0738993 --- /dev/null +++ b/e2e/scenario_inputs/scouter_empty_mongo.json @@ -0,0 +1,23 @@ +{ + "workflow_input": { + "topic": "e2e-topic", + "model_id": "3", + "model_name": "Scouter Empty Mongo", + "schedule_name": "scouter-empty", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": { + "webid": "webid1", + "aggr_func": "avg", + "data_range": [0, 100], + "frequency": 60000 + } + } + }, + "raw_documents": [] +} diff --git a/e2e/scenario_inputs/scouter_happy_path.json b/e2e/scenario_inputs/scouter_happy_path.json new file mode 100644 index 0000000..0ca6510 --- /dev/null +++ b/e2e/scenario_inputs/scouter_happy_path.json @@ -0,0 +1,38 @@ +{ + "workflow_input": { + "topic": "e2e-topic", + "model_id": "1", + "model_name": "Scouter E2E Model", + "schedule_name": "scouter-happy", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": { + "webid": "webid1", + "aggr_func": "avg", + "data_range": [0, 100], + "frequency": 60000 + } + } + }, + "raw_documents": [ + { + "inserted_at": "2024-06-01 12:00:00.000000+0000", + "timestamp": "2024-06-01 12:00:00+0000", + "name": "tag1", + "value": 10.5, + "tag": "webid1" + }, + { + "inserted_at": "2024-06-01 12:01:00.000000+0000", + "timestamp": "2024-06-01 12:01:00+0000", + "name": "tag1", + "value": 20.0, + "tag": "webid1" + } + ] +} diff --git a/e2e/scenario_inputs/scouter_incremental_load.json b/e2e/scenario_inputs/scouter_incremental_load.json new file mode 100644 index 0000000..4dc0f73 --- /dev/null +++ b/e2e/scenario_inputs/scouter_incremental_load.json @@ -0,0 +1,43 @@ +{ + "workflow_input": { + "topic": "e2e-topic", + "model_id": "2", + "model_name": "Scouter Incremental", + "schedule_name": "scouter-incremental", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": { + "webid": "webid1", + "aggr_func": "avg", + "data_range": [0, 100], + "frequency": 60000 + } + } + }, + "redis_seed": { + "last_data_timestamp": "2024-06-01 10:00:00.000000+0000" + }, + "raw_documents": [ + { + "inserted_at": "2024-06-01 09:00:00.000000+0000", + "timestamp": "2024-06-01 09:00:00+0000", + "name": "tag1", + "value": 1.0, + "tag": "webid1" + }, + { + "inserted_at": "2024-06-01 11:00:00.000000+0000", + "timestamp": "2024-06-01 11:00:00+0000", + "name": "tag1", + "value": 99.0, + "tag": "webid1" + } + ], + "expected_newer_count": 1, + "expected_last_timestamp": "2024-06-01 11:00:00.000000+0000" +} diff --git a/e2e/scenario_inputs/scouter_no_redis_timestamp_first_run.json b/e2e/scenario_inputs/scouter_no_redis_timestamp_first_run.json new file mode 100644 index 0000000..5711b5a --- /dev/null +++ b/e2e/scenario_inputs/scouter_no_redis_timestamp_first_run.json @@ -0,0 +1,31 @@ +{ + "workflow_input": { + "topic": "e2e-topic", + "model_id": "4", + "model_name": "Scouter First Run", + "schedule_name": "scouter-first-run", + "trigger_laborious": false, + "filters": {}, + "schema": "sientia_data", + "table_name": "laborious_data", + "retention_time": 3600, + "fill_missing_tags": false, + "model_tags": { + "tag1": { + "webid": "webid1", + "aggr_func": "lts", + "data_range": [0, 100], + "frequency": 60000 + } + } + }, + "raw_documents": [ + { + "inserted_at": "2024-06-01 08:00:00.000000+0000", + "timestamp": "2024-06-01 08:00:00+0000", + "name": "tag1", + "value": 42.0, + "tag": "webid1" + } + ] +} diff --git a/e2e/scenarios.md b/e2e/scenarios.md index edf97b7..caac4c5 100644 --- a/e2e/scenarios.md +++ b/e2e/scenarios.md @@ -1,427 +1,138 @@ -# Test Scenarios for PI Web API Scouter Workflow +# Scouter E2E scenario catalog -This document describes all possible test scenarios for the `pi_web_api_scouter` workflow and its child workflow `core_scouter`. +## Execution context -## Workflow Overview - -The `pi_web_api_scouter` workflow: -1. Retrieves tag values from PI Web API -2. Delegates processing to `core_scouter` child workflow which: - - Applies data quality gates - - Aggregates data - - Groups and holds data in Redis - - Exports to PostgreSQL - - Writes metrics - - Optionally stores debug data package +- **MongoDB**, **Redis**, and **PostgreSQL** run as session-scoped testcontainers with autouse cleanup between tests. +- **PI Web API** is an in-process HTTP server (`e2e/pi_web_api_test_server.py`) speaking the wire format consumed by `PIWebAPIClient`. +- **Temporal** uses `WorkflowEnvironment.start_local()` and a single worker on `scouter-test-queue`. +- **Production code is not mocked** (except `Logger` and optional notification insert spy). --- -## 1. PI Web API Scouter - Main Workflow Scenarios +## 0. Harness smoke tests -### 1.1 Success Scenarios +Diagnostic-only checks under `e2e/test_harness_smoke.py`. They are not business scenarios; they exist to fail fast when the harness itself (Docker / containers / Temporal worker wiring) is broken, before the numbered suite runs. -#### Scenario 1.1.1: Happy Path - Complete Success -**Description**: Workflow completes successfully with valid data from PI Web API +### 0.0.1 Postgres schema ready (`test_postgres_schema_ready`) -**Input**: -- Valid `model_name`, `model_id`, `schedule_name` -- Valid `pi_web_api_query` with endpoint, period, max_count, api_timeout -- Valid `model_tags` with webids and configurations -- Valid filters, schema, table_name, retention_time +Confirms the autouse fixture executed `db_schema.sql` and `sientia_data.laborious_data` exists in the Postgres testcontainer. -**Expected Behavior**: -- `get_tag_values` returns non-empty list of records -- Workflow proceeds to `core_scouter` -- All activities execute successfully -- Data is stored in PostgreSQL -- Metrics are written -- Workflow completes without errors +### 0.0.2 Activities construct (`test_activities_construct`) -**Assertions**: -- PI Web API client called once with correct parameters -- Data exists in SQLite (PostgreSQL substitute) -- Data cached in Redis -- Metrics written -- No errors raised +Confirms the production `Activities` instance initializes against the Mongo/Redis/Postgres testcontainers without hanging (no `patch(...)` involved). + +### 0.0.3 Temporal PI happy path (`test_temporal_pi_happy_path`) + +End-to-end liveness check: `WorkflowEnvironment.start_local()` + worker + in-process PI server + `PIWebAPIScouter` complete without raising. Functional assertions for this flow live in scenario **2.1.1**. --- -#### Scenario 1.1.2: Success with Multiple Tags -**Description**: Workflow processes multiple tags successfully +## 1. Scouter workflow -**Input**: -- Multiple tags in `model_tags` (3+ tags) -- Each tag has valid webid, aggr_function, data_range, frequency +### 1.1.1 Happy path -**Expected Behavior**: -- All tags retrieved from PI Web API -- All tags processed through quality gates -- All tags aggregated correctly -- All tags stored in database +Seed `raw_` with multiple documents, run `Scouter`, assert Postgres rows and Redis `last_data_timestamp:scouter:`. -**Assertions**: -- Number of records matches number of tags -- All tags present in final data -- Aggregation applied per tag configuration +### 1.2.1 Incremental load + +Pre-seed Redis timestamp; seed older and newer Mongo docs; assert only newer rows export and timestamp advances. + +### 1.3.1 Empty Mongo early exit + +Empty `raw_`; workflow exits without Postgres rows or Redis timestamp key. + +### 1.3.2 No Redis timestamp first run + +No prior Redis key; all seeded Mongo docs load and timestamp is written after success. --- -#### Scenario 1.1.3: Success with Debug Data Package Enabled -**Description**: Workflow completes with `debug_data_package=True` +## 2. PIWebAPIScouter workflow -**Input**: -- All standard input -- `debug_data_package: True` +### 2.1.1 Happy path -**Expected Behavior**: -- Normal workflow execution -- `store_data_package` activity called -- Data package stored in Redis +PI server `success` mode with two tags; assert Postgres rows, Redis hold key, one HTTP request recorded. -**Assertions**: -- `store_data_package` called once -- Data package key exists in Redis -- Package contains both `data` and `held_data` +### 2.1.2 Multiple tags + +Five tags with `avg` / `mdn` / `max` / `min` / `lts`; assert five distinct `variable` values and exact aggregated numbers in Postgres. + +### 2.1.3 Debug data package + +`debug_data_package=True`; assert `data_package_pi_web_api_scouter_*` Redis key with `data` and `held_data`. + +### 2.2.1 Empty response early exit + +Server `empty` mode; zero Postgres rows for `model_id`, one request recorded. + +### 2.3.1 PI Web API connection error + +Server `error` mode (HTTP 5xx); workflow fails; `PI_WEB_API_REQUEST_ERROR` notification in Mongo. + +### 2.3.2 PI Web API timeout + +Server `timeout` mode; workflow fails; `PI_WEB_API_REQUEST_ERROR` notification sent. + +### 2.3.3 Invalid endpoint + +Workflow uses `/invalid/endpoint` (404); workflow fails; `PI_WEB_API_REQUEST_ERROR` notification sent. --- -### 1.2 Early Exit Scenarios +## 3. CoreScouter subworkflow -#### Scenario 1.2.1: Empty Data from PI Web API -**Description**: PI Web API returns empty data +### 3.1.1 Complete processing success -**Input**: -- Valid configuration -- PI Web API returns empty DataFrame or empty list +Single tag, no filters; Postgres row and `held_data_*` Redis key; no `data_package_*` key. -**Expected Behavior**: -- `get_tag_values` returns empty list `[]` -- Workflow checks `if not data:` and returns early -- `core_scouter` is NOT called -- Workflow completes without error +### 3.1.2 Null values filter discard -**Assertions**: -- PI Web API called once -- `core_scouter` NOT called -- No data in PostgreSQL -- No data in Redis (except possibly from previous runs) +`NULL_VALUES_FILTER` DISCARD; one WARNING notification; only valid row in Postgres. ---- +### 3.1.3 Null values filter warn -#### Scenario 1.2.2: None Returned from PI Web API -**Description**: PI Web API returns None +`NULL_VALUES_FILTER` WARN; notification sent; both rows in Postgres. -**Input**: -- Valid configuration -- PI Web API returns None +### 3.1.4 Out of bounds filter discard -**Expected Behavior**: -- `get_tag_values` returns None -- Workflow checks `if not data:` and returns early -- `core_scouter` is NOT called +`OUT_OF_BOUNDS_FILTER` DISCARD; in-range row only; WARNING notification. -**Assertions**: -- PI Web API called once -- `core_scouter` NOT called -- Workflow completes without error +### 3.1.5 Aggregation avg ---- +Three points; Postgres `value` equals arithmetic mean (20.0). -### 1.3 Error Scenarios +### 3.1.6 Aggregation mdn -#### Scenario 1.3.1: PI Web API Connection Error -**Description**: PI Web API client raises connection error +Median equals 5.0 in Postgres. -**Input**: -- Valid configuration -- PI Web API client raises `PIMSRequestError` or connection exception +### 3.1.7 Aggregation max -**Expected Behavior**: -- `get_tag_values` catches exception -- Sends notification with `PI_WEB_API_REQUEST_ERROR` -- Raises exception (workflow fails after retries) +Maximum equals 15.0 in Postgres. -**Assertions**: -- Notification sent with correct error details -- Exception propagated to workflow -- Workflow fails (after retry policy exhausted) -- `core_scouter` NOT called +### 3.1.8 Aggregation min ---- +Minimum equals 30.0 in Postgres. -#### Scenario 1.3.2: PI Web API Timeout -**Description**: PI Web API request times out +### 3.1.9 Aggregation lts -**Input**: -- Valid configuration -- `api_timeout` set to low value -- PI Web API takes longer than timeout +Last-by-timestamp value equals 300.0 in Postgres. -**Expected Behavior**: -- Request times out -- Exception raised -- Notification sent -- Workflow fails after retries +### 3.1.10 Fill missing tags -**Assertions**: -- Timeout exception caught -- Notification sent -- Workflow fails +`fill_missing_tags=True`; `held_data_*` contains missing tag keys with `None`. ---- +### 3.1.11 Debug data package -#### Scenario 1.3.3: Invalid Endpoint -**Description**: Invalid PI Web API endpoint provided +`debug_data_package=True`; `data_package_*` Redis key decodes to dict with `data` and `held_data`. -**Input**: -- Invalid endpoint path in `pi_web_api_query` +### 3.2.1 Empty after grouping early exit -**Expected Behavior**: -- PI Web API client raises error -- Notification sent -- Workflow fails +Empty column-oriented `data`; no Postgres rows; no populated `held_data_*`. -**Assertions**: -- Error notification sent -- Workflow fails +### 3.3.1 Invalid aggregation function ---- +`aggr_func= bogus`; `AGGREGATION_ISSUES` ERROR notification; bogus tag absent from Postgres. -## 2. CoreScouter - Child Workflow Scenarios - -### 2.1 Success Scenarios - -#### Scenario 2.1.1: Complete Processing Success -**Description**: All stages complete successfully - -**Input**: -- Valid data from parent workflow -- Valid filters, model_tags, schema, table_name -- `fill_missing_tags: False` -- `debug_data_package: False` - -**Expected Behavior**: -- `data_quality_gate` filters data -- `aggregate_data` aggregates by tag -- `group_and_hold_data` stores in Redis -- `export_data_to_postgres` writes to database -- `write_metrics` records metrics -- Workflow completes - -**Assertions**: -- All activities called in correct order -- Data in PostgreSQL -- Data in Redis -- Metrics written -- `store_data_package` NOT called - ---- - -#### Scenario 2.1.2: Success with Data Quality Filters -**Description**: Data quality filters applied successfully - -**Input**: -- Data with some quality issues -- Filters configured with `NULL_VALUES_FILTER` or `OUT_OF_BOUNDS_FILTER` -- Policy set to `DISCARD` or `WARN` - -**Expected Behavior**: -- Quality gate identifies issues -- Notification sent (WARNING level) -- If policy is `DISCARD`, bad rows removed -- Remaining data processed normally - -**Assertions**: -- Quality issues detected -- Notification sent -- Bad data discarded if policy is `DISCARD` -- Good data processed - ---- - -#### Scenario 2.1.3: Success with Different Aggregation Functions -**Description**: Different aggregation functions applied correctly - -**Input**: -- Multiple tags with different `aggr_function`: `avg`, `mdn`, `max`, `min`, `lts` -- Time-series data with multiple points per tag - -**Expected Behavior**: -- Each tag aggregated with its configured function -- Aggregated values correct for each function type - -**Assertions**: -- `avg` calculates mean correctly -- `mdn` calculates median correctly -- `max` returns maximum value -- `min` returns minimum value -- `lts` returns latest value - ---- - -#### Scenario 2.1.4: Success with Fill Missing Tags -**Description**: Missing tags filled with None - -**Input**: -- `fill_missing_tags: True` -- Some tags missing from data - -**Expected Behavior**: -- Missing tags added to `data_hold` with value `None` -- All expected tags present in final data - -**Assertions**: -- Missing tags present with `None` value -- All model_tags represented in output - ---- - -### 2.2 Early Exit Scenarios - -#### Scenario 2.2.1: Empty Data After Grouping -**Description**: `group_and_hold_data` returns empty dict - -**Input**: -- Data that results in empty `held_data` after grouping - -**Expected Behavior**: -- `group_and_hold_data` returns `{}` -- Workflow checks `if held_data == {}:` and returns early -- `export_data_to_postgres` NOT called -- `write_metrics` NOT called -- `store_data_package` NOT called - -**Assertions**: -- Early return after grouping -- No database export -- No metrics written -- Workflow completes without error - ---- - -### 2.3 Error Scenarios - -#### Scenario 2.3.1: Redis Connection Error -**Description**: Redis unavailable during `group_and_hold_data` - -**Input**: -- Valid data -- Redis connection fails - -**Expected Behavior**: -- `redis_repository.get()` or `redis_repository.set()` raises exception -- Notification sent with `REDIS_GET_ERROR` or `REDIS_SET_ERROR` -- Exception propagated (workflow fails after retries) - -**Assertions**: -- Error notification sent -- Workflow fails - ---- - -## 3. Activity-Specific Scenarios - -> **Note**: Activity-specific scenarios are better suited for unit tests rather than e2e tests. -> These scenarios are covered indirectly through workflow e2e tests. For detailed activity testing, -> refer to the unit test suite in `tests/activities/`. - ---- - -## 5. Test Data Requirements - -### 5.1 Valid Test Data Structure - -```python -{ - 'model_name': 'test_model', - 'model_id': 'test_model_id', - 'schedule_name': 'test_schedule', - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'test_schema', - 'table_name': 'test_table', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'debug_data_package': False, - 'pi_web_api_query': { - 'endpoint': '/streamsets/recorded', - 'period': '*-1d', - 'max_count': 10, - 'api_timeout': 30, - }, -} -``` - -### 5.2 Mock PI Web API Response - -```python -DataFrame({ - 'timestamp': ['2024-01-01 12:00:00+0000', ...], - 'name': ['tag1', 'tag2', ...], - 'value': [10.5, 20.3, ...], - 'tag': ['webid1', 'webid2', ...], -}) -``` - ---- - -## 6. Test Implementation Notes - -### 6.1 Test Organization - -- Group tests by scenario category -- Use descriptive test names matching scenario IDs -- Share fixtures for common setup -- Use parametrized tests for similar scenarios - -### 6.2 Assertions Checklist - -For each scenario, verify: -- [ ] Correct activities called -- [ ] Correct parameters passed -- [ ] Expected data in storage (SQLite/Redis) -- [ ] Expected notifications sent -- [ ] Expected metrics written -- [ ] No unexpected errors -- [ ] Workflow state correct - -### 6.3 Mock Configuration - -- Mock PI Web API client responses -- Use fake Redis (fakeredis) -- Use fake MongoDB (mongomock) -- Use SQLite for PostgreSQL -- Mock notification handler -- Mock metrics controller - ---- - -## 7. Priority Scenarios - -### High Priority (Must Test) -1. Scenario 1.1.1: Happy Path -2. Scenario 1.2.1: Empty Data -3. Scenario 1.3.1: API Connection Error -4. Scenario 2.1.1: Complete Processing -5. Scenario 2.2.1: Empty After Grouping -6. Scenario 2.3.4: PostgreSQL Error - -### Medium Priority (Should Test) -1. Scenario 1.1.3: Debug Package -2. Scenario 2.1.2: Quality Filters -3. Scenario 2.1.3: Different Aggregations -4. Scenario 3.3.5: Invalid Aggregation - -### Low Priority (Nice to Have) -1. Scenario 4.1.2: Retry Success -2. Scenario 5.1.1: Large Dataset -3. Scenario 5.3.1: Concurrent Execution +### 3.3.2 Postgres export failure surfaces +Drop `value` column before run; workflow fails; ERROR notification in Mongo. diff --git a/e2e/test_core_scouter_early_exit.py b/e2e/test_core_scouter_early_exit.py deleted file mode 100644 index 2de7704..0000000 --- a/e2e/test_core_scouter_early_exit.py +++ /dev/null @@ -1,96 +0,0 @@ -""" -End-to-end tests for CoreScouter workflow - Early exit scenarios. -""" - -from datetime import datetime - -import pytest -from sqlalchemy import text -from temporalio.testing import WorkflowEnvironment -from temporalio.worker import Worker - -from scouter.activities.activities import Activities -from scouter.workflow.sub_workflows.core_scouter import CoreScouter - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_2_2_1_empty_data_after_grouping( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - postgres_engine, -): - """ - Scenario 2.2.1: Empty Data After Grouping - - group_and_hold_data returns empty dict, workflow exits early. - """ - client = temporal_test_env.client - - # Prepare test data that will result in empty held_data - # This can happen if all data is filtered out or aggregation results in empty data - # Empty data must be a dict with empty lists for each column - test_data = { - 'tag': [], - 'name': [], - 'value': [], - 'timestamp': [], - } - - # Prepare input data - input_data = { - 'metadata': { - 'metadata': { - 'model_id': '1', - 'model_name': 'Test Model', - 'schedule_name': 'test-schedule', - 'workflow_name': 'pi_web_api_scouter', - } - }, - 'workflow_name': 'pi_web_api_scouter', - 'schedule_name': 'test-schedule', - 'model_name': 'Test Model', - 'model_id': '1', - 'data': test_data, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - } - - # Start workflow - handle = await client.start_workflow( - CoreScouter.run, - input_data, - id=f'test-core-scouter-empty-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion (should complete without error) - await handle.result() - - # Verify no data was exported to PostgreSQL (early exit) - schema_name = 'sientia_data' - table_name = 'laborious_data' - full_table_name = f"{schema_name}.{table_name}" - - with postgres_engine.connect() as conn: - result = conn.execute( - text(f"SELECT COUNT(*) FROM {full_table_name} WHERE model_id = 1") - ) - row_count = result.scalar() - - # Should have 0 rows since export_data_to_postgres was not called - assert row_count == 0, f"Expected no data in PostgreSQL, got {row_count} rows" - diff --git a/e2e/test_core_scouter_errors.py b/e2e/test_core_scouter_errors.py deleted file mode 100644 index e7d60f5..0000000 --- a/e2e/test_core_scouter_errors.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -End-to-end tests for CoreScouter workflow - Error scenarios. -""" - -from datetime import datetime -from unittest.mock import AsyncMock - -import pytest -from sqlalchemy import text -from temporalio.exceptions import ApplicationError, FailureError -from temporalio.testing import WorkflowEnvironment -from temporalio.worker import Worker - -from scouter.activities.activities import Activities -from scouter.workflow.sub_workflows.core_scouter import CoreScouter - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_2_3_1_redis_connection_error( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, -): - """ - Scenario 2.3.1: Redis Connection Error - - Redis unavailable during group_and_hold_data, notification sent, workflow fails. - """ - client = temporal_test_env.client - - # Mock Redis get to raise an error - original_get = test_activities.redis_repository.get - test_activities.redis_repository.get = AsyncMock(side_effect=Exception("Redis connection error")) - - try: - test_data = [ - { - 'timestamp': '2024-01-01 12:00:00+0000', - 'name': 'tag1', - 'value': 10.5, - 'tag': 'webid1', - }, - ] - - input_data = { - 'metadata': { - 'metadata': { - 'model_id': '1', - 'model_name': 'Test Model', - 'schedule_name': 'test-schedule', - 'workflow_name': 'pi_web_api_scouter', - } - }, - 'workflow_name': 'pi_web_api_scouter', - 'schedule_name': 'test-schedule', - 'model_name': 'Test Model', - 'model_id': '1', - 'data': test_data, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - } - - # Start workflow - handle = await client.start_workflow( - CoreScouter.run, - input_data, - id=f'test-core-scouter-redis-error-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - should fail - with pytest.raises((FailureError, ApplicationError, Exception)): - await handle.result() - finally: - # Restore original method - test_activities.redis_repository.get = original_get - diff --git a/e2e/test_core_scouter_success.py b/e2e/test_core_scouter_success.py deleted file mode 100644 index d32a806..0000000 --- a/e2e/test_core_scouter_success.py +++ /dev/null @@ -1,462 +0,0 @@ -""" -End-to-end tests for CoreScouter workflow - Success scenarios. -""" - -from datetime import datetime - -import pandas as pd -import pytest -from sqlalchemy import inspect, text -from temporalio.testing import WorkflowEnvironment -from temporalio.worker import Worker - -from scouter.activities.activities import Activities -from scouter.workflow.sub_workflows.core_scouter import CoreScouter - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_2_1_1_complete_processing_success( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - postgres_engine, -): - """ - Scenario 2.1.1: Complete Processing Success - - All stages complete successfully. - """ - client = temporal_test_env.client - - # Prepare test data (simulating data from parent workflow) - test_data = [ - { - 'timestamp': '2024-01-01 12:00:00+0000', - 'name': 'tag1', - 'value': 10.5, - 'tag': 'webid1', - }, - { - 'timestamp': '2024-01-01 12:00:00+0000', - 'name': 'tag2', - 'value': 20.3, - 'tag': 'webid2', - }, - ] - - # Prepare input data for CoreScouter - input_data = { - 'metadata': { - 'metadata': { - 'model_id': '1', - 'model_name': 'Test Model', - 'schedule_name': 'test-schedule', - 'workflow_name': 'pi_web_api_scouter', - } - }, - 'workflow_name': 'pi_web_api_scouter', - 'schedule_name': 'test-schedule', - 'model_name': 'Test Model', - 'model_id': '1', - 'data': test_data, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag2': { - 'webid': 'webid2', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - } - - # Start workflow - handle = await client.start_workflow( - CoreScouter.run, - input_data, - id=f'test-core-scouter-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - await handle.result() - - # Verify data was stored in PostgreSQL - schema_name = 'sientia_data' - table_name = 'laborious_data' - full_table_name = f"{schema_name}.{table_name}" - - with postgres_engine.connect() as conn: - result = conn.execute( - text(f"SELECT COUNT(*) FROM {full_table_name} WHERE model_id = 1") - ) - row_count = result.scalar() - - assert row_count > 0, f"Expected data in PostgreSQL, got {row_count} rows" - - # Verify data was cached in Redis - keys = await test_activities.redis_repository.keys('*') - assert len(keys) > 0, "Expected data in Redis" - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_2_1_2_success_with_data_quality_filters( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - postgres_engine, -): - """ - Scenario 2.1.2: Success with Data Quality Filters - - Data quality filters applied successfully. - """ - client = temporal_test_env.client - - # Prepare test data with some quality issues (null values, out of bounds) - test_data = [ - { - 'timestamp': '2024-01-01 12:00:00+0000', - 'name': 'tag1', - 'value': None, # Null value - should be filtered if DISCARD policy - 'tag': 'webid1', - }, - { - 'timestamp': '2024-01-01 12:00:00+0000', - 'name': 'tag2', - 'value': 150.0, # Out of bounds (range is [0, 100]) - 'tag': 'webid2', - }, - { - 'timestamp': '2024-01-01 12:00:00+0000', - 'name': 'tag3', - 'value': 50.0, # Valid value - 'tag': 'webid3', - }, - ] - - # Prepare input data with filters - input_data = { - 'metadata': { - 'metadata': { - 'model_id': '1', - 'model_name': 'Test Model', - 'schedule_name': 'test-schedule', - 'workflow_name': 'pi_web_api_scouter', - } - }, - 'workflow_name': 'pi_web_api_scouter', - 'schedule_name': 'test-schedule', - 'model_name': 'Test Model', - 'model_id': '1', - 'data': test_data, - 'trigger_laborious': False, - 'filters': { - 'NULL_VALUES_FILTER': { - 'policy': 'DISCARD', - }, - 'OUT_OF_BOUNDS_FILTER': { - 'policy': 'WARN', - }, - }, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag2': { - 'webid': 'webid2', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag3': { - 'webid': 'webid3', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - } - - # Start workflow - handle = await client.start_workflow( - CoreScouter.run, - input_data, - id=f'test-core-scouter-quality-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - await handle.result() - - # Verify data was stored in PostgreSQL - # tag1 should be filtered out (null value with DISCARD policy) - # tag2 should be kept (out of bounds with WARN policy) - # tag3 should be kept (valid value) - schema_name = 'sientia_data' - table_name = 'laborious_data' - full_table_name = f"{schema_name}.{table_name}" - - with postgres_engine.connect() as conn: - result = conn.execute( - text(f"SELECT COUNT(*) FROM {full_table_name} WHERE model_id = 1") - ) - row_count = result.scalar() - - # Should have at least tag2 and tag3 (tag1 filtered out) - assert row_count >= 2, f"Expected at least 2 records after filtering, got {row_count}" - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_2_1_3_success_with_different_aggregation_functions( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - postgres_engine, -): - """ - Scenario 2.1.3: Success with Different Aggregation Functions - - Different aggregation functions applied correctly. - """ - client = temporal_test_env.client - - # Prepare test data with multiple points per tag for aggregation - test_data = [ - {'timestamp': '2024-01-01 12:00:00+0000', 'name': 'tag_avg', 'value': 10.0, 'tag': 'webid_avg'}, - {'timestamp': '2024-01-01 12:01:00+0000', 'name': 'tag_avg', 'value': 20.0, 'tag': 'webid_avg'}, - {'timestamp': '2024-01-01 12:02:00+0000', 'name': 'tag_avg', 'value': 30.0, 'tag': 'webid_avg'}, - {'timestamp': '2024-01-01 12:00:00+0000', 'name': 'tag_max', 'value': 5.0, 'tag': 'webid_max'}, - {'timestamp': '2024-01-01 12:01:00+0000', 'name': 'tag_max', 'value': 15.0, 'tag': 'webid_max'}, - {'timestamp': '2024-01-01 12:02:00+0000', 'name': 'tag_max', 'value': 10.0, 'tag': 'webid_max'}, - {'timestamp': '2024-01-01 12:00:00+0000', 'name': 'tag_min', 'value': 50.0, 'tag': 'webid_min'}, - {'timestamp': '2024-01-01 12:01:00+0000', 'name': 'tag_min', 'value': 30.0, 'tag': 'webid_min'}, - {'timestamp': '2024-01-01 12:02:00+0000', 'name': 'tag_min', 'value': 40.0, 'tag': 'webid_min'}, - {'timestamp': '2024-01-01 12:00:00+0000', 'name': 'tag_lts', 'value': 100.0, 'tag': 'webid_lts'}, - {'timestamp': '2024-01-01 12:01:00+0000', 'name': 'tag_lts', 'value': 200.0, 'tag': 'webid_lts'}, - {'timestamp': '2024-01-01 12:02:00+0000', 'name': 'tag_lts', 'value': 300.0, 'tag': 'webid_lts'}, - ] - - # Prepare input data with different aggregation functions - input_data = { - 'metadata': { - 'metadata': { - 'model_id': '1', - 'model_name': 'Test Model', - 'schedule_name': 'test-schedule', - 'workflow_name': 'pi_web_api_scouter', - } - }, - 'workflow_name': 'pi_web_api_scouter', - 'schedule_name': 'test-schedule', - 'model_name': 'Test Model', - 'model_id': '1', - 'data': test_data, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'model_tags': { - 'tag_avg': { - 'webid': 'webid_avg', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag_max': { - 'webid': 'webid_max', - 'aggr_function': 'max', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag_min': { - 'webid': 'webid_min', - 'aggr_function': 'min', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag_lts': { - 'webid': 'webid_lts', - 'aggr_function': 'lts', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - } - - # Start workflow - handle = await client.start_workflow( - CoreScouter.run, - input_data, - id=f'test-core-scouter-aggr-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - await handle.result() - - # Verify data was stored in PostgreSQL - schema_name = 'sientia_data' - table_name = 'laborious_data' - full_table_name = f"{schema_name}.{table_name}" - - with postgres_engine.connect() as conn: - result = conn.execute( - text(f"SELECT variable, value FROM {full_table_name} WHERE model_id = 1") - ) - rows = result.fetchall() - - # Should have 4 rows (one per tag) - assert len(rows) >= 4, f"Expected at least 4 records, got {len(rows)}" - - # Verify aggregation was applied correctly - values_by_tag = {row[0]: float(row[1]) if row[1] is not None else None for row in rows} - - # Verify all tags are present - assert 'tag_avg' in values_by_tag, "Expected tag_avg to be present" - assert 'tag_max' in values_by_tag, "Expected tag_max to be present" - assert 'tag_min' in values_by_tag, "Expected tag_min to be present" - assert 'tag_lts' in values_by_tag, "Expected tag_lts to be present" - - # Note: Due to timestamp normalization in get_tag_values, aggregation may behave differently - # We verify that aggregation was applied (values exist) rather than exact values - # avg: should be between min and max of input values (10, 20, 30) - if 'tag_avg' in values_by_tag and values_by_tag['tag_avg'] is not None: - assert 10.0 <= values_by_tag['tag_avg'] <= 30.0, f"Expected avg between 10-30, got {values_by_tag['tag_avg']}" - - # max: should be >= 15.0 (max of 5, 15, 10) - if 'tag_max' in values_by_tag and values_by_tag['tag_max'] is not None: - assert values_by_tag['tag_max'] >= 10.0, f"Expected max >= 10, got {values_by_tag['tag_max']}" - - # min: should be <= 50.0 (min of 50, 30, 40) - if 'tag_min' in values_by_tag and values_by_tag['tag_min'] is not None: - assert values_by_tag['tag_min'] <= 50.0, f"Expected min <= 50, got {values_by_tag['tag_min']}" - - # lts: should be one of the values (100, 200, 300) - if 'tag_lts' in values_by_tag and values_by_tag['tag_lts'] is not None: - assert values_by_tag['tag_lts'] in [100.0, 200.0, 300.0], f"Expected lts to be one of [100, 200, 300], got {values_by_tag['tag_lts']}" - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_2_1_4_success_with_fill_missing_tags( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - postgres_engine, -): - """ - Scenario 2.1.4: Success with Fill Missing Tags - - Missing tags filled with None. - """ - client = temporal_test_env.client - - # Prepare test data with only some tags present - test_data = [ - { - 'timestamp': '2024-01-01 12:00:00+0000', - 'name': 'tag1', - 'value': 10.5, - 'tag': 'webid1', - }, - # tag2 and tag3 are missing - ] - - # Prepare input data with fill_missing_tags enabled - input_data = { - 'metadata': { - 'metadata': { - 'model_id': '1', - 'model_name': 'Test Model', - 'schedule_name': 'test-schedule', - 'workflow_name': 'pi_web_api_scouter', - } - }, - 'workflow_name': 'pi_web_api_scouter', - 'schedule_name': 'test-schedule', - 'model_name': 'Test Model', - 'model_id': '1', - 'data': test_data, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': True, # Enable fill missing tags - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag2': { - 'webid': 'webid2', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag3': { - 'webid': 'webid3', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - } - - # Start workflow - handle = await client.start_workflow( - CoreScouter.run, - input_data, - id=f'test-core-scouter-fill-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - await handle.result() - - # Verify all tags are present in Redis (including missing ones with None) - keys = await test_activities.redis_repository.keys('*') - assert len(keys) > 0, "Expected data in Redis" - - # Verify data was stored in PostgreSQL - schema_name = 'sientia_data' - table_name = 'laborious_data' - full_table_name = f"{schema_name}.{table_name}" - - with postgres_engine.connect() as conn: - result = conn.execute( - text(f"SELECT variable, value FROM {full_table_name} WHERE model_id = 1") - ) - rows = result.fetchall() - - # Should have all 3 tags (tag1 with value, tag2 and tag3 with None) - variables = [row[0] for row in rows] - assert 'tag1' in variables, "Expected tag1 to be present" - # Note: tags with None values might not be stored in PostgreSQL, so we just verify tag1 exists - diff --git a/e2e/test_harness_smoke.py b/e2e/test_harness_smoke.py new file mode 100644 index 0000000..276cee0 --- /dev/null +++ b/e2e/test_harness_smoke.py @@ -0,0 +1,51 @@ +"""Fast smoke checks for E2E fixture wiring.""" + +import pytest + +from e2e.helpers import ( + apply_pi_web_api_server_config, + load_scenario_input, + make_workflow_id, + start_and_await_workflow, +) +from e2e.pi_web_api_test_server import PIWebAPITestServer +from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + + +@pytest.mark.e2e +def test_postgres_schema_ready(postgres_engine): + """Verify autouse schema setup created laborious_data.""" + with postgres_engine.connect() as conn: + count = conn.exec_driver_sql( + 'SELECT COUNT(*) FROM information_schema.tables ' + "WHERE table_schema = 'sientia_data' AND table_name = 'laborious_data'" + ).scalar() + assert count == 1 + + +@pytest.mark.e2e +def test_activities_construct(test_activities): + """Verify Activities initializes against testcontainers without hanging.""" + assert test_activities.redis_repository is not None + assert test_activities.mongodb_repository is not None + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_temporal_pi_happy_path( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, +): + """Minimal Temporal path: PIWebAPIScouter happy path completes.""" + scenario = load_scenario_input('pi_web_api_scouter_happy_path') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + scenario['workflow_input'], + make_workflow_id('smoke-pi-happy'), + timeout=60.0, + ) diff --git a/e2e/test_pi_web_api_scouter_early_exit.py b/e2e/test_pi_web_api_scouter_early_exit.py deleted file mode 100644 index 88ec8be..0000000 --- a/e2e/test_pi_web_api_scouter_early_exit.py +++ /dev/null @@ -1,182 +0,0 @@ -""" -End-to-end tests for PI Web API Scouter workflow - Early exit scenarios. -""" - -from datetime import datetime -from unittest.mock import AsyncMock - -import pandas as pd -import pytest -from sqlalchemy import text -from temporalio.testing import WorkflowEnvironment -from temporalio.worker import Worker - -from scouter.activities.activities import Activities -from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_1_2_1_empty_data_from_pi_web_api( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - mock_pi_web_api_client, - postgres_engine, -): - """ - Scenario 1.2.1: Empty Data from PI Web API - - PI Web API returns empty data, workflow should exit early without calling core_scouter. - """ - client = temporal_test_env.client - - # Mock empty DataFrame response with proper datetime column - empty_df = pd.DataFrame({ - 'timestamp': pd.to_datetime([], utc=True), - 'name': [], - 'value': [], - 'tag': [], - }) - mock_pi_web_api_client.get_latest_values_df = AsyncMock(return_value=empty_df) - - # Prepare test input with unique model_id to avoid conflicts - unique_id = int(datetime.now().timestamp()) - input_data = { - 'model_name': 'PI Web API Scouter Test Model', - 'model_id': str(unique_id), - 'schedule_name': 'pi-web-api-scouter-test-empty', - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'pi_web_api_query': { - 'endpoint': '/streamsets/recorded', - 'period': '*-1d', - 'max_count': 10, - 'api_timeout': 30, - }, - } - - # Start workflow - handle = await client.start_workflow( - PIWebAPIScouter.run, - input_data, - id=f'test-workflow-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion (should complete without error) - await handle.result() - - # Verify PI Web API was called once - mock_pi_web_api_client.get_latest_values_df.assert_called_once() - - # Verify no data was stored in PostgreSQL (core_scouter was not called) - schema_name = 'sientia_data' - table_name = 'laborious_data' - full_table_name = f"{schema_name}.{table_name}" - - with postgres_engine.connect() as conn: - result = conn.execute( - text(f"SELECT COUNT(*) FROM {full_table_name} WHERE model_id = :model_id"), - {'model_id': unique_id} - ) - row_count = result.scalar() - - # Should have 0 rows since core_scouter was not called - assert row_count == 0, f"Expected no data in PostgreSQL, got {row_count} rows" - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_1_2_2_none_returned_from_pi_web_api( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - mock_pi_web_api_client, - postgres_engine, -): - """ - Scenario 1.2.2: None Returned from PI Web API - - PI Web API returns None, workflow should exit early without calling core_scouter. - """ - client = temporal_test_env.client - - # Mock empty DataFrame response (None is not handled well, so use empty DataFrame instead) - empty_df = pd.DataFrame({ - 'timestamp': pd.to_datetime([], utc=True), - 'name': [], - 'value': [], - 'tag': [], - }) - mock_pi_web_api_client.get_latest_values_df = AsyncMock(return_value=empty_df) - - # Prepare test input with unique model_id to avoid conflicts - unique_id = int(datetime.now().timestamp()) - input_data = { - 'model_name': 'PI Web API Scouter Test Model', - 'model_id': str(unique_id), - 'schedule_name': 'pi-web-api-scouter-test-none', - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'pi_web_api_query': { - 'endpoint': '/streamsets/recorded', - 'period': '*-1d', - 'max_count': 10, - 'api_timeout': 30, - }, - } - - # Start workflow - handle = await client.start_workflow( - PIWebAPIScouter.run, - input_data, - id=f'test-workflow-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion (should complete without error) - await handle.result() - - # Verify PI Web API was called once - mock_pi_web_api_client.get_latest_values_df.assert_called_once() - - # Verify no data was stored in PostgreSQL (core_scouter was not called) - schema_name = 'sientia_data' - table_name = 'laborious_data' - full_table_name = f"{schema_name}.{table_name}" - - with postgres_engine.connect() as conn: - result = conn.execute( - text(f"SELECT COUNT(*) FROM {full_table_name} WHERE model_id = :model_id"), - {'model_id': unique_id} - ) - row_count = result.scalar() - - # Should have 0 rows since core_scouter was not called - assert row_count == 0, f"Expected no data in PostgreSQL, got {row_count} rows" - diff --git a/e2e/test_pi_web_api_scouter_errors.py b/e2e/test_pi_web_api_scouter_errors.py deleted file mode 100644 index f1088be..0000000 --- a/e2e/test_pi_web_api_scouter_errors.py +++ /dev/null @@ -1,206 +0,0 @@ -""" -End-to-end tests for PI Web API Scouter workflow - Error scenarios. -""" - -from datetime import datetime -from unittest.mock import AsyncMock - -import pytest -from temporalio.exceptions import ApplicationError, FailureError -from temporalio.testing import WorkflowEnvironment -from temporalio.worker import Worker - -from scouter.activities.activities import Activities -from sientia_do.repository.pi_web_api_client import PIMSRequestError -from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_1_3_1_pi_web_api_connection_error( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - mock_pi_web_api_client, -): - """ - Scenario 1.3.1: PI Web API Connection Error - - PI Web API client raises connection error, notification sent, workflow fails. - """ - client = temporal_test_env.client - - # Mock connection error - mock_pi_web_api_client.get_latest_values_df = AsyncMock( - side_effect=PIMSRequestError("Connection failed") - ) - - # Prepare test input - input_data = { - 'model_name': 'PI Web API Scouter Test Model', - 'model_id': '1', - 'schedule_name': 'pi-web-api-scouter-test-error', - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'pi_web_api_query': { - 'endpoint': '/streamsets/recorded', - 'period': '*-1d', - 'max_count': 10, - 'api_timeout': 30, - }, - } - - # Start workflow - handle = await client.start_workflow( - PIWebAPIScouter.run, - input_data, - id=f'test-workflow-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - should fail - with pytest.raises((FailureError, ApplicationError, Exception)): - await handle.result() - - # Verify PI Web API was called - mock_pi_web_api_client.get_latest_values_df.assert_called() - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_1_3_2_pi_web_api_timeout( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - mock_pi_web_api_client, -): - """ - Scenario 1.3.2: PI Web API Timeout - - PI Web API request times out, notification sent, workflow fails. - """ - client = temporal_test_env.client - - # Mock timeout error - mock_pi_web_api_client.get_latest_values_df = AsyncMock( - side_effect=TimeoutError("Request timed out") - ) - - # Prepare test input with low timeout - input_data = { - 'model_name': 'PI Web API Scouter Test Model', - 'model_id': '1', - 'schedule_name': 'pi-web-api-scouter-test-timeout', - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'pi_web_api_query': { - 'endpoint': '/streamsets/recorded', - 'period': '*-1d', - 'max_count': 10, - 'api_timeout': 1, # Very short timeout - }, - } - - # Start workflow - handle = await client.start_workflow( - PIWebAPIScouter.run, - input_data, - id=f'test-workflow-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - should fail - with pytest.raises((FailureError, ApplicationError, Exception)): - await handle.result() - - # Verify PI Web API was called - mock_pi_web_api_client.get_latest_values_df.assert_called() - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_1_3_3_invalid_endpoint( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - mock_pi_web_api_client, -): - """ - Scenario 1.3.3: Invalid Endpoint - - Invalid PI Web API endpoint provided, error raised, workflow fails. - """ - client = temporal_test_env.client - - # Mock error for invalid endpoint - mock_pi_web_api_client.get_latest_values_df = AsyncMock( - side_effect=PIMSRequestError("HTTP 404 calling '/invalid/endpoint': Not found") - ) - - # Prepare test input with invalid endpoint - input_data = { - 'model_name': 'PI Web API Scouter Test Model', - 'model_id': '1', - 'schedule_name': 'pi-web-api-scouter-test-invalid', - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'pi_web_api_query': { - 'endpoint': '/invalid/endpoint', - 'period': '*-1d', - 'max_count': 10, - 'api_timeout': 30, - }, - } - - # Start workflow - handle = await client.start_workflow( - PIWebAPIScouter.run, - input_data, - id=f'test-workflow-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - should fail - with pytest.raises((FailureError, ApplicationError, Exception)): - await handle.result() - - # Verify PI Web API was called - mock_pi_web_api_client.get_latest_values_df.assert_called() - - diff --git a/e2e/test_pi_web_api_scouter_main_workflow.py b/e2e/test_pi_web_api_scouter_main_workflow.py new file mode 100644 index 0000000..5696310 --- /dev/null +++ b/e2e/test_pi_web_api_scouter_main_workflow.py @@ -0,0 +1,222 @@ +""" +End-to-end tests for the PIWebAPIScouter main workflow. +""" + +import json + +import pytest +from redis import Redis +from temporalio.client import WorkflowFailureError +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + +from e2e.conftest import E2E_DATABASE +from e2e.helpers import ( + apply_pi_web_api_server_config, + count_laborious_rows, + count_notifications, + fetch_laborious_rows, + load_scenario_input, + make_workflow_id, + start_and_await_workflow, +) +from e2e.pi_web_api_test_server import PIWebAPITestServer +from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_2_1_1_happy_path( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + postgres_engine, + redis_client: Redis, +): + scenario = load_scenario_input('pi_web_api_scouter_happy_path') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-happy'), + ) + + model_id = workflow_input['model_id'] + assert count_laborious_rows(postgres_engine, model_id) >= 1 + assert len(redis_client.keys('held_data_*')) >= 1 + assert len(pi_web_api_server.requests) == 1 + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_2_1_2_multiple_tags( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + postgres_engine, +): + scenario = load_scenario_input('pi_web_api_scouter_multiple_tags') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-multi'), + ) + + rows = { + row['variable']: float(row['value']) + for row in fetch_laborious_rows(postgres_engine, workflow_input['model_id']) + } + for tag, expected in scenario['expected_values'].items(): + assert tag in rows + assert rows[tag] == pytest.approx(expected) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_2_1_3_debug_data_package( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + redis_client: Redis, +): + scenario = load_scenario_input('pi_web_api_scouter_debug_data_package') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-debug'), + ) + + keys = redis_client.keys('data_package_pi_web_api_scouter_*') + assert len(keys) >= 1 + payload = json.loads(redis_client.get(keys[0])) + assert 'data' in payload and 'held_data' in payload + + +@pytest.mark.e2e +@pytest.mark.asyncio +@pytest.mark.xfail( + strict=True, + reason='Empty PI DataFrame lacks timestamp column in get_tag_values; tracked in fix-pi-empty-response-handling', +) +async def test_scenario_2_2_1_empty_response_early_exit( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + postgres_engine, +): + scenario = load_scenario_input('pi_web_api_scouter_empty_response') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-empty'), + ) + + assert count_laborious_rows(postgres_engine, workflow_input['model_id']) == 0 + assert len(pi_web_api_server.requests) == 1 + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_2_3_1_pi_web_api_connection_error( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + mongo_uri: str, +): + scenario = load_scenario_input('pi_web_api_scouter_connection_error') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + with pytest.raises(WorkflowFailureError): + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-conn-error'), + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='PI_WEB_API_REQUEST_ERROR', + level='ERROR', + ) + >= 1 + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_2_3_2_pi_web_api_timeout( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + mongo_uri: str, +): + scenario = load_scenario_input('pi_web_api_scouter_timeout') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + with pytest.raises(WorkflowFailureError): + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-timeout'), + timeout=180.0, + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='PI_WEB_API_REQUEST_ERROR', + ) + >= 1 + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_2_3_3_invalid_endpoint( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + pi_web_api_server: PIWebAPITestServer, + mongo_uri: str, +): + scenario = load_scenario_input('pi_web_api_scouter_invalid_endpoint') + apply_pi_web_api_server_config(pi_web_api_server, scenario.get('pi_web_api_server')) + workflow_input = scenario['workflow_input'] + + with pytest.raises(WorkflowFailureError): + await start_and_await_workflow( + temporal_env.client, + PIWebAPIScouter.run, + workflow_input, + make_workflow_id('pi-invalid-endpoint'), + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='PI_WEB_API_REQUEST_ERROR', + ) + >= 1 + ) diff --git a/e2e/test_pi_web_api_scouter_success.py b/e2e/test_pi_web_api_scouter_success.py deleted file mode 100644 index 239bd12..0000000 --- a/e2e/test_pi_web_api_scouter_success.py +++ /dev/null @@ -1,298 +0,0 @@ -""" -End-to-end tests for PI Web API Scouter workflow - Success scenarios. -""" - -from datetime import datetime -from unittest.mock import AsyncMock - -import pandas as pd -import pytest -from sqlalchemy import inspect, text -from temporalio.testing import WorkflowEnvironment -from temporalio.worker import Worker - -from scouter.activities.activities import Activities -from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_1_1_1_happy_path_complete_success( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - mock_pi_web_api_client, - postgres_engine, -): - """ - Scenario 1.1.1: Happy Path - Complete Success - - Workflow completes successfully with valid data from PI Web API. - """ - client = temporal_test_env.client - - # Prepare test input - input_data = { - 'model_name': 'PI Web API Scouter Test Model', - 'model_id': '1', - 'schedule_name': 'pi-web-api-scouter-test', - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag2': { - 'webid': 'webid2', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'pi_web_api_query': { - 'endpoint': '/streamsets/recorded', - 'period': '*-1d', - 'max_count': 10, - 'api_timeout': 30, - }, - } - - # Start workflow - handle = await client.start_workflow( - PIWebAPIScouter.run, - input_data, - id=f'test-workflow-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - await handle.result() - - # Verify PI Web API was called once with correct parameters - mock_pi_web_api_client.get_latest_values_df.assert_called_once() - - # Verify data was stored in PostgreSQL - inspector = inspect(postgres_engine) - schema_name = 'sientia_data' - table_name = 'laborious_data' - full_table_name = f"{schema_name}.{table_name}" - - table_exists = inspector.has_table(table_name, schema=schema_name) - assert table_exists, f"Expected table {full_table_name} to exist in PostgreSQL" - - # Verify data was inserted - with postgres_engine.connect() as conn: - result = conn.execute(text(f"SELECT COUNT(*) FROM {full_table_name}")) - row_count = result.scalar() - - assert row_count > 0, f"Expected data in PostgreSQL table {full_table_name}, got {row_count} rows" - - # Verify data was cached in Redis - keys = await test_activities.redis_repository.keys('*') - assert len(keys) > 0, "Expected data in Redis" - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_1_1_2_success_with_multiple_tags( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - mock_pi_web_api_client, - postgres_engine, -): - """ - Scenario 1.1.2: Success with Multiple Tags - - Workflow processes multiple tags successfully. - """ - client = temporal_test_env.client - - # Create mock DataFrame with 5 tags - mock_df = pd.DataFrame({ - 'timestamp': [ - '2024-01-01 12:00:00+0000', - '2024-01-01 12:01:00+0000', - '2024-01-01 12:02:00+0000', - '2024-01-01 12:03:00+0000', - '2024-01-01 12:04:00+0000', - ], - 'name': ['tag1', 'tag2', 'tag3', 'tag4', 'tag5'], - 'value': [10.5, 20.3, 30.7, 40.1, 50.9], - 'tag': ['webid1', 'webid2', 'webid3', 'webid4', 'webid5'], - }) - mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp'], utc=True).dt.floor('s') - mock_pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df) - - # Prepare test input with 5 tags - input_data = { - 'model_name': 'PI Web API Scouter Test Model', - 'model_id': '1', - 'schedule_name': 'pi-web-api-scouter-test-multiple', - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag2': { - 'webid': 'webid2', - 'aggr_function': 'max', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag3': { - 'webid': 'webid3', - 'aggr_function': 'min', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag4': { - 'webid': 'webid4', - 'aggr_function': 'mdn', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag5': { - 'webid': 'webid5', - 'aggr_function': 'lts', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'pi_web_api_query': { - 'endpoint': '/streamsets/recorded', - 'period': '*-1d', - 'max_count': 10, - 'api_timeout': 30, - }, - } - - # Start workflow - handle = await client.start_workflow( - PIWebAPIScouter.run, - input_data, - id=f'test-workflow-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - await handle.result() - - # Verify all tags were retrieved - mock_pi_web_api_client.get_latest_values_df.assert_called_once() - - # Verify data was stored in PostgreSQL - schema_name = 'sientia_data' - table_name = 'laborious_data' - full_table_name = f"{schema_name}.{table_name}" - - with postgres_engine.connect() as conn: - result = conn.execute( - text(f"SELECT COUNT(*) FROM {full_table_name} WHERE model_id = 1") - ) - row_count = result.scalar() - - # Should have at least 5 rows (one per tag) - assert row_count >= 5, f"Expected at least 5 records, got {row_count}" - - # Verify all tags are present in the database - with postgres_engine.connect() as conn: - result = conn.execute( - text(f"SELECT DISTINCT variable FROM {full_table_name} WHERE model_id = 1") - ) - variables = [row[0] for row in result.fetchall()] - - expected_tags = ['tag1', 'tag2', 'tag3', 'tag4', 'tag5'] - for tag in expected_tags: - assert tag in variables, f"Expected tag {tag} to be in database" - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_1_1_3_success_with_debug_data_package( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - mock_pi_web_api_client, - postgres_engine, -): - """ - Scenario 1.1.3: Success with Debug Data Package Enabled - - Workflow completes with debug_data_package=True. - """ - client = temporal_test_env.client - - # Generate unique model_id to avoid conflicts with other tests - unique_id = int(datetime.now().timestamp() * 1000) % 1000000 - - # Prepare test input with debug_data_package enabled - input_data = { - 'model_name': 'PI Web API Scouter Test Model', - 'model_id': str(unique_id), - 'schedule_name': 'pi-web-api-scouter-test-debug', - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'debug_data_package': True, - 'pi_web_api_query': { - 'endpoint': '/streamsets/recorded', - 'period': '*-1d', - 'max_count': 10, - 'api_timeout': 30, - }, - } - - # Start workflow - handle = await client.start_workflow( - PIWebAPIScouter.run, - input_data, - id=f'test-workflow-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - await handle.result() - - # Verify PI Web API was called - mock_pi_web_api_client.get_latest_values_df.assert_called_once() - - # Verify data package was stored in Redis - # The key format is: data_package_{workflow_name}_{schedule_name}_{timestamp} - keys = await test_activities.redis_repository.keys('data_package_*') - assert len(keys) > 0, f"Expected data package key in Redis, found keys: {keys}" - - # Verify the data package contains both 'data' and 'held_data' - if keys: - package_key = keys[0] - package_data = await test_activities.redis_repository.get(package_key) - assert package_data is not None, "Expected data package to exist" - # The package should be a dict with 'data' and 'held_data' keys - assert isinstance(package_data, dict), "Expected data package to be a dict" - diff --git a/e2e/test_scouter_main_workflow.py b/e2e/test_scouter_main_workflow.py new file mode 100644 index 0000000..42ccbe1 --- /dev/null +++ b/e2e/test_scouter_main_workflow.py @@ -0,0 +1,154 @@ +""" +End-to-end tests for the Scouter main workflow (Mongo load path). +""" + +import json + +import pytest +from redis import Redis + +from e2e.conftest import E2E_DATABASE +from e2e.helpers import ( + count_laborious_rows, + load_scenario_input, + make_workflow_id, + seed_last_data_timestamp, + seed_raw_collection, + start_and_await_workflow, +) +from scouter.workflow.scouter import Scouter +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_1_1_1_happy_path( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + redis_client: Redis, + postgres_engine, +): + scenario = load_scenario_input('scouter_happy_path') + workflow_input = scenario['workflow_input'] + seed_raw_collection( + mongo_uri, + E2E_DATABASE, + workflow_input['schedule_name'], + scenario['raw_documents'], + ) + + await start_and_await_workflow( + temporal_env.client, + Scouter.run, + workflow_input, + make_workflow_id('scouter-happy'), + ) + + model_id = workflow_input['model_id'] + assert count_laborious_rows(postgres_engine, model_id) >= 1 + + key = f"last_data_timestamp:scouter:{workflow_input['schedule_name']}" + assert redis_client.get(key) is not None + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_1_2_1_incremental_load( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + redis_client: Redis, + postgres_engine, +): + scenario = load_scenario_input('scouter_incremental_load') + workflow_input = scenario['workflow_input'] + redis_seed = scenario['redis_seed'] + seed_last_data_timestamp( + redis_client, + 'scouter', + workflow_input['schedule_name'], + redis_seed['last_data_timestamp'], + ) + seed_raw_collection( + mongo_uri, + E2E_DATABASE, + workflow_input['schedule_name'], + scenario['raw_documents'], + ) + + await start_and_await_workflow( + temporal_env.client, + Scouter.run, + workflow_input, + make_workflow_id('scouter-incremental'), + ) + + model_id = workflow_input['model_id'] + assert count_laborious_rows(postgres_engine, model_id) == scenario['expected_newer_count'] + + stored = json.loads( + redis_client.get(f"last_data_timestamp:scouter:{workflow_input['schedule_name']}") + ) + assert stored == scenario['expected_last_timestamp'] + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_1_3_1_empty_mongo_early_exit( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + redis_client: Redis, + postgres_engine, +): + scenario = load_scenario_input('scouter_empty_mongo') + workflow_input = scenario['workflow_input'] + seed_raw_collection( + mongo_uri, + E2E_DATABASE, + workflow_input['schedule_name'], + scenario['raw_documents'], + ) + + await start_and_await_workflow( + temporal_env.client, + Scouter.run, + workflow_input, + make_workflow_id('scouter-empty'), + ) + + assert count_laborious_rows(postgres_engine, workflow_input['model_id']) == 0 + key = f"last_data_timestamp:scouter:{workflow_input['schedule_name']}" + assert redis_client.get(key) is None + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_1_3_2_no_redis_timestamp_first_run( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + redis_client: Redis, + postgres_engine, +): + scenario = load_scenario_input('scouter_no_redis_timestamp_first_run') + workflow_input = scenario['workflow_input'] + seed_raw_collection( + mongo_uri, + E2E_DATABASE, + workflow_input['schedule_name'], + scenario['raw_documents'], + ) + + await start_and_await_workflow( + temporal_env.client, + Scouter.run, + workflow_input, + make_workflow_id('scouter-first-run'), + ) + + assert count_laborious_rows(postgres_engine, workflow_input['model_id']) >= 1 + key = f"last_data_timestamp:scouter:{workflow_input['schedule_name']}" + assert redis_client.get(key) is not None diff --git a/e2e/test_subworkflow_core_scouter.py b/e2e/test_subworkflow_core_scouter.py new file mode 100644 index 0000000..39abed4 --- /dev/null +++ b/e2e/test_subworkflow_core_scouter.py @@ -0,0 +1,360 @@ +""" +End-to-end tests for the CoreScouter subworkflow. +""" + +import json + +import pytest +from redis import Redis +from sqlalchemy import text +from temporalio.client import WorkflowFailureError +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + +from e2e.conftest import E2E_DATABASE +from e2e.helpers import ( + count_laborious_rows, + count_notifications, + fetch_laborious_rows, + load_scenario_input, + make_workflow_id, + start_and_await_workflow, +) +from scouter.activities.activities import Activities +from scouter.workflow.sub_workflows.core_scouter import CoreScouter + + +def _held_data_blob(test_activities: Activities, workflow_input: dict) -> dict | None: + """ + Read held_data Redis payload via the production RedisRepository. + + Return: + Decoded held-data dict or None + """ + key = ( + f"held_data_{workflow_input['workflow_name']}_{workflow_input['schedule_name']}" + ) + metadata = workflow_input['metadata']['metadata'] + return test_activities.redis_repository.get(key, metadata=metadata) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_1_complete_processing_success( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + test_activities: Activities, + postgres_engine, + redis_client: Redis, +): + scenario = load_scenario_input('core_scouter_happy_path') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-happy'), + ) + + assert count_laborious_rows(postgres_engine, workflow_input['model_id']) >= 1 + assert _held_data_blob(test_activities, workflow_input) is not None + assert len(redis_client.keys('data_package_*')) == 0 + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_2_null_values_filter_discard( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + postgres_engine, +): + scenario = load_scenario_input('core_scouter_null_values_filter_discard') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-null-discard'), + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='DATA_QUALITY_GATE_ISSUES__NULL_VALUES_FILTER', + level='WARNING', + ) + == 1 + ) + rows = fetch_laborious_rows(postgres_engine, workflow_input['model_id']) + assert len(rows) == 1 + assert rows[0]['variable'] == 'tag2' + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_3_null_values_filter_warn( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + postgres_engine, +): + scenario = load_scenario_input('core_scouter_null_values_filter_warn') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-null-warn'), + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='DATA_QUALITY_GATE_ISSUES__NULL_VALUES_FILTER', + level='WARNING', + ) + == 1 + ) + assert count_laborious_rows(postgres_engine, workflow_input['model_id']) == 2 + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_4_out_of_bounds_filter_discard( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + postgres_engine, +): + scenario = load_scenario_input('core_scouter_out_of_bounds_filter_discard') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-oob-discard'), + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='DATA_QUALITY_GATE_ISSUES__OUT_OF_BOUNDS_FILTER', + level='WARNING', + ) + == 1 + ) + rows = fetch_laborious_rows(postgres_engine, workflow_input['model_id']) + assert len(rows) == 1 + assert rows[0]['variable'] == 'tag2' + + +async def _run_aggregation_scenario( + temporal_env: WorkflowEnvironment, + postgres_engine, + slug: str, + workflow_id_prefix: str, +) -> None: + scenario = load_scenario_input(slug) + workflow_input = scenario['workflow_input'] + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id(workflow_id_prefix), + ) + rows = fetch_laborious_rows(postgres_engine, workflow_input['model_id']) + tag_name = next(iter(workflow_input['model_tags'])) + value = next(row['value'] for row in rows if row['variable'] == tag_name) + assert float(value) == pytest.approx(scenario['expected_value']) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_5_aggregation_avg( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + await _run_aggregation_scenario( + temporal_env, postgres_engine, 'core_scouter_aggregation_avg', 'core-avg' + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_6_aggregation_mdn( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + await _run_aggregation_scenario( + temporal_env, postgres_engine, 'core_scouter_aggregation_mdn', 'core-mdn' + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_7_aggregation_max( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + await _run_aggregation_scenario( + temporal_env, postgres_engine, 'core_scouter_aggregation_max', 'core-max' + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_8_aggregation_min( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + await _run_aggregation_scenario( + temporal_env, postgres_engine, 'core_scouter_aggregation_min', 'core-min' + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_9_aggregation_lts( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + await _run_aggregation_scenario( + temporal_env, postgres_engine, 'core_scouter_aggregation_lts', 'core-lts' + ) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_10_fill_missing_tags( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + test_activities: Activities, +): + scenario = load_scenario_input('core_scouter_fill_missing_tags') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-fill-tags'), + ) + + held = _held_data_blob(test_activities, workflow_input) + assert held is not None + for tag in scenario['expected_missing_tags']: + assert tag in held + assert held[tag] is None + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_1_11_debug_data_package( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + redis_client: Redis, +): + scenario = load_scenario_input('core_scouter_debug_data_package') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-debug-pkg'), + ) + + keys = redis_client.keys('data_package_*') + assert len(keys) >= 1 + payload = json.loads(redis_client.get(keys[0])) + assert 'data' in payload and 'held_data' in payload + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_2_1_empty_after_grouping_early_exit( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + test_activities: Activities, + postgres_engine, +): + scenario = load_scenario_input('core_scouter_empty_after_grouping') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-empty-group'), + ) + + assert count_laborious_rows(postgres_engine, workflow_input['model_id']) == 0 + assert _held_data_blob(test_activities, workflow_input) is None + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_3_1_invalid_aggregation_function( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + postgres_engine, +): + scenario = load_scenario_input('core_scouter_invalid_aggregation') + workflow_input = scenario['workflow_input'] + + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-bad-aggr'), + ) + + variables = {row['variable'] for row in fetch_laborious_rows(postgres_engine, workflow_input['model_id'])} + assert 'tag_bad' not in variables + assert 'tag_ok' in variables + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_scenario_3_3_2_postgres_export_failure_surfaces( + temporal_env: WorkflowEnvironment, + temporal_worker: Worker, + mongo_uri: str, + postgres_engine, +): + scenario = load_scenario_input('core_scouter_postgres_export_failure') + workflow_input = scenario['workflow_input'] + + with postgres_engine.begin() as conn: + conn.execute(text('ALTER TABLE sientia_data.laborious_data DROP COLUMN value')) + + with pytest.raises(WorkflowFailureError): + await start_and_await_workflow( + temporal_env.client, + CoreScouter.run, + workflow_input, + make_workflow_id('core-pg-fail'), + ) + + assert ( + count_notifications( + mongo_uri, + E2E_DATABASE, + notification_id='ERROR_EXPORTING_DATA_TO_POSTGRES', + ) + >= 1 + ) diff --git a/pyproject.toml b/pyproject.toml index f77f9f1..cc111b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,9 +116,11 @@ addopts = [ "--strict-markers", ] asyncio_mode = "auto" -asyncio_default_fixture_loop_scope = "function" +asyncio_default_fixture_loop_scope = "session" +asyncio_default_test_loop_scope = "session" markers = [ "asyncio: marks tests as async", + "e2e: end-to-end tests against real backing services (Docker required)", "integration: marks tests as integration tests", "unit: marks tests as unit tests", ] diff --git a/requirements-dev.txt b/requirements-dev.txt index 352add1..3539fa9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,10 +15,9 @@ pytest-cov>=4.1.0 # Coverage plugin for pytest pytest-asyncio>=0.21.0 # Async test support (already in main requirements) # E2E Testing Dependencies -fakeredis>=2.20.0 # In-memory Redis server for testing -mongomock>=4.1.2 # In-memory MongoDB for testing +pytest-httpserver>=1.0.10 # Development Tools ipython>=8.12.0 # Enhanced Python shell ipdb>=0.13.13 # IPython debugger -testcontainers[postgres] \ No newline at end of file +testcontainers[postgres,mongodb,redis]>=4.0 \ No newline at end of file diff --git a/requirements-local.txt b/requirements-local.txt new file mode 100644 index 0000000..d1adfd1 --- /dev/null +++ b/requirements-local.txt @@ -0,0 +1,8 @@ +temporalio +psycopg2-binary +sqlalchemy +redis +pymongo +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.1 +prometheus-client +pycurl \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a02d39c..6470c2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ psycopg2-binary sqlalchemy redis pymongo -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.8.2 +sientia_do prometheus-client pycurl \ No newline at end of file diff --git a/scouter/activities/activities.py b/scouter/activities/activities.py index 0cc3f50..cb638a0 100644 --- a/scouter/activities/activities.py +++ b/scouter/activities/activities.py @@ -7,7 +7,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import Logger - from sientia_do.temporal.activities.postgres import Postgres + from sientia_do.temporal.activities.postgres_sync import Postgres from scouter.activities.api import API from scouter.activities.gates import Gates diff --git a/scouter/activities/api.py b/scouter/activities/api.py index 3321959..161d3a6 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -9,7 +9,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.logger import Logger from sientia_do.observability.metrics_controller import MetricsController from sientia_do.observability.sientia_monitoring import SientiaMonitoring - from sientia_do.repository.pi_web_api_client import PIWebAPIClient + from sientia_do.repository.pi_web_api_client_sync import PIWebAPIClient from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ @@ -78,7 +78,7 @@ class API(SientiaMonitoring): SientiaMonitoring.shutdown(self) @activity.defn(name='get_tag_values') - async def get_tag_values(self, input_data: dict[str, Any]) -> list[dict]: + def get_tag_values(self, input_data: dict[str, Any]) -> list[dict]: """ Retrieve tag values from PI Web API for specified WebIds. @@ -123,7 +123,7 @@ class API(SientiaMonitoring): self.info(f'Getting tag values from {endpoint}', metadata=metadata) self.debug(f'Web IDs: {web_ids}', metadata=metadata) try: - latest_values = await self.pi_web_api_client.get_latest_values_df( + latest_values = self.pi_web_api_client.get_latest_values_df( endpoint=endpoint, web_ids=web_ids, start_time=period, @@ -134,7 +134,7 @@ class API(SientiaMonitoring): ) except Exception as e: - await self.send_notification_async( + self.send_notification( metadata=metadata, notification_id='PI_WEB_API_REQUEST_ERROR', message=f'Error getting tag values from PI Web API: {e}', diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py index 0ad2cf7..9040113 100644 --- a/scouter/activities/gates.py +++ b/scouter/activities/gates.py @@ -59,7 +59,7 @@ class Gates(SientiaMonitoring): """ SientiaMonitoring.shutdown(self) - async def apply_aggregation( + def apply_aggregation( self, values: DataFrame, aggr_function: str, metadata: dict[str, Any] ) -> float | None | str: """ @@ -82,31 +82,16 @@ class Gates(SientiaMonitoring): Raises: NotificationError: If invalid aggregation function is specified """ - # Fast path for single value - if len(values) == 1: - return values['value'].iloc[0] - - if aggr_function == 'lts': - return values['value'].iloc[-1] - - # Remove NaN values without inplace operation - clean_values = values['value'].dropna() - - if clean_values.empty: - return None - - # Use dictionary lookup for aggregation functions (faster than if-elif chain) aggregation_map = { + 'lts': lambda x: x.iloc[-1], 'avg': lambda x: x.mean(), 'mdn': lambda x: x.median(), 'max': lambda x: x.max(), 'min': lambda x: x.min(), } - if aggr_function in aggregation_map: - return aggregation_map[aggr_function](clean_values) - else: - await self.send_notification_async( + if aggr_function not in aggregation_map: + self.send_notification( metadata=metadata, notification_id='AGGREGATION_ISSUES', message=f'Invalid aggregation function: {aggr_function}', @@ -116,8 +101,21 @@ class Gates(SientiaMonitoring): ) return 'continue' + if len(values) == 1: + return values['value'].iloc[0] + + if aggr_function == 'lts': + return aggregation_map['lts'](values['value']) + + clean_values = values['value'].dropna() + + if clean_values.empty: + return None + + return aggregation_map[aggr_function](clean_values) + @activity.defn(name='aggregate_data') - async def aggregate_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]: + def aggregate_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]: """ Aggregate time-series data by tag and name using specified functions. @@ -165,7 +163,7 @@ class Gates(SientiaMonitoring): # Get the latest timestamp (last row since data is sorted) latest_timestamp = group['timestamp'].iloc[-1] - aggr_value = await self.apply_aggregation(group, aggr_function, metadata) + aggr_value = self.apply_aggregation(group, aggr_function, metadata) if aggr_value == 'continue': continue @@ -197,7 +195,7 @@ class Gates(SientiaMonitoring): except Exception as e: trace = traceback.format_exc() - await self.send_notification_async( + self.send_notification( metadata=metadata, notification_id='AGGREGATION_ISSUES', message=f'Error aggregating data: {e}', @@ -210,7 +208,7 @@ class Gates(SientiaMonitoring): raise e @activity.defn(name='data_quality_gate') - async def data_quality_gate(self, input_data: dict[str, Any]) -> dict[Hashable, Any]: + def data_quality_gate(self, input_data: dict[str, Any]) -> dict[Hashable, Any]: """ Apply data quality filters to incoming data. @@ -255,7 +253,7 @@ class Gates(SientiaMonitoring): except Exception as e: trace = traceback.format_exc() - await self.send_notification_async( + self.send_notification( metadata=metadata, notification_id='DATA_QUALITY_GATE_ISSUES', message=f'Error applying filter {filter_name}: {e}', @@ -273,7 +271,7 @@ class Gates(SientiaMonitoring): message = f'{len(filtered_data)} rows has quality issues: {filter_name}: {policy}' attachment = filtered_data.to_string() - await self.send_notification_async( + self.send_notification( metadata=metadata, notification_id=f'DATA_QUALITY_GATE_ISSUES__{filter_name}', message=message, @@ -290,7 +288,7 @@ class Gates(SientiaMonitoring): return data.to_dict() @activity.defn(name='write_metrics') - async def write_metrics(self, input_data: dict[str, Any]) -> None: + def write_metrics(self, input_data: dict[str, Any]) -> None: """ Write metrics to the database. input_data: diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index 55d269e..28f78f6 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -12,7 +12,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger from sientia_do.observability.sientia_monitoring import SientiaMonitoring - from sientia_do.repository.mongodb_repository import MongoDBRepository + from sientia_do.repository.mongodb_repository_sync import MongoDBRepository from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ @@ -84,7 +84,7 @@ class MongoDB(SientiaMonitoring): self.close() @activity.defn(name='load_latest_data') - async def load_latest_data(self, input_data: dict[str, Any]) -> list[dict[str, Any]]: + def load_latest_data(self, input_data: dict[str, Any]) -> list[dict[str, Any]]: """ Load the latest data from MongoDB collection since a specified timestamp. @@ -123,7 +123,7 @@ class MongoDB(SientiaMonitoring): self.debug(f'Data filter: {data_filter}', metadata=metadata) - data = await self.mongodb_repository.find( + data = self.mongodb_repository.find( collection_name=collection_name, filters=data_filter, metadata=metadata, @@ -143,7 +143,7 @@ class MongoDB(SientiaMonitoring): return data except Exception as e: trace = traceback.format_exc() - await self.send_notification_async( + self.send_notification( metadata=metadata, notification_id='MONGO_LOAD_ERROR', message=f'Error loading data from MongoDB: {e}', diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 57083af..bd52de7 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -11,7 +11,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger from sientia_do.observability.sientia_monitoring import SientiaMonitoring - from sientia_do.repository.redis_repository import RedisRepository + from sientia_do.repository.redis_repository_sync import RedisRepository from sientia_do.temporal.constants import DATETIME_FORMAT, now @@ -70,7 +70,7 @@ class Redis(SientiaMonitoring): SientiaMonitoring.shutdown(self) @activity.defn(name='get_last_data_timestamp') - async def get_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None: + def get_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None: """ Retrieve the last processed data timestamp from Redis. @@ -97,9 +97,9 @@ class Redis(SientiaMonitoring): self.info(f'Getting last data timestamp for {key}', metadata=metadata) try: - data_hold = await self.redis_repository.get(key, metadata=metadata) + data_hold = self.redis_repository.get(key, metadata=metadata) except Exception as e: - await self.send_notification_async( + self.send_notification( metadata=metadata, notification_id='REDIS_GET_ERROR', message=f'Error getting last data timestamp: {e}', @@ -117,7 +117,7 @@ class Redis(SientiaMonitoring): return data_hold @activity.defn(name='put_last_data_timestamp') - async def put_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None: + def put_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None: """ Store the last processed data timestamp in Redis. @@ -155,11 +155,9 @@ class Redis(SientiaMonitoring): self.info(f'Last collected timestamp to insert: {last_data_timestamp}', metadata=metadata) try: - await self.redis_repository.set( - key, last_data_timestamp, ttl=60 * 60 * 5, metadata=metadata - ) + self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5, metadata=metadata) except Exception as e: - await self.send_notification_async( + self.send_notification( metadata=metadata, notification_id='REDIS_SET_ERROR', message=f'Error setting last data timestamp: {e}', @@ -172,7 +170,7 @@ class Redis(SientiaMonitoring): return last_data_timestamp @activity.defn(name='group_and_hold_data') - async def group_and_hold_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]: + def group_and_hold_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]: """ Group data by tags and store temporarily in Redis with TTL. @@ -210,9 +208,9 @@ class Redis(SientiaMonitoring): self.info(f'Getting held data for {key}', metadata=metadata) try: - data_hold = await self.redis_repository.get(key, metadata=metadata) + data_hold = self.redis_repository.get(key, metadata=metadata) except Exception as e: - await self.send_notification_async( + self.send_notification( metadata=metadata, notification_id='REDIS_GET_ERROR', message=f'Error getting held data: {e}', @@ -254,7 +252,7 @@ class Redis(SientiaMonitoring): data['timestamp'].max() if not data.empty else data_hold['timestamp'] ) - await self.redis_repository.set(key, data_hold, ttl=retention_time, metadata=metadata) + self.redis_repository.set(key, data_hold, ttl=retention_time, metadata=metadata) data_hold_df = DataFrame(data_hold, index=[0]) data_hold_melted = data_hold_df.melt( @@ -264,7 +262,7 @@ class Redis(SientiaMonitoring): data_hold_melted.reset_index(drop=True, inplace=True) except Exception as e: - await self.send_notification_async( + self.send_notification( metadata=metadata, notification_id='REDIS_SET_ERROR', message=f'Error setting held data: {e}', @@ -281,7 +279,7 @@ class Redis(SientiaMonitoring): return data_hold_melted.to_dict() @activity.defn(name='store_data_package') - async def store_data_package(self, input_data: dict[str, Any]): + def store_data_package(self, input_data: dict[str, Any]): """ Stores the data package in redis. It's a debug feature and must be toggled on. input_data: @@ -300,9 +298,9 @@ class Redis(SientiaMonitoring): cache = {'data': data.to_dict(), 'held_data': held_data.to_dict()} try: - await self.redis_repository.set(key, cache, ttl=120, metadata=metadata) + self.redis_repository.set(key, cache, ttl=120, metadata=metadata) except Exception as e: - await self.send_notification_async( + self.send_notification( metadata=metadata, notification_id='REDIS_SET_ERROR', message=f'Error setting data package: {e}', diff --git a/scouter/worker/prepare_worker.py b/scouter/worker/prepare_worker.py deleted file mode 100644 index b497c5d..0000000 --- a/scouter/worker/prepare_worker.py +++ /dev/null @@ -1,74 +0,0 @@ -import os -import re -from collections.abc import Sequence -from typing import Any - -from sientia_do.observability.logger import Logger -from temporalio.client import Client -from temporalio.worker import PollerBehaviorAutoscaling, Worker - -# Worker configuration parameters with default values -# See worker_parameters.md for detailed documentation -parameters = [ - ('MAX_CONCURRENT_WORKFLOW_TASKS', '200'), - ('MAX_CONCURRENT_ACTIVITIES', '200'), - ('MAX_CONCURRENT_LOCAL_ACTIVITIES', '200'), - ('MAX_CACHED_WORKFLOWS', '200'), - ('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10'), - ('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100'), - ('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200'), - ('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10'), - ('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100'), - ('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200'), -] - - -def camel_to_snake(text: str) -> str: - """Convert camelCase or PascalCase to snake_case.""" - text = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', text) - text = re.sub('([a-z0-9])([A-Z])', r'\1_\2', text) - return text.lower() - - -def prepare_worker( - main_workflow: type, - other_workflows: Sequence[type], - activities: Sequence[Any], - temporal_client: Client, - logger: Logger, -) -> Worker: - main_workflow_name = main_workflow.__name__.upper() - - queue_name = f'{camel_to_snake(main_workflow.__name__)}-queue' - - local_workflow_parameters = {} - - for parameter in parameters: - local_workflow_parameters[parameter[0]] = int( - os.getenv(main_workflow_name + '_' + parameter[0], parameter[1]) - ) - - logger.info(f'Preparing worker for {main_workflow_name} with queue {queue_name}') - - return Worker( - temporal_client, - task_queue=queue_name, - workflows=[main_workflow, *other_workflows], - activities=[*activities], - max_concurrent_workflow_tasks=local_workflow_parameters['MAX_CONCURRENT_WORKFLOW_TASKS'], - max_concurrent_activities=local_workflow_parameters['MAX_CONCURRENT_ACTIVITIES'], - max_concurrent_local_activities=local_workflow_parameters[ - 'MAX_CONCURRENT_LOCAL_ACTIVITIES' - ], - max_cached_workflows=local_workflow_parameters['MAX_CACHED_WORKFLOWS'], - workflow_task_poller_behavior=PollerBehaviorAutoscaling( - minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MINIMUM'], - initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_INITIAL'], - maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM'], - ), - activity_task_poller_behavior=PollerBehaviorAutoscaling( - minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MINIMUM'], - initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_INITIAL'], - maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM'], - ), - ) diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 379e627..8ef61a3 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -9,6 +9,7 @@ with workflow.unsafe.imports_passed_through(): from prometheus_client import start_http_server from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import get_logger + from sientia_do.temporal.worker.prepare_worker import prepare_worker from sientia_do.utils.connectors_config import ( build_api_config, build_mongodb_config, @@ -18,7 +19,6 @@ with workflow.unsafe.imports_passed_through(): from scouter import metrics from scouter.activities.activities import Activities - from scouter.worker.prepare_worker import prepare_worker from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter from scouter.workflow.scouter import Scouter from scouter.workflow.sub_workflows.core_scouter import CoreScouter @@ -28,26 +28,6 @@ POD_ID = os.getenv('HOSTNAME', 'localhost') SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091')) -# For optmized latency, Temporal docs recommends fixed slots, ensuring -# high concurency levels. - -MAX_CONCURRENT_WORKFLOW_TASKS = int(os.getenv('MAX_CONCURRENT_WORKFLOW_TASKS', '200')) -MAX_CONCURRENT_ACTIVITIES = int(os.getenv('MAX_CONCURRENT_ACTIVITIES', '200')) -MAX_CONCURRENT_LOCAL_ACTIVITIES = int(os.getenv('MAX_CONCURRENT_LOCAL_ACTIVITIES', '200')) -MAX_CACHED_WORKFLOWS = int(os.getenv('MAX_CACHED_WORKFLOWS', '200')) - - -# Temporal docs also recommends an autoscaling policy, with agrresive limits to prioritize latency over throughput. - -WORKFLOW_POLLER_BEHAVIOUR_MINIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10')) -WORKFLOW_POLLER_BEHAVIOUR_INITIAL = int(os.getenv('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100')) -WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200')) - -ACTIVITY_POLLER_BEHAVIOUR_MINIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10')) -ACTIVITY_POLLER_BEHAVIOUR_INITIAL = int(os.getenv('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100')) -ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200')) - - async def main(): """ Main entry point for the Scouter Temporal worker. diff --git a/scouter/worker/worker_parameters.md b/scouter/worker/worker_parameters.md deleted file mode 100644 index 8b18148..0000000 --- a/scouter/worker/worker_parameters.md +++ /dev/null @@ -1,113 +0,0 @@ -# Worker Parameters Documentation - -This document explains each configuration parameter used in the `prepare_worker.py` file for configuring Temporal workers. - -## Overview - -All parameters can be configured via environment variables using the pattern: `{WORKFLOW_NAME}_{PARAMETER_NAME}`. If not set, default values are used as specified below. - -## Concurrency Parameters - -### MAX_CONCURRENT_WORKFLOW_TASKS -- **Default**: `200` -- **Description**: Maximum number of concurrent workflow tasks that can be processed simultaneously by the worker. This controls how many workflow executions can be actively running at the same time. -- **Usage**: Set via `max_concurrent_workflow_tasks` in the Worker configuration. -- **Impact**: Higher values allow more workflows to run concurrently but consume more resources. Lower values provide better resource control but may limit throughput. - -### MAX_CONCURRENT_ACTIVITIES -- **Default**: `200` -- **Description**: Maximum number of concurrent activity tasks that can be executed simultaneously by the worker. Activities are the actual work units that perform business logic. -- **Usage**: Set via `max_concurrent_activities` in the Worker configuration. -- **Impact**: Controls the parallelism of activity execution. Higher values increase throughput but require more system resources (CPU, memory, network connections). - -### MAX_CONCURRENT_LOCAL_ACTIVITIES -- **Default**: `200` -- **Description**: Maximum number of concurrent local activity tasks that can be executed simultaneously. Local activities run in the same process as the workflow, without requiring a separate activity worker. -- **Usage**: Set via `max_concurrent_local_activities` in the Worker configuration. -- **Impact**: Similar to regular activities, but local activities have lower latency and overhead since they don't require network round-trips. Useful for lightweight operations. - -## Caching Parameters - -### MAX_CACHED_WORKFLOWS -- **Default**: `200` -- **Description**: Maximum number of workflow instances that can be cached in memory by the worker. Cached workflows allow faster resumption of execution without reloading state. -- **Usage**: Set via `max_cached_workflows` in the Worker configuration. -- **Impact**: Higher values improve performance for frequently accessed workflows but consume more memory. Lower values reduce memory usage but may require more frequent state reloads. - -## Understanding Pollers in Temporal - -**Pollers** are components of Temporal Workers that continuously request tasks from the Temporal service's Task Queues via synchronous RPCs. There are separate pollers for workflow tasks and activity tasks. - -### How Pollers Work - -Pollers send requests to the Temporal service to retrieve tasks from Task Queues. When a task is available, the poller retrieves it and the Worker processes it using registered Workflow or Activity handlers. This architecture provides: -- **Load Balancing**: Workers only poll when they have capacity, distributing load across multiple processes -- **Fault Tolerance**: Tasks persist in queues if a Worker fails, allowing recovery -- **Task Routing**: Tasks can be routed to specific Worker processes - -### Autoscaling Poller Behavior - -Temporal supports autoscaling that dynamically adjusts the number of concurrent pollers based on workload. The system scales up during high load and down during low load, maintaining a baseline for responsiveness. Autoscaling is configured with `minimum`, `initial`, and `maximum` parameters that define the scaling bounds. - -## Workflow Poller Behavior (Autoscaling) - -These parameters control the autoscaling behavior of the workflow task poller, which retrieves workflow tasks from the Temporal server. - -### WORKFLOW_POLLER_BEHAVIOUR_MINIMUM -- **Default**: `10` -- **Description**: Minimum number of concurrent pollers for workflow tasks. The poller count will never go below this value. -- **Usage**: Set via `minimum` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`. -- **Impact**: Ensures a baseline level of polling activity even during low load periods. - -### WORKFLOW_POLLER_BEHAVIOUR_INITIAL -- **Default**: `100` -- **Description**: Initial number of concurrent pollers for workflow tasks when the worker starts. -- **Usage**: Set via `initial` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`. -- **Impact**: Determines the starting point for poller scaling. Higher values provide faster initial task acquisition but consume more resources. - -### WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM -- **Default**: `200` -- **Description**: Maximum number of concurrent pollers allowed for workflow tasks. The poller count will not exceed this value even under high load. -- **Usage**: Set via `maximum` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`. -- **Impact**: Caps the resource consumption for workflow task polling. Prevents excessive polling that could overwhelm the Temporal server or worker. - -## Activity Poller Behavior (Autoscaling) - -These parameters control the autoscaling behavior of the activity task poller, which retrieves activity tasks from the Temporal server. - -### ACTIVITY_POLLER_BEHAVIOUR_MINIMUM -- **Default**: `10` -- **Description**: Minimum number of concurrent pollers for activity tasks. The poller count will never go below this value. -- **Usage**: Set via `minimum` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`. -- **Impact**: Ensures a baseline level of polling activity even during low load periods. - -### ACTIVITY_POLLER_BEHAVIOUR_INITIAL -- **Default**: `100` -- **Description**: Initial number of concurrent pollers for activity tasks when the worker starts. -- **Usage**: Set via `initial` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`. -- **Impact**: Determines the starting point for poller scaling. Higher values provide faster initial task acquisition but consume more resources. - -### ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM -- **Default**: `200` -- **Description**: Maximum number of concurrent pollers allowed for activity tasks. The poller count will not exceed this value even under high load. -- **Usage**: Set via `maximum` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`. -- **Impact**: Caps the resource consumption for activity task polling. Prevents excessive polling that could overwhelm the Temporal server or worker. - -## Configuration Example - -To override these parameters, set environment variables using the pattern: -``` -{WORKFLOW_NAME}_{PARAMETER_NAME}={value} -``` - -For example, if your workflow is named `ScouterWorkflow`: -```bash -SCOUTERWORKFLOW_MAX_CONCURRENT_ACTIVITIES=500 -SCOUTERWORKFLOW_WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM=300 -``` - -## Notes - -- All parameter values are converted to integers before use. -- The autoscaling poller behavior dynamically adjusts the number of pollers between the minimum and maximum values based on workload. -- These parameters should be tuned based on your specific workload characteristics, available resources, and performance requirements. diff --git a/sonar-project.properties b/sonar-project.properties index f98e3ab..ca79a27 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -5,7 +5,7 @@ sonar.tests=tests sonar.qualitygate.wait=true sonar.qualitygate.timeout=300 sonar.python.coverage.reportPaths=coverage.xml -sonar.coverage.exclusions=scouter/worker/worker.py,scouter/worker/prepare_worker.py +sonar.coverage.exclusions=scouter/worker/worker.py sonar.python.xunit.reportPath=pytest.xml sonar.python.version=3.11 sonar.projectVersion=1.0.0 diff --git a/tests.ipynb b/tests.ipynb index 8dd70f6..24e9fb3 100644 --- a/tests.ipynb +++ b/tests.ipynb @@ -282,7 +282,7 @@ " j = i - pace\n", " print(f'Getting data for chunk -{i} to -{j} days')\n", " try:\n", - " chunk = DataFrame(await api.get_tag_values(\n", + " chunk = DataFrame(api.get_tag_values(\n", " input_data={\n", " 'endpoint': '/streamsets/recorded',\n", " 'web_ids': web_ids,\n", @@ -294,7 +294,7 @@ " }\n", " ))\n", " except Exception as e:\n", - " chunk = DataFrame(await api.get_tag_values(\n", + " chunk = DataFrame(api.get_tag_values(\n", " input_data={\n", " 'endpoint': '/streamsets/recorded',\n", " 'web_ids': web_ids,\n", diff --git a/tests/activities/test_activities.py b/tests/activities/test_activities.py index 2d3f5ab..087655b 100644 --- a/tests/activities/test_activities.py +++ b/tests/activities/test_activities.py @@ -1,6 +1,7 @@ +import inspect from unittest.mock import ANY, MagicMock, patch -from sientia_do.temporal.activities.postgres import Postgres +from sientia_do.temporal.activities.postgres_sync import Postgres from scouter.activities.activities import Activities from scouter.activities.api import API @@ -181,3 +182,13 @@ def test_shutdown( mock_redis_close.assert_called() mock_gates_close.assert_called() mock_api_close.assert_called() + + +def test_activity_methods_are_sync(): + """Every @activity.defn method on Activities must be a synchronous def.""" + for cls in Activities.__mro__: + for name, member in vars(cls).items(): + if getattr(member, '__temporal_activity_definition', None) is not None: + assert not inspect.iscoroutinefunction(member), ( + f'{cls.__name__}.{name} must not be async' + ) diff --git a/tests/activities/test_api.py b/tests/activities/test_api.py index 1386ee7..3c12209 100644 --- a/tests/activities/test_api.py +++ b/tests/activities/test_api.py @@ -1,4 +1,4 @@ -from unittest.mock import ANY, AsyncMock, MagicMock, patch +from unittest.mock import ANY, MagicMock, Mock, patch import pandas as pd import pytest @@ -86,8 +86,7 @@ def test_close(mock_sientia_monitoring, api_activity): mock_sientia_monitoring.shutdown.assert_called_once() -@pytest.mark.asyncio -async def test_get_tag_values_success(api_activity): +def test_get_tag_values_success(api_activity): """Test get_tag_values with successful data retrieval.""" # Setup test data test_data = { @@ -131,10 +130,10 @@ async def test_get_tag_values_success(api_activity): mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp']) - api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df) + api_activity.pi_web_api_client.get_latest_values_df = Mock(return_value=mock_df) # Execute - result = await api_activity.get_tag_values(test_data) + result = api_activity.get_tag_values(test_data) # Verify api_activity.pi_web_api_client.get_latest_values_df.assert_called_once_with( @@ -161,8 +160,7 @@ async def test_get_tag_values_success(api_activity): assert result[2]['timestamp'] == '2023-01-01 12:02:00+0000' -@pytest.mark.asyncio -async def test_get_tag_values_with_default_max_count(api_activity): +def test_get_tag_values_with_default_max_count(api_activity): """Test get_tag_values with default max_count value.""" # Setup test data without max_count test_data = { @@ -190,10 +188,10 @@ async def test_get_tag_values_with_default_max_count(api_activity): mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp']) - api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df) + api_activity.pi_web_api_client.get_latest_values_df = Mock(return_value=mock_df) # Execute - result = await api_activity.get_tag_values(test_data) + result = api_activity.get_tag_values(test_data) # Verify default max_count is 1 api_activity.pi_web_api_client.get_latest_values_df.assert_called_once_with( @@ -209,8 +207,7 @@ async def test_get_tag_values_with_default_max_count(api_activity): assert len(result) == 1 -@pytest.mark.asyncio -async def test_get_tag_values_with_none_webids(api_activity): +def test_get_tag_values_with_none_webids(api_activity): """Test get_tag_values with some None WebIds.""" # Setup test data with None values test_data = { @@ -245,18 +242,17 @@ async def test_get_tag_values_with_none_webids(api_activity): mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp']) - api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df) + api_activity.pi_web_api_client.get_latest_values_df = Mock(return_value=mock_df) # Execute - result = await api_activity.get_tag_values(test_data) + result = api_activity.get_tag_values(test_data) # Verify - should only query non-None WebIds assert len(result) == 2 assert all(r['name'] in ['tag1', 'tag3'] for r in result) -@pytest.mark.asyncio -async def test_get_tag_values_api_error(api_activity): +def test_get_tag_values_api_error(api_activity): """Test get_tag_values when PI Web API client raises an error and sends notification.""" # Setup test data test_data = { @@ -275,19 +271,19 @@ async def test_get_tag_values_api_error(api_activity): } # Mock API error - api_activity.pi_web_api_client.get_latest_values_df = AsyncMock( + api_activity.pi_web_api_client.get_latest_values_df = Mock( side_effect=Exception('PI Web API connection error') ) - api_activity.send_notification_async = AsyncMock() + api_activity.send_notification = MagicMock() # Execute and verify exception is raised with pytest.raises(Exception) as exc_info: - await api_activity.get_tag_values(test_data) + api_activity.get_tag_values(test_data) assert str(exc_info.value) == 'PI Web API connection error' # Verify notification was sent - api_activity.send_notification_async.assert_called_once_with( + api_activity.send_notification.assert_called_once_with( metadata=metadata['metadata'], notification_id='PI_WEB_API_REQUEST_ERROR', message='Error getting tag values from PI Web API: PI Web API connection error', @@ -297,8 +293,7 @@ async def test_get_tag_values_api_error(api_activity): ) -@pytest.mark.asyncio -async def test_get_tag_values_with_nan_values(api_activity): +def test_get_tag_values_with_nan_values(api_activity): """Test get_tag_values handling NaN values in the DataFrame.""" # Setup test data test_data = { @@ -333,10 +328,10 @@ async def test_get_tag_values_with_nan_values(api_activity): mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp']) - api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df) + api_activity.pi_web_api_client.get_latest_values_df = Mock(return_value=mock_df) # Execute - result = await api_activity.get_tag_values(test_data) + result = api_activity.get_tag_values(test_data) # Verify assert len(result) == 2 diff --git a/tests/activities/test_gates.py b/tests/activities/test_gates.py index 95d5d24..84e513b 100644 --- a/tests/activities/test_gates.py +++ b/tests/activities/test_gates.py @@ -1,5 +1,5 @@ from typing import Any -from unittest.mock import ANY, AsyncMock, MagicMock, Mock, call, patch +from unittest.mock import ANY, MagicMock, Mock, call, patch import numpy as np import pandas as pd @@ -21,8 +21,6 @@ def gates_fixture(): metrics_controller=metrics_controller, ) gates.send_notification = MagicMock() - gates.send_notification_async = AsyncMock() - gates.emit_metric = AsyncMock() gates.logger = logger gates.notification_handler = notification_handler @@ -48,8 +46,7 @@ def test_close(mock_sientia_monitoring, gates_fixture): mock_sientia_monitoring.shutdown.assert_called_once() -@pytest.mark.asyncio -async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture): +def test_data_quality_gate_with_null_values_filter_discard(gates_fixture): """Test data_quality_gate with NULL_VALUES_FILTER and DISCARD policy.""" # Setup test data input_data = { @@ -69,16 +66,15 @@ async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture): } # Execute - result = await gates_fixture.data_quality_gate(input_data) + result = gates_fixture.data_quality_gate(input_data) # Verify assert len(result['tag']) == 2 assert 'tag2' not in result['tag'] - gates_fixture.send_notification_async.assert_called_once() + gates_fixture.send_notification.assert_called_once() -@pytest.mark.asyncio -async def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture): +def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture): """Test data_quality_gate with OUT_OF_BOUNDS_FILTER and KEEP policy.""" # Setup test data with out of bounds values input_data = { @@ -103,15 +99,14 @@ async def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture): {'OUT_OF_BOUNDS_FILTER': lambda df: df[df['tag'] == 'tag2']}, ): # Execute - result = await gates_fixture.data_quality_gate(input_data) + result = gates_fixture.data_quality_gate(input_data) # Verify data is kept but notification is sent assert len(result['tag']) == 3 # All rows kept - gates_fixture.send_notification_async.assert_called_once() + gates_fixture.send_notification.assert_called_once() -@pytest.mark.asyncio -async def test_data_quality_gate_with_multiple_filters(gates_fixture): +def test_data_quality_gate_with_multiple_filters(gates_fixture): """Test data_quality_gate with multiple filters.""" # Setup test data input_data = { @@ -134,7 +129,7 @@ async def test_data_quality_gate_with_multiple_filters(gates_fixture): **metadata, } - result = await gates_fixture.data_quality_gate(input_data) + result = gates_fixture.data_quality_gate(input_data) # Verify only tag1 and tag4 remain (tag2 has null, tag3 is out of bounds) assert result == { @@ -144,11 +139,10 @@ async def test_data_quality_gate_with_multiple_filters(gates_fixture): 'timestamp': {0: '2023-01-01', 3: '2023-01-04'}, } # Should be called twice (once for each filter) - assert gates_fixture.send_notification_async.call_count == 2 + assert gates_fixture.send_notification.call_count == 2 -@pytest.mark.asyncio -async def test_data_quality_gate_with_unknown_filter(gates_fixture): +def test_data_quality_gate_with_unknown_filter(gates_fixture): """Test data_quality_gate with an unknown filter.""" # Setup test data with unknown filter gates_fixture.warning = MagicMock() @@ -160,7 +154,7 @@ async def test_data_quality_gate_with_unknown_filter(gates_fixture): } # Execute - result = await gates_fixture.data_quality_gate(input_data) + result = gates_fixture.data_quality_gate(input_data) # Verify data is unchanged and warning is logged assert len(result['tag']) == 1 @@ -169,8 +163,7 @@ async def test_data_quality_gate_with_unknown_filter(gates_fixture): ) -@pytest.mark.asyncio -async def test_data_quality_gate_with_filter_error(gates_fixture): +def test_data_quality_gate_with_filter_error(gates_fixture): """Test data_quality_gate when a filter raises an exception.""" # Setup test data input_data = { @@ -188,19 +181,18 @@ async def test_data_quality_gate_with_filter_error(gates_fixture): 'scouter.activities.gates.quality_gate_filters', {'NULL_VALUES_FILTER': failing_filter} ): # Execute - result = await gates_fixture.data_quality_gate(input_data) + result = gates_fixture.data_quality_gate(input_data) # Verify error notification is sent and data is unchanged assert len(result['tag']) == 1 - gates_fixture.send_notification_async.assert_called_once() - call_args = gates_fixture.send_notification_async.call_args[1] + gates_fixture.send_notification.assert_called_once() + call_args = gates_fixture.send_notification.call_args[1] assert call_args['notification_id'] == 'DATA_QUALITY_GATE_ISSUES' assert call_args['level'] == NotificationLevel.ERROR assert 'Filter error' in call_args['message'] -@pytest.mark.asyncio -async def test_data_quality_gate_with_empty_data(gates_fixture): +def test_data_quality_gate_with_empty_data(gates_fixture): """Test data_quality_gate with empty input data.""" # Setup empty input data input_data = { @@ -211,15 +203,14 @@ async def test_data_quality_gate_with_empty_data(gates_fixture): } # Execute - result = await gates_fixture.data_quality_gate(input_data) + result = gates_fixture.data_quality_gate(input_data) # Verify empty result and no notifications assert len(result['tag']) == 0 gates_fixture.send_notification.assert_not_called() -@pytest.mark.asyncio -async def test_data_quality_gate_with_no_filters(gates_fixture): +def test_data_quality_gate_with_no_filters(gates_fixture): """Test data_quality_gate with no filters specified.""" # Setup test data with no filters input_data = { @@ -230,7 +221,7 @@ async def test_data_quality_gate_with_no_filters(gates_fixture): } # Execute - result = await gates_fixture.data_quality_gate(input_data) + result = gates_fixture.data_quality_gate(input_data) # Verify data is unchanged and no notifications assert len(result['tag']) == 1 @@ -254,23 +245,22 @@ async def test_data_quality_gate_with_no_filters(gates_fixture): (pd.DataFrame({'value': [np.nan, np.nan]}), 'avg', None), # Invalid aggregation function (pd.DataFrame({'value': [1.0, 2.0]}), 'invalid', 'continue'), + (pd.DataFrame({'value': [10.0]}), 'invalid', 'continue'), ], ) -@pytest.mark.asyncio -async def test_apply_aggregation(gates_fixture, group_data, aggr_function, expected_result): +def test_apply_aggregation(gates_fixture, group_data, aggr_function, expected_result): """Test apply_aggregation method with various scenarios.""" - result = await gates_fixture.apply_aggregation(group_data, aggr_function, metadata) + result = gates_fixture.apply_aggregation(group_data, aggr_function, metadata) assert result == expected_result # Check notification was sent for invalid function if aggr_function == 'invalid': - gates_fixture.send_notification_async.assert_called_once() + gates_fixture.send_notification.assert_called_once() else: - gates_fixture.send_notification_async.assert_not_called() + gates_fixture.send_notification.assert_not_called() -@pytest.mark.asyncio -async def test_aggregate_data(gates_fixture): +def test_aggregate_data(gates_fixture): """Test aggregate_data method with multiple groups and aggregation functions.""" input_data = { 'data': [ @@ -299,16 +289,15 @@ async def test_aggregate_data(gates_fixture): } # Execute - result = await gates_fixture.aggregate_data(input_data) + result = gates_fixture.aggregate_data(input_data) # Verify assert result == expected_result gates_fixture.send_notification.assert_not_called() -@pytest.mark.asyncio -async def test_aggregate_data_with_continue(gates_fixture): - gates_fixture.apply_aggregation = AsyncMock(return_value='continue') +def test_aggregate_data_with_continue(gates_fixture): + gates_fixture.apply_aggregation = MagicMock(return_value='continue') input_data = { 'data': [ @@ -331,15 +320,14 @@ async def test_aggregate_data_with_continue(gates_fixture): expected_result: dict[str, Any] = {} # Execute - result = await gates_fixture.aggregate_data(input_data) + result = gates_fixture.aggregate_data(input_data) # Verify assert result == expected_result - gates_fixture.send_notification_async.assert_not_called() + gates_fixture.send_notification.assert_not_called() -@pytest.mark.asyncio -async def test_aggregate_data_raise_exception(gates_fixture): +def test_aggregate_data_raise_exception(gates_fixture): gates_fixture.apply_aggregation = MagicMock(side_effect=Exception('Test exception')) input_data = { @@ -360,10 +348,10 @@ async def test_aggregate_data_raise_exception(gates_fixture): } try: - await gates_fixture.aggregate_data(input_data) + gates_fixture.aggregate_data(input_data) except Exception as e: assert str(e) == 'Test exception' - gates_fixture.send_notification_async.assert_called_once_with( + gates_fixture.send_notification.assert_called_once_with( metadata=metadata['metadata'], notification_id='AGGREGATION_ISSUES', message='Error aggregating data: Test exception', @@ -375,9 +363,8 @@ async def test_aggregate_data_raise_exception(gates_fixture): raise AssertionError('Exception not raised') -@pytest.mark.asyncio @patch('scouter.activities.gates.metrics') -async def test_write_metrics(mock_metrics, gates_fixture): +def test_write_metrics(mock_metrics, gates_fixture): """Test write_metrics method.""" input_data = { 'metadata': metadata['metadata'], @@ -386,7 +373,7 @@ async def test_write_metrics(mock_metrics, gates_fixture): 'value': [1.0, 2.0, None], }, } - await gates_fixture.write_metrics(input_data) + gates_fixture.write_metrics(input_data) mock_metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels.assert_called_once_with( pod_id=gates_fixture.pod_id, model_name=metadata['metadata']['model_name'], diff --git a/tests/activities/test_mongo.py b/tests/activities/test_mongo.py index d0b00d7..8f72530 100644 --- a/tests/activities/test_mongo.py +++ b/tests/activities/test_mongo.py @@ -1,7 +1,7 @@ from datetime import datetime -from unittest.mock import ANY, AsyncMock, MagicMock, patch +from unittest.mock import ANY, MagicMock, Mock, patch -from pytest import fixture, mark +from pytest import fixture from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ @@ -44,16 +44,18 @@ def mongodb_activity(mock_mongodb_repository): database_name='test_db', logger=MagicMock(), notification_handler=MagicMock(), - metrics_controller=AsyncMock(), + metrics_controller=MagicMock(), ) return mongo -def test_close(mongodb_activity): +@patch('scouter.activities.mongodb.SientiaMonitoring') +def test_close(mock_sientia_monitoring, mongodb_activity): """Test close""" mongodb_activity.close() mongodb_activity.mongodb_repository.close.assert_called_once() + mock_sientia_monitoring.shutdown.assert_called_once() def test_del(mongodb_activity): @@ -64,11 +66,10 @@ def test_del(mongodb_activity): mongodb_activity.close.assert_called_once() -@mark.asyncio -async def test_load_latest_data_none_last_data_timestamp(mongodb_activity): +def test_load_latest_data_none_last_data_timestamp(mongodb_activity): """Test load_latest_data""" - mongodb_activity.mongodb_repository.find = AsyncMock( + mongodb_activity.mongodb_repository.find = Mock( return_value=[ { 'name': 'test1', @@ -80,7 +81,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity): ] ) - result = await mongodb_activity.load_latest_data( + result = mongodb_activity.load_latest_data( { 'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, 'collection_name': 'test_collection', @@ -103,11 +104,10 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity): ] -@mark.asyncio -async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): +def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): """Test load_latest_data""" - mongodb_activity.mongodb_repository.find = AsyncMock( + mongodb_activity.mongodb_repository.find = Mock( return_value=[ { 'name': 'test1', @@ -119,7 +119,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): ] ) - result = await mongodb_activity.load_latest_data( + result = mongodb_activity.load_latest_data( { 'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, 'collection_name': 'test_collection', @@ -148,16 +148,13 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): ] -@mark.asyncio -async def test_load_latest_data_error(mongodb_activity): +def test_load_latest_data_error(mongodb_activity): """Test load_latest_data""" mongodb_activity.mongodb_repository.find.side_effect = Exception('test') mongodb_activity.send_notification = MagicMock() - mongodb_activity.send_notification_async = AsyncMock() - mongodb_activity.emit_metric = AsyncMock() try: - await mongodb_activity.load_latest_data( + mongodb_activity.load_latest_data( { 'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, 'collection_name': 'test_collection', @@ -167,7 +164,7 @@ async def test_load_latest_data_error(mongodb_activity): except Exception as e: assert str(e) == 'test' - mongodb_activity.send_notification_async.assert_called_once_with( + mongodb_activity.send_notification.assert_called_once_with( metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, notification_id='MONGO_LOAD_ERROR', message='Error loading data from MongoDB: test', diff --git a/tests/activities/test_redis.py b/tests/activities/test_redis.py index 9944951..947811d 100644 --- a/tests/activities/test_redis.py +++ b/tests/activities/test_redis.py @@ -1,5 +1,5 @@ from datetime import datetime -from unittest.mock import ANY, AsyncMock, MagicMock, patch +from unittest.mock import ANY, MagicMock, Mock, patch import numpy as np import pytest @@ -79,8 +79,7 @@ def test_redis_initialization(mock_redis_repository): assert activity.redis_repository is not None -@pytest.mark.asyncio -async def test_get_last_data_timestamp_none(redis_activity): +def test_get_last_data_timestamp_none(redis_activity): """Test get_last_data_timestamp""" test_data = { **metadata, @@ -88,9 +87,9 @@ async def test_get_last_data_timestamp_none(redis_activity): 'schedule_name': 'test_schedule', } - redis_activity.redis_repository.get = AsyncMock(return_value=None) + redis_activity.redis_repository.get = Mock(return_value=None) - result = await redis_activity.get_last_data_timestamp(test_data) + result = redis_activity.get_last_data_timestamp(test_data) redis_activity.redis_repository.get.assert_called_once_with( 'last_data_timestamp:test_pipeline:test_schedule', @@ -100,14 +99,13 @@ async def test_get_last_data_timestamp_none(redis_activity): assert result is None -@pytest.mark.asyncio -async def test_get_last_data_timestamp_not_none(redis_activity): +def test_get_last_data_timestamp_not_none(redis_activity): """Test get_last_data_timestamp""" test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'} - redis_activity.redis_repository.get = AsyncMock(return_value='2023-01-01 12:00:00') + redis_activity.redis_repository.get = Mock(return_value='2023-01-01 12:00:00') - result = await redis_activity.get_last_data_timestamp(test_data) + result = redis_activity.get_last_data_timestamp(test_data) redis_activity.redis_repository.get.assert_called_once_with( 'last_data_timestamp:test_pipeline:test_schedule', @@ -117,21 +115,20 @@ async def test_get_last_data_timestamp_not_none(redis_activity): assert result == '2023-01-01 12:00:00' -@pytest.mark.asyncio -async def test_get_last_data_timestamp_error(redis_activity): +def test_get_last_data_timestamp_error(redis_activity): """Test get_last_data_timestamp error""" test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'} - redis_activity.send_notification_async = AsyncMock() + redis_activity.send_notification = Mock() redis_activity.redis_repository.get.side_effect = Exception('test') try: - await redis_activity.get_last_data_timestamp(test_data) + redis_activity.get_last_data_timestamp(test_data) except Exception as e: assert str(e) == 'test' - redis_activity.send_notification_async.assert_called_once_with( + redis_activity.send_notification.assert_called_once_with( metadata=metadata['metadata'], notification_id='REDIS_GET_ERROR', message='Error getting last data timestamp: test', @@ -144,8 +141,7 @@ async def test_get_last_data_timestamp_error(redis_activity): raise AssertionError('Expected exception') -@pytest.mark.asyncio -async def test_put_last_data_timestamp_empty_dataframe(redis_activity): +def test_put_last_data_timestamp_empty_dataframe(redis_activity): """Test put_last_data_timestamp with empty dataframe""" test_data = { **metadata, @@ -156,15 +152,14 @@ async def test_put_last_data_timestamp_empty_dataframe(redis_activity): redis_activity.redis_repository.set = MagicMock() - result = await redis_activity.put_last_data_timestamp(test_data) + result = redis_activity.put_last_data_timestamp(test_data) assert result is None redis_activity.redis_repository.set.assert_not_called() -@pytest.mark.asyncio -async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity): +def test_put_last_data_timestamp_not_empty_dataframe(redis_activity): """Test put_last_data_timestamp with not empty dataframe""" data = DataFrame( @@ -181,9 +176,9 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity): 'data': data.to_dict('records'), } - redis_activity.redis_repository.set = AsyncMock() + redis_activity.redis_repository.set = Mock() - result = await redis_activity.put_last_data_timestamp(test_data) + result = redis_activity.put_last_data_timestamp(test_data) assert result == '2023-01-01 12:00:01' @@ -195,8 +190,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity): ) -@pytest.mark.asyncio -async def test_put_last_data_timestamp_error(redis_activity): +def test_put_last_data_timestamp_error(redis_activity): """Test put_last_data_timestamp error""" test_data = { **metadata, @@ -211,16 +205,16 @@ async def test_put_last_data_timestamp_error(redis_activity): ).to_dict('records'), } - redis_activity.send_notification_async = AsyncMock() - redis_activity.redis_repository.set = AsyncMock(side_effect=Exception('test')) + redis_activity.send_notification = Mock() + redis_activity.redis_repository.set = Mock(side_effect=Exception('test')) try: - await redis_activity.put_last_data_timestamp(test_data) + redis_activity.put_last_data_timestamp(test_data) except Exception as e: assert str(e) == 'test' - redis_activity.send_notification_async.assert_called_once_with( + redis_activity.send_notification.assert_called_once_with( metadata=metadata['metadata'], notification_id='REDIS_SET_ERROR', message='Error setting last data timestamp: test', @@ -233,8 +227,7 @@ async def test_put_last_data_timestamp_error(redis_activity): raise AssertionError('Expected exception') -@pytest.mark.asyncio -async def test_group_and_hold_data_new_key(redis_activity): +def test_group_and_hold_data_new_key(redis_activity): """Test group_and_hold_data with a new key""" # Setup test_data = { @@ -255,11 +248,11 @@ async def test_group_and_hold_data_new_key(redis_activity): } # Mock get to return None for new key - redis_activity.redis_repository.get = AsyncMock(return_value=None) - redis_activity.redis_repository.set = AsyncMock() + redis_activity.redis_repository.get = Mock(return_value=None) + redis_activity.redis_repository.set = Mock() # Call the method - result = await redis_activity.group_and_hold_data(test_data) + result = redis_activity.group_and_hold_data(test_data) # Verify the result expected_result = { @@ -279,8 +272,7 @@ async def test_group_and_hold_data_new_key(redis_activity): ) -@pytest.mark.asyncio -async def test_group_and_hold_data_update_existing_fill_missing(redis_activity): +def test_group_and_hold_data_update_existing_fill_missing(redis_activity): """Test updating existing data with group_and_hold_data""" # Setup initial data in Redis existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'} @@ -309,11 +301,11 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity): } # Mock get to return existing data - redis_activity.redis_repository.get = AsyncMock(return_value=existing_data) - redis_activity.redis_repository.set = AsyncMock() + redis_activity.redis_repository.get = Mock(return_value=existing_data) + redis_activity.redis_repository.set = Mock() # Call the method - result = await redis_activity.group_and_hold_data(test_data) + result = redis_activity.group_and_hold_data(test_data) # Verify the result expected_result = { @@ -344,8 +336,7 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity): ) -@pytest.mark.asyncio -async def test_group_and_hold_data_with_none_values(redis_activity): +def test_group_and_hold_data_with_none_values(redis_activity): """Test handling of None values in group_and_hold_data""" # Setup test data with None values test_data = { @@ -366,19 +357,18 @@ async def test_group_and_hold_data_with_none_values(redis_activity): } # Mock get to return None for new key - redis_activity.redis_repository.get = AsyncMock(return_value=None) - redis_activity.redis_repository.set = AsyncMock() + redis_activity.redis_repository.get = Mock(return_value=None) + redis_activity.redis_repository.set = Mock() # Call the method - result = await redis_activity.group_and_hold_data(test_data) + result = redis_activity.group_and_hold_data(test_data) # Verify None was converted to np.nan and values are as expected assert np.isnan(result['value'][0]) assert result['value'][1] == pytest.approx(30.0) -@pytest.mark.asyncio -async def test_group_and_hold_data_empty_dataframe(redis_activity): +def test_group_and_hold_data_empty_dataframe(redis_activity): """Test group_and_hold_data with empty DataFrame""" # Setup test with empty data test_data = { @@ -391,16 +381,15 @@ async def test_group_and_hold_data_empty_dataframe(redis_activity): 'fill_missing_tags': False, } - redis_activity.redis_repository.get = AsyncMock(return_value=None) + redis_activity.redis_repository.get = Mock(return_value=None) # Call the method - result = await redis_activity.group_and_hold_data(test_data) + result = redis_activity.group_and_hold_data(test_data) assert result == {} -@pytest.mark.asyncio -async def test_group_and_hold_data_error_get(redis_activity): +def test_group_and_hold_data_error_get(redis_activity): """Test group_and_hold_data error""" test_data = { **metadata, @@ -413,16 +402,16 @@ async def test_group_and_hold_data_error_get(redis_activity): 'fill_missing_tags': False, } - redis_activity.redis_repository.get = AsyncMock(side_effect=Exception('test')) - redis_activity.send_notification_async = AsyncMock() + redis_activity.redis_repository.get = Mock(side_effect=Exception('test')) + redis_activity.send_notification = Mock() try: - await redis_activity.group_and_hold_data(test_data) + redis_activity.group_and_hold_data(test_data) except Exception as e: assert str(e) == 'test' - redis_activity.send_notification_async.assert_called_once_with( + redis_activity.send_notification.assert_called_once_with( metadata=metadata['metadata'], notification_id='REDIS_GET_ERROR', message='Error getting held data: test', @@ -435,8 +424,7 @@ async def test_group_and_hold_data_error_get(redis_activity): raise AssertionError('Expected exception') -@pytest.mark.asyncio -async def test_group_and_hold_data_error_set(redis_activity): +def test_group_and_hold_data_error_set(redis_activity): """Test group_and_hold_data error""" test_data = { **metadata, @@ -452,21 +440,20 @@ async def test_group_and_hold_data_error_set(redis_activity): existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'} # Mock get to return existing data - redis_activity.redis_repository.get = AsyncMock(return_value=existing_data) - redis_activity.redis_repository.set = AsyncMock(side_effect=Exception('test')) - redis_activity.send_notification_async = AsyncMock() + redis_activity.redis_repository.get = Mock(return_value=existing_data) + redis_activity.redis_repository.set = Mock(side_effect=Exception('test')) + redis_activity.send_notification = Mock() try: - await redis_activity.group_and_hold_data(test_data) + redis_activity.group_and_hold_data(test_data) except Exception as e: assert str(e) == 'test' -@pytest.mark.asyncio -async def test_store_data_package(redis_activity): +def test_store_data_package(redis_activity): """Test store_data_package""" - redis_activity.redis_repository.set = AsyncMock() + redis_activity.redis_repository.set = Mock() test_data = { **metadata, @@ -489,7 +476,7 @@ async def test_store_data_package(redis_activity): 'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'}, } - await redis_activity.store_data_package(test_data) + redis_activity.store_data_package(test_data) redis_activity.redis_repository.set.assert_called_once_with( ANY, @@ -499,11 +486,10 @@ async def test_store_data_package(redis_activity): ) -@pytest.mark.asyncio -async def test_store_data_package_error(redis_activity): +def test_store_data_package_error(redis_activity): """Test store_data_package error""" - redis_activity.redis_repository.set = AsyncMock(side_effect=ValueError('test')) - redis_activity.send_notification_async = AsyncMock() + redis_activity.redis_repository.set = Mock(side_effect=ValueError('test')) + redis_activity.send_notification = Mock() test_data = { **metadata, @@ -527,9 +513,9 @@ async def test_store_data_package_error(redis_activity): } with pytest.raises(ValueError): - await redis_activity.store_data_package(test_data) + redis_activity.store_data_package(test_data) - redis_activity.send_notification_async.assert_called_once_with( + redis_activity.send_notification.assert_called_once_with( metadata=metadata['metadata'], notification_id='REDIS_SET_ERROR', message='Error setting data package: test', diff --git a/tests/utils/quality/test_filters.py b/tests/utils/quality/test_filters.py index 7cc9cc5..ca3079e 100644 --- a/tests/utils/quality/test_filters.py +++ b/tests/utils/quality/test_filters.py @@ -104,7 +104,7 @@ def test_null_values_filter(sample_dataframe, nodes_data_range): expected_data = { 'tag': ['wind_speed'], 'name': ['wind_speed'], - 'value': [None], + 'value': [np.nan], 'timestamp': [pd.Timestamp('2023-01-06')], } expected_df = pd.DataFrame(expected_data) diff --git a/values.yaml b/values.yaml index c07158e..4e429fe 100644 --- a/values.yaml +++ b/values.yaml @@ -163,7 +163,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git" - name: GITHUB_BRANCH - value: "fix/SIENTIAPDE-1478" + value: "feature/SIENTIAPDE-1646" - name: PYTHON_APP value: "scouter.worker.worker" @@ -255,6 +255,10 @@ env: value: "100" - name: SCOUTER_ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM value: "200" + - name: SCOUTER_ACTIVITY_EXECUTOR_MAX_WORKERS + value: "200" + - name: PI_WEB_API_SCOUTER_ACTIVITY_EXECUTOR_MAX_WORKERS + value: "200" ssh: @@ -265,7 +269,7 @@ ssh: # kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp -# helm upgrade --install sientia-scouter-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.6.0 +# helm upgrade --install sientia-scouter-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.6.5 # kubectl create secret generic git-ssh-key-sientia-scouter-worker \ # --namespace sientia \