gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Code import - branch feature/SIENTIAPDE-1646

Browse Source
This commit is contained in:
gitea_admin
2026-06-28 03:02:59 +00:00
commit 76abba185a
95 changed files with 16706 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
e2e/README.md Normal file
View File

@@ -0,0 +1,35 @@
# Orchestrator end-to-end tests
End-to-end tests run every external dependency for real (MongoDB, Redis, PostgreSQL via testcontainers; SMTP via in-process `aiosmtpd`; Temporal via `WorkflowEnvironment.start_local()`).
## Requirements
- Docker (for testcontainers)
- Python dev dependencies: `pip install -r requirements-dev.txt`
## Run locally
```bash
source ./venv/bin/activate
pytest e2e/ --override-ini testpaths=e2e -m e2e -v
```
Stop on first failure:
```bash
pytest e2e/ --override-ini testpaths=e2e -m e2e -x
```
## Coverage (separate from unit tests)
```bash
COVERAGE_FILE=.coverage.e2e pytest e2e/ --override-ini testpaths=e2e -m e2e --cov=orchestrator --cov-branch
coverage combine .coverage .coverage.e2e
coverage report
```
Unit tests keep the default `.coverage` file; the E2E run must set `COVERAGE_FILE=.coverage.e2e` so reports do not overwrite each other.
## Scenario catalog
See [scenarios.md](scenarios.md) for numbered scenarios and which test module implements each case.

0
e2e/__init__.py Normal file
View File

446
e2e/conftest.py Normal file
View File

@@ -0,0 +1,446 @@
"""Pytest configuration and fixtures for orchestrator E2E tests."""
import asyncio
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from unittest.mock import MagicMock
import pytest
import pytest_asyncio
from google.protobuf.duration_pb2 import Duration
from pymongo import MongoClient
from redis import Redis
from sientia_do.notifications.handlers import CoreNotificationHandler
from sientia_do.observability.logger import Logger
from sientia_do.temporal.worker.prepare_worker import build_queue_name
from sqlalchemy import create_engine
from temporalio.api.enums.v1 import IndexedValueType
from temporalio.api.operatorservice.v1 import AddSearchAttributesRequest
from temporalio.api.workflowservice.v1 import (
DescribeNamespaceRequest,
RegisterNamespaceRequest,
)
from temporalio.client import Client
from temporalio.common import SearchAttributeKey
from temporalio.service import RPCError, RPCStatusCode
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import Worker
from testcontainers.core.container import DockerContainer
from testcontainers.postgres import PostgresContainer
from e2e.helpers import MONGO_COLLECTIONS, ORCHESTRATOR_TASK_QUEUE
from e2e.smtp_test_server import SmtpTestServer
from e2e.stub_workflows import STUB_WORKFLOW_CLASSES
from orchestrator.activities.activities import Activities
from orchestrator.activities.formatters import schedule_types
from orchestrator.workflows.alerts import Alerts
from orchestrator.workflows.orchestrator import Orchestrator
from orchestrator.workflows.reports import Reports
from orchestrator.workflows.subworkflows.load_notification_package import (
LoadNotificationPackage,
)
from orchestrator.workflows.subworkflows.process_notifications import ProcessNotifications
DB_SCHEMA_SQL_PATH = Path(__file__).parent / 'db_schema.sql'
E2E_DATABASE = 'orchestrator_test'
E2E_RUNTIMES = ('legacy', 'gpu')
MANAGED_NAMESPACES = ('scouter', 'laborious')
E2E_SEARCH_ATTRIBUTES = [
SearchAttributeKey.for_keyword('model_id'),
SearchAttributeKey.for_keyword('model_name'),
SearchAttributeKey.for_keyword('orchestrated'),
]
E2E_NAMESPACE_SEARCH_ATTRIBUTES = {
'model_id': IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
'model_name': IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
'orchestrated': IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
}
async def register_namespace_if_missing(env: WorkflowEnvironment, namespace: str) -> None:
"""
Register a Temporal namespace on the local dev server and wait until it is ready.
Args:
env: Session WorkflowEnvironment from start_local().
namespace: Namespace name to register.
"""
service = env.client.service_client
try:
await service.workflow_service.register_namespace(
RegisterNamespaceRequest(
namespace=namespace,
workflow_execution_retention_period=Duration(seconds=86400),
)
)
except RPCError as err:
if err.status != RPCStatusCode.ALREADY_EXISTS:
raise
deadline = time.monotonic() + 5.0
while time.monotonic() < deadline:
try:
await service.workflow_service.describe_namespace(
DescribeNamespaceRequest(namespace=namespace)
)
return
except RPCError:
await asyncio.sleep(0.1)
raise TimeoutError(f'Namespace {namespace} not ready within 5s')
async def ensure_namespace_search_attributes(
env: WorkflowEnvironment, namespace: str
) -> None:
"""
Register the orchestrator search attributes on a namespace, if missing.
The local Temporal dev server only registers search attributes on the default
namespace at start time. Schedules created in additional namespaces fail with
"no mapping defined for search attribute ..." unless we explicitly add the
same attribute mappings to those namespaces via the operator service.
Args:
env: Session WorkflowEnvironment from start_local().
namespace: Namespace where attributes must be available.
"""
service = env.client.service_client
try:
await service.operator_service.add_search_attributes(
AddSearchAttributesRequest(
namespace=namespace,
search_attributes=dict(E2E_NAMESPACE_SEARCH_ATTRIBUTES),
)
)
except RPCError as err:
if err.status != RPCStatusCode.ALREADY_EXISTS:
raise
def temporal_host_from_env(env: WorkflowEnvironment) -> str:
"""Return target host:port for the in-process Temporal dev server."""
return env.client.service_client.config.target_host
def mongo_uri_from_container(mongo_container) -> str:
"""Build a Mongo connection string for the testcontainer."""
port = mongo_container.get_exposed_port(27017)
return f'mongodb://localhost:{port}'
@pytest_asyncio.fixture(scope='session')
def postgres_container():
"""PostgreSQL testcontainer used by all E2E tests."""
postgres = PostgresContainer('postgres:15')
postgres.start()
yield postgres
postgres.stop()
@pytest_asyncio.fixture(scope='session')
def mongo_container():
"""MongoDB testcontainer used by real CoreNotificationHandler."""
mongo = DockerContainer('mongo:7').with_exposed_ports(27017)
mongo.start()
yield mongo
mongo.stop()
@pytest_asyncio.fixture(scope='session')
def redis_container():
"""Redis testcontainer for slot and notification timestamp paths."""
redis = DockerContainer('redis:7').with_exposed_ports(6379)
redis.start()
yield redis
redis.stop()
@pytest_asyncio.fixture
def postgres_engine(postgres_container):
"""SQLAlchemy engine bound to the PostgreSQL testcontainer."""
engine = create_engine(postgres_container.get_connection_url())
yield engine
engine.dispose()
def _create_schema_and_tables(engine):
sql_text = DB_SCHEMA_SQL_PATH.read_text(encoding='utf-8')
with engine.begin() as conn:
conn.exec_driver_sql(sql_text)
@pytest_asyncio.fixture(autouse=True)
def setup_postgres_schema_and_tables(postgres_engine):
"""Recreate Postgres schema from e2e/db_schema.sql before each test."""
_create_schema_and_tables(postgres_engine)
yield
@pytest.fixture
def mongo_uri(mongo_container):
return mongo_uri_from_container(mongo_container)
@pytest_asyncio.fixture(autouse=True)
def reset_mongo_collections(mongo_uri):
"""Drop orchestrator-managed Mongo collections between tests."""
client = MongoClient(mongo_uri)
try:
db = client[E2E_DATABASE]
for name in MONGO_COLLECTIONS:
db[name].drop()
finally:
client.close()
yield
@pytest.fixture
def redis_client(redis_container):
"""Redis client bound to the testcontainer."""
port = int(redis_container.get_exposed_port(6379))
client = Redis(host='localhost', port=port, decode_responses=True)
yield client
client.close()
@pytest_asyncio.fixture(autouse=True)
def reset_redis(redis_client):
"""Flush Redis between tests."""
redis_client.flushdb()
yield
@pytest.fixture(scope='session')
def smtp_server():
"""Session-scoped in-process SMTP server."""
server = SmtpTestServer()
server.start()
yield server
server.stop()
@pytest.fixture(autouse=True)
def smtp_messages_cleanup(smtp_server):
"""Clear captured SMTP messages between tests."""
smtp_server.clear()
yield
@pytest_asyncio.fixture(scope='session')
async def temporal_env():
"""Real Temporal dev server (schedule APIs supported)."""
env = await WorkflowEnvironment.start_local(search_attributes=E2E_SEARCH_ATTRIBUTES)
for namespace in MANAGED_NAMESPACES:
await register_namespace_if_missing(env, namespace)
await ensure_namespace_search_attributes(env, namespace)
yield env
await env.shutdown()
@pytest.fixture(scope='session')
def temporal_host(temporal_env):
return temporal_host_from_env(temporal_env)
@pytest_asyncio.fixture(autouse=True)
async def cleanup_temporal_schedules(temporal_env):
"""Delete orphan schedules in scouter/laborious before each test."""
host = temporal_host_from_env(temporal_env)
for namespace in MANAGED_NAMESPACES:
client = await Client.connect(host, namespace=namespace)
async for schedule in await client.list_schedules():
handle = client.get_schedule_handle(schedule.id)
await handle.delete()
yield
@pytest_asyncio.fixture
def mock_logger():
"""Logger double with readable console output for E2E runs."""
logger = MagicMock(spec=Logger)
logger.info = MagicMock(side_effect=lambda msg: print(f'[LOG] {msg}'))
logger.debug = MagicMock(side_effect=lambda msg: print(f'[LOG] {msg}'))
logger.error = MagicMock(side_effect=lambda msg: print(f'[LOG] {msg}'))
logger.warning = MagicMock(side_effect=lambda msg: print(f'[LOG] {msg}'))
logger.custom_info = MagicMock(side_effect=lambda msg, _meta=None: print(f'[LOG] {msg}'))
logger.custom_debug = MagicMock(side_effect=lambda msg, _meta=None: print(f'[LOG] {msg}'))
logger.custom_error = MagicMock(side_effect=lambda msg, _meta=None: print(f'[LOG] {msg}'))
logger.custom_warning = MagicMock(side_effect=lambda msg, _meta=None: print(f'[LOG] {msg}'))
return logger
@pytest_asyncio.fixture
async def notification_handler(mock_logger, mongo_container):
"""Real notification handler using MongoDB testcontainer."""
handler = CoreNotificationHandler(
connection_string=mongo_uri_from_container(mongo_container),
database=E2E_DATABASE,
logger=mock_logger,
project_name='orchestrator-e2e',
)
try:
yield handler
finally:
handler.shutdown()
@pytest.fixture
def notification_inserts(notification_handler):
"""Spy on real Mongo insert calls issued by notification handler."""
collection = notification_handler.mongo_collection
original_insert_one = collection.insert_one
spy = MagicMock(wraps=original_insert_one)
collection.insert_one = spy
try:
yield spy
finally:
collection.insert_one = original_insert_one
@pytest_asyncio.fixture
async def test_activities(
postgres_container,
mongo_container,
redis_container,
smtp_server,
mock_logger,
notification_handler,
temporal_host,
):
"""Real Activities wired to testcontainers and in-process SMTP."""
mongo_port = mongo_container.get_exposed_port(27017)
redis_port = int(redis_container.get_exposed_port(6379))
activities = Activities(
temporal_config={
'temporal_host': temporal_host,
'temporal_scouter_namespace': 'scouter',
'temporal_laborious_namespace': 'laborious',
},
redis_config={
'host': 'localhost',
'port': redis_port,
'username': '',
'password': '',
},
mongodb_config={
'connection_string': f'mongodb://localhost:{mongo_port}',
'database_name': E2E_DATABASE,
'ttl_index_seconds': 3600,
},
email_config={
'sender_email': 'e2e@example.com',
'sender_password': '',
'smtp_server': smtp_server.host,
'smtp_port': smtp_server.port,
},
postgres_config={
'host': 'localhost',
'port': int(postgres_container.get_exposed_port(5432)),
'user': postgres_container.username,
'password': postgres_container.password,
'dbname': postgres_container.dbname,
'min_connections': 1,
'max_connections': 5,
},
logger=mock_logger,
notification_handler=notification_handler,
)
await activities.connect_to_temporal()
try:
yield activities
finally:
activities.shutdown()
def _orchestrator_activity_list(activities: Activities) -> list:
return [
activities.load_active_ingestors,
activities.load_opc_slots,
activities.update_slots,
activities.delete_slots,
activities.aggregate_documents_in_mongodb,
activities.find_documents_in_mongodb,
activities.update_pipelines_timestamps,
activities.create_pipelines_timestamps,
activities.delete_pipelines_timestamps,
activities.create_collection_with_ttl_index,
activities.create_schedules,
activities.update_schedules,
activities.delete_schedules,
activities.normalize_schedules,
activities.process_schedules,
activities.process_slots,
activities.create_schedule_config,
activities.create_slot_config,
activities.report_schedule_orchestration,
activities.report_slot_orchestration,
activities.format_schedule_config,
activities.get_last_data_timestamp,
activities.load_latest_data,
activities.put_last_data_timestamp,
activities.filter_notification_alerts,
activities.filter_notification_reports,
activities.build_email_html,
activities.send_email,
activities.format_log_report,
activities.export_data_to_postgres,
activities.store_notification_cache,
]
@pytest_asyncio.fixture
async def orchestrator_worker(temporal_env, test_activities):
"""Worker for orchestrator workflows and all activities on the default namespace."""
with ThreadPoolExecutor(max_workers=32) as activity_executor:
async with Worker(
temporal_env.client,
task_queue=ORCHESTRATOR_TASK_QUEUE,
workflows=[
Orchestrator,
Alerts,
Reports,
LoadNotificationPackage,
ProcessNotifications,
],
activities=_orchestrator_activity_list(test_activities),
activity_executor=activity_executor,
) as worker:
yield worker
@pytest_asyncio.fixture
async def stub_workers(temporal_env):
"""No-op workers on scouter/laborious namespaces for every managed workflow type."""
host = temporal_host_from_env(temporal_env)
worker_contexts: list[Worker] = []
clients: list[Client] = []
stub_types = list(schedule_types.keys()) + [
'xgboost_predictions_batch',
'xgboost_minimal_retrain',
]
try:
for namespace in MANAGED_NAMESPACES:
client = await Client.connect(host, namespace=namespace)
clients.append(client)
queues = {
build_queue_name(workflow_type, runtime)
for workflow_type in stub_types
for runtime in E2E_RUNTIMES
}
for queue in queues:
worker = Worker(
client,
task_queue=queue,
workflows=STUB_WORKFLOW_CLASSES,
)
await worker.__aenter__()
worker_contexts.append(worker)
yield worker_contexts
finally:
for worker in reversed(worker_contexts):
await worker.__aexit__(None, None, None)

33
e2e/db_schema.sql Normal file
View File

@@ -0,0 +1,33 @@
-- =============================================================================
-- E2E test database schema for the ``sientia_data`` namespace.
--
-- SINGLE SOURCE OF TRUTH: mirrors production DDL for tables the orchestrator
-- writes to. Any production DDL change must be pasted into this file (same
-- pattern as sientia-dataops-laborious_temporal/e2e/db_schema.sql).
-- =============================================================================
CREATE SCHEMA IF NOT EXISTS sientia_data;
-- -----------------------------------------------------------------------------
-- sientia_data.log_report
-- Written by ProcessNotifications via export_data_to_postgres.
-- The table is dropped between tests so each scenario starts with a clean
-- slate; the per-test autouse fixture re-runs this script.
-- -----------------------------------------------------------------------------
DROP TABLE IF EXISTS sientia_data.log_report;
CREATE TABLE sientia_data.log_report (
status text,
"timestamp" timestamptz,
groups text,
message text,
level text,
notification_id text,
block text,
schedule text,
pipeline text,
project text,
model_name text,
model_id text,
mail_type text,
created_at timestamptz DEFAULT CURRENT_TIMESTAMP
);

356
e2e/helpers.py Normal file
View File

@@ -0,0 +1,356 @@
"""
Shared helpers for orchestrator E2E tests (Temporal workflows + Mongo + Redis + Postgres).
"""
import asyncio
import json
import re
import uuid
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from pymongo import MongoClient
from redis import Redis
from sqlalchemy import text
from sqlalchemy.engine import Engine
from temporalio.client import Client
SCENARIO_INPUTS_DIR = Path(__file__).parent / 'scenario_inputs'
ORCHESTRATOR_TASK_QUEUE = 'orchestrator-test-queue'
MONGO_COLLECTIONS = (
'notification_queue',
'receiver_groups',
'orchestrated_schedules',
'pipelines',
'opc_servers',
'opc-servers',
)
DATETIME_FORMAT_MS_WITH_TZ = '%Y-%m-%d %H:%M:%S.%f%z'
DATETIME_FORMAT_WITH_TZ = '%Y-%m-%d %H:%M:%S%z'
_TIMESTAMP_MARKER_PATTERN = re.compile(r'^@now(?:([+-])(\d+)([smhd]))?$')
_TIMESTAMP_UNIT_TO_KWARG = {'s': 'seconds', 'm': 'minutes', 'h': 'hours', 'd': 'days'}
def _resolve_timestamp_marker(value: Any) -> Any:
"""
Convert ``@now`` / ``@now-1h`` markers into timezone-aware datetimes.
Args:
value: Any JSON value. Only strings matching the marker pattern are converted.
Return:
Any: The resolved datetime or the original value unchanged.
"""
if not isinstance(value, str):
return value
match = _TIMESTAMP_MARKER_PATTERN.match(value)
if not match:
return value
sign, amount, unit = match.groups()
now = datetime.now(UTC)
if sign is None:
return now
delta = timedelta(**{_TIMESTAMP_UNIT_TO_KWARG[unit]: int(amount)})
return now + delta if sign == '+' else now - delta
def _resolve_payload(payload: Any) -> Any:
"""Recursively walk a JSON-like structure resolving ``@now`` timestamp markers."""
if isinstance(payload, dict):
return {key: _resolve_payload(value) for key, value in payload.items()}
if isinstance(payload, list):
return [_resolve_payload(item) for item in payload]
return _resolve_timestamp_marker(payload)
def load_scenario_input(name: str, **overrides: Any) -> dict[str, Any]:
"""
Load a scenario JSON file from e2e/scenario_inputs and apply overrides.
Strings matching ``@now`` or ``@now[+-]<int>[smhd]`` (anywhere in the payload)
are converted to timezone-aware ``datetime`` instances. This lets scenario
files declare relative timestamps such as ``"updated_at": "@now-1h"``.
Args:
name: File name (with or without .json suffix).
**overrides: Top-level keys to replace in the loaded dict.
Return:
dict[str, Any]: Scenario payload with timestamp markers resolved.
"""
file_name = name if name.endswith('.json') else f'{name}.json'
file_path = SCENARIO_INPUTS_DIR / file_name
with file_path.open('r', encoding='utf-8') as handle:
payload = json.load(handle)
payload = _resolve_payload(payload)
payload.update(overrides)
return payload
def make_workflow_id(prefix: str) -> str:
"""Build a unique workflow id using a prefix and UUID suffix."""
return f'{prefix}-{uuid.uuid4().hex[:12]}'
async def start_and_await_workflow(
client: Client,
workflow_run,
input_data: dict[str, Any],
workflow_id: str,
*,
task_queue: str = ORCHESTRATOR_TASK_QUEUE,
timeout: float = 120.0,
) -> Any:
"""
Start a workflow and wait for its result.
Args:
client: Temporal client (default namespace).
workflow_run: Workflow run method (e.g. Orchestrator.run).
input_data: Workflow input payload.
workflow_id: Unique workflow id.
task_queue: Task queue for the orchestrator worker.
timeout: Max seconds to wait for completion.
Return:
Workflow result value.
"""
handle = await client.start_workflow(
workflow_run,
input_data,
id=workflow_id,
task_queue=task_queue,
)
return await asyncio.wait_for(handle.result(), timeout=timeout)
def seed_pipelines(
mongo_uri: str,
database: str,
pipelines: list[dict[str, Any]],
) -> None:
"""Insert pipeline documents into the test Mongo database."""
client = MongoClient(mongo_uri)
try:
collection = client[database]['pipelines']
if pipelines:
collection.insert_many(pipelines)
finally:
client.close()
def seed_opc_servers(
mongo_uri: str,
database: str,
servers: list[dict[str, Any]],
*,
collection: str = 'opc_servers',
) -> None:
"""Insert OPC server documents into the test Mongo database."""
client = MongoClient(mongo_uri)
try:
coll = client[database][collection]
if servers:
coll.insert_many(servers)
finally:
client.close()
def seed_receiver_groups(
mongo_uri: str,
database: str,
groups: list[dict[str, Any]],
) -> None:
"""Insert receiver group documents into the test Mongo database."""
client = MongoClient(mongo_uri)
try:
collection = client[database]['receiver_groups']
if groups:
collection.insert_many(groups)
finally:
client.close()
def seed_notifications(
mongo_uri: str,
database: str,
notifications: list[dict[str, Any]],
) -> None:
"""Insert notification_queue documents into the test Mongo database."""
client = MongoClient(mongo_uri)
try:
collection = client[database]['notification_queue']
if notifications:
collection.insert_many(notifications)
finally:
client.close()
def seed_orchestrated_schedules(
mongo_uri: str,
database: str,
schedules: list[dict[str, Any]],
) -> None:
"""Insert orchestrated_schedules tracking documents."""
client = MongoClient(mongo_uri)
try:
collection = client[database]['orchestrated_schedules']
if schedules:
collection.insert_many(schedules)
finally:
client.close()
def seed_opc_slots(redis_client: Redis, slots: dict[str, str]) -> None:
"""Write OPC slot keys (slot:opc_tags:*) in Redis."""
for key, value in slots.items():
redis_client.set(key, value)
def seed_active_ingestors(redis_client: Redis, ingestor_keys: list[str]) -> None:
"""Seed heartbeat:ingestor:* keys so load_active_ingestors returns ingestors."""
for key in ingestor_keys:
redis_client.set(key, '1')
def seed_last_timestamp(redis_client: Redis, mail_type: str, value: str) -> None:
"""
Set notification_last_timestamp for a mail type, JSON-encoded.
Values must be JSON-encoded so ``redis_repository.get`` (which calls
``json.loads`` on the raw payload) can deserialize them. The value
must follow the exact format ``sientia_do.notifications.models.Notification``
writes into ``notification_queue.timestamp``: ``DATETIME_FORMAT_WITH_TZ``
(e.g. ``"2026-05-22 16:47:02+0000"``) — no microseconds and no colon in
the timezone offset.
Args:
redis_client: Redis client connected to the test instance.
mail_type: Mail type identifier (e.g. ``"Alerts"``, ``"Reports"``).
value: Timestamp string in ``DATETIME_FORMAT_WITH_TZ``
(e.g. ``"2024-06-01 10:30:00+0000"``).
"""
redis_client.set(f'notification_last_timestamp:{mail_type}', json.dumps(value))
def seed_notification_cache(
redis_client: Redis,
trigger: str,
notification_id: str,
*,
sent_at: str | None = None,
ttl: int | None = None,
) -> None:
"""
Pre-seed alerts sent cache entry, JSON-encoded.
The value must be JSON-encoded because ``filter_notification_alerts``
reads via ``redis_repository.get`` (which applies ``json.loads``) and
parses the resulting string with ``DATETIME_FORMAT_MS_WITH_TZ``.
Args:
redis_client: Redis client connected to the test instance.
trigger: Schedule/trigger name used to compose the cache key.
notification_id: Notification id used to compose the cache key.
sent_at: Optional timestamp string in ``DATETIME_FORMAT_MS_WITH_TZ``.
ttl: Optional TTL in seconds for the cache entry.
"""
key = f'{trigger}:{notification_id}'
value = sent_at or datetime.now(UTC).strftime(DATETIME_FORMAT_MS_WITH_TZ)
encoded = json.dumps(value)
if ttl is not None:
redis_client.set(key, encoded, ex=ttl)
else:
redis_client.set(key, encoded)
def count_log_report_rows(engine: Engine, mail_type: str | None = None) -> int:
"""Count rows in sientia_data.log_report, optionally filtered by mail_type."""
sql = 'SELECT COUNT(*) FROM sientia_data.log_report'
params: dict[str, Any] = {}
if mail_type is not None:
sql += ' WHERE mail_type = :mail_type'
params['mail_type'] = mail_type
with engine.connect() as conn:
return int(conn.execute(text(sql), params).scalar() or 0)
def fetch_log_report(engine: Engine, mail_type: str | None = None) -> list[dict[str, Any]]:
"""Fetch log_report rows as dicts."""
sql = 'SELECT * FROM sientia_data.log_report'
params: dict[str, Any] = {}
if mail_type is not None:
sql += ' WHERE mail_type = :mail_type'
params['mail_type'] = mail_type
with engine.connect() as conn:
rows = conn.execute(text(sql), params).mappings().all()
return [dict(row) for row in rows]
def default_notification(
*,
notification_id: str,
level: str = 'ERROR',
timestamp: str | None = None,
model_name: str = 'model-a',
model_id: str = '1',
) -> dict[str, Any]:
"""
Build a minimal notification_queue document mirroring production layout.
The ``timestamp`` field is stored as a string in ``DATETIME_FORMAT_WITH_TZ``
because that is exactly what ``sientia_do.notifications.models.Notification``
writes into ``notification_queue`` in production (``now().strftime(
DATETIME_FORMAT_WITH_TZ)``). Tests intentionally use this same format so we
surface, rather than hide, real production behavior in downstream
activities.
Args:
notification_id: Unique identifier for the notification.
level: Notification level (e.g. ``"ERROR"``, ``"WARNING"``).
timestamp: Optional production-format timestamp string. ``None`` falls
back to a fixed sample value.
model_name: Model name attached to the notification.
model_id: Model id attached to the notification.
Return:
dict[str, Any]: A notification document ready for insertion.
"""
ts = timestamp if timestamp is not None else datetime(
2024, 6, 1, 12, 0, 0, tzinfo=UTC
).strftime(DATETIME_FORMAT_WITH_TZ)
return {
'notification_id': notification_id,
'level': level,
'timestamp': ts,
'message': f'{level} on {model_name}',
'trigger': 'test-schedule',
'block': 'test-block',
'pipeline': 'test-pipeline',
'project': 'orchestrator-e2e',
'model_name': model_name,
'model_id': model_id,
}
def default_receiver_group(
*,
group_name: str = 'admins',
members: list[str] | None = None,
levels: list[str] | None = None,
contents: list[str] | None = None,
ignore_models: list[str] | None = None,
) -> dict[str, Any]:
"""Minimal active receiver_groups document."""
return {
'group_name': group_name,
'active': True,
'members': members or ['admin@example.com'],
'levels': levels or ['ERROR', 'WARNING', 'INFO'],
'contents': contents or ['core_alerts', 'persistent_alerts', 'reports'],
'ignore_models': ignore_models or [],
}

5
e2e/scenario_inputs/alerts_duplicate.json Normal file
View File

@@ -0,0 +1,5 @@
{
"schedule_name": "alerts-e2e-dup",
"notification_ttl": 300,
"sent_ttl": 600
}

5
e2e/scenario_inputs/alerts_empty.json Normal file
View File

@@ -0,0 +1,5 @@
{
"schedule_name": "alerts-e2e-empty",
"notification_ttl": 300,
"sent_ttl": 600
}

5
e2e/scenario_inputs/alerts_happy_path.json Normal file
View File

@@ -0,0 +1,5 @@
{
"schedule_name": "alerts-e2e",
"notification_ttl": 300,
"sent_ttl": 600
}

5
e2e/scenario_inputs/alerts_persistent.json Normal file
View File

@@ -0,0 +1,5 @@
{
"schedule_name": "alerts-e2e-persistent",
"notification_ttl": 1,
"sent_ttl": 600
}

34
e2e/scenario_inputs/orchestrator_conflict.json Normal file
View File

@@ -0,0 +1,34 @@
{
"workflow_input": {
"schedule_name": "orchestrator-e2e-conflict",
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
{
"schedule_name": "conflict-pred",
"workflow_type": "predictions_batch",
"runtime": "legacy",
"model_id": "model-1",
"model": {"name": "Model model-1"},
"active": true,
"updated_at": "@now",
"frequency": "1m",
"offset": "0m",
"query": "SELECT 1",
"write_tags": [
{"server_id": "srv-1", "type": "prediction", "addr": "ns=2;s=Tag1"}
]
}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],
"active_ingestors": ["heartbeat:ingestor:1"]
}

34
e2e/scenario_inputs/orchestrator_create_only.json Normal file
View File

@@ -0,0 +1,34 @@
{
"workflow_input": {
"schedule_name": "orchestrator-e2e-create",
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
{
"schedule_name": "create-only-pred",
"workflow_type": "predictions_batch",
"runtime": "legacy",
"model_id": "model-1",
"model": {"name": "Model model-1"},
"active": true,
"updated_at": "@now",
"frequency": "1m",
"offset": "0m",
"query": "SELECT 1",
"write_tags": [
{"server_id": "srv-1", "type": "prediction", "addr": "ns=2;s=Tag1"}
]
}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],
"active_ingestors": ["heartbeat:ingestor:1"]
}

31
e2e/scenario_inputs/orchestrator_delete_only.json Normal file
View File

@@ -0,0 +1,31 @@
{
"workflow_input": {
"schedule_name": "orchestrator-e2e-delete",
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
{
"schedule_name": "delete-me",
"workflow_type": "drift",
"runtime": "legacy",
"model_id": "model-1",
"model": {"name": "Model model-1"},
"active": true,
"updated_at": "@now-1h",
"frequency": "1m",
"offset": "0m",
"interval_minutes": 60
}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],
"active_ingestors": ["heartbeat:ingestor:1"]
}

16
e2e/scenario_inputs/orchestrator_empty.json Normal file
View File

@@ -0,0 +1,16 @@
{
"workflow_input": {
"schedule_name": "orchestrator-e2e-empty",
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [],
"opc_servers": [],
"active_ingestors": []
}

46
e2e/scenario_inputs/orchestrator_happy_path.json Normal file
View File

@@ -0,0 +1,46 @@
{
"workflow_input": {
"schedule_name": "orchestrator-e2e",
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
{
"schedule_name": "pred-legacy",
"workflow_type": "predictions_batch",
"runtime": "legacy",
"model_id": "model-1",
"model": {"name": "Model model-1"},
"active": true,
"updated_at": "@now",
"frequency": "1m",
"offset": "0m",
"query": "SELECT 1",
"write_tags": [
{"server_id": "srv-1", "type": "prediction", "addr": "ns=2;s=Tag1"}
]
},
{
"schedule_name": "drift-gpu",
"workflow_type": "drift",
"runtime": "gpu",
"model_id": "model-2",
"model": {"name": "Model model-2"},
"active": true,
"updated_at": "@now",
"frequency": "1m",
"offset": "0m",
"interval_minutes": 60
}
],
"opc_servers": [
{"id": "srv-1", "active": true, "name": "opc-1"}
],
"active_ingestors": ["heartbeat:ingestor:1"]
}

41
e2e/scenario_inputs/orchestrator_noop.json Normal file
View File

@@ -0,0 +1,41 @@
{
"workflow_input": {
"schedule_name": "orchestrator-e2e-noop",
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
{
"schedule_name": "noop-pred",
"workflow_type": "predictions_batch",
"runtime": "legacy",
"model_id": "model-1",
"model": {"name": "Model model-1"},
"active": true,
"updated_at": "@now-1h",
"frequency": "1m",
"offset": "0m",
"query": "SELECT 1",
"write_tags": [
{"server_id": "srv-1", "type": "prediction", "addr": "ns=2;s=Tag1"}
]
}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],
"active_ingestors": ["heartbeat:ingestor:1"],
"orchestrated_schedules": [
{
"schedule_name": "noop-pred",
"namespace": "laborious",
"updated_at": "@now-1h"
}
]
}

45
e2e/scenario_inputs/orchestrator_ttl_index.json Normal file
View File

@@ -0,0 +1,45 @@
{
"workflow_input": {
"schedule_name": "orchestrator-e2e-ttl",
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
{
"schedule_name": "scouter-ttl",
"workflow_type": "scouter",
"runtime": "legacy",
"model_id": "model-1",
"model": {"name": "Model model-1"},
"active": true,
"updated_at": "@now",
"frequency": "1m",
"offset": "0m",
"read_tags": [
{
"server_id": "srv-1",
"tag_name": "Read1",
"tag_address": "ns=2;s=Read1",
"aggr_func": "lts",
"frequency": 1000
}
]
}
],
"opc_servers": [
{
"id": "srv-1",
"active": true,
"server_name": "opc-1",
"url": "opc.tcp://localhost:4840",
"uri": "urn:opcfoundation:UA:DemoServer"
}
],
"active_ingestors": ["heartbeat:ingestor:1"]
}

41
e2e/scenario_inputs/orchestrator_update_only.json Normal file
View File

@@ -0,0 +1,41 @@
{
"workflow_input": {
"schedule_name": "orchestrator-e2e-update",
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
{
"schedule_name": "update-pred",
"workflow_type": "predictions_batch",
"runtime": "legacy",
"model_id": "model-1",
"model": {"name": "Model model-1"},
"active": true,
"updated_at": "@now",
"frequency": "5m",
"offset": "0m",
"query": "SELECT 1",
"write_tags": [
{"server_id": "srv-1", "type": "prediction", "addr": "ns=2;s=Tag1"}
]
}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],
"active_ingestors": ["heartbeat:ingestor:1"],
"orchestrated_schedules": [
{
"schedule_name": "update-pred",
"namespace": "laborious",
"updated_at": "@now-1h"
}
]
}

3
e2e/scenario_inputs/reports_empty.json Normal file
View File

@@ -0,0 +1,3 @@
{
"schedule_name": "reports-e2e-empty"
}

3
e2e/scenario_inputs/reports_happy_path.json Normal file
View File

@@ -0,0 +1,3 @@
{
"schedule_name": "reports-e2e"
}

3
e2e/scenario_inputs/reports_multi_level.json Normal file
View File

@@ -0,0 +1,3 @@
{
"schedule_name": "reports-e2e-levels"
}

5
e2e/scenario_inputs/subworkflow_load_notification_package.json Normal file
View File

@@ -0,0 +1,5 @@
{
"schedule_name": "load-pkg-e2e",
"mail_type": "Alerts",
"base_data_filter": {"level": "ERROR"}
}

6
e2e/scenario_inputs/subworkflow_process_notifications.json Normal file
View File

@@ -0,0 +1,6 @@
{
"schedule_name": "process-notif-e2e",
"mail_type": "Alerts",
"schema": "sientia_data",
"table_name": "log_report"
}

111
e2e/scenarios.md Normal file
View File

@@ -0,0 +1,111 @@
# E2E Scenario Documentation — Orchestrator
Functional reference for orchestrator E2E scenarios. Tests live under `e2e/`, use `@pytest.mark.e2e`, and run with:
```bash
pytest e2e/ --override-ini testpaths=e2e -m e2e
```
## Execution context
- MongoDB, Redis, PostgreSQL: testcontainers (session-scoped).
- SMTP: in-process `aiosmtpd` (`e2e/smtp_test_server.py`).
- Temporal: `WorkflowEnvironment.start_local()` with stub workers on `scouter` / `laborious`.
- Production code under `orchestrator/**` is not mocked; only `Logger` may be a `MagicMock`.
---
## 1. Orchestrator workflow
Source: `e2e/test_orchestrator_main_workflow.py`
### 1.1.1 Happy path
Pipelines in Mongo → schedules created in correct namespace/task queue, Redis slots written, `orchestrated_schedules` updated.
### 1.2.1 No-op tick
Mongo, Redis, and Temporal already match desired state → no new schedules or slot writes.
### 1.3.1 Create-only
New pipeline only → schedules created, timestamps inserted.
### 1.3.2 Update-only
Existing pipeline with newer `updated_at` → schedule updated in Temporal.
### 1.3.3 Delete-only
Pipeline removed from Mongo → schedule deleted from Temporal.
### 1.4.1 Conflict ordering
Pipeline update and slot delete on same OPC server → slot insert before delete (production ordering).
### 1.5.1 Empty pipelines
No active pipelines → orphan schedules removed, no new orchestration writes.
### 1.6.1 TTL index bootstrap
First run creates TTL index on notification collection used by scouter pipelines.
---
## 2. Alerts workflow
Source: `e2e/test_alerts_main_workflow.py`
### A.1.1 Happy path
ERROR notification → one SMTP message, one `log_report` row, Redis cache key.
### A.1.2 TTL duplicate suppression
Second run with same data and cache seeded → no extra email or log row.
### A.1.3 Persistent escalation
Alert past `notification_ttl` with cache cleared → new email sent.
### A.2.1 Group filtering
Receiver group `levels` / `ignore_models` honored.
### A.3.1 Empty queue
No notifications → no SMTP, no Postgres row.
---
## 3. Reports workflow
Source: `e2e/test_reports_main_workflow.py`
### R.1.1 Happy path
Mixed ERROR/WARNING/INFO → one HTML email with all section headings.
### R.1.2 Per-level rendering
Single-level notifications → only matching section in HTML body.
### R.2.1 Empty queue
No notifications → no SMTP, no Postgres row.
---
## 4. Subworkflows
### LoadNotificationPackage — `e2e/test_subworkflow_load_notification_package.py`
- No prior Redis timestamp → all matching notifications returned, max timestamp stored.
- Prior timestamp → only newer notifications returned.
- Empty Mongo → no Redis timestamp write.
### ProcessNotifications — `e2e/test_subworkflow_process_notifications.py`
- Full round trip: HTML → SMTP → `log_report` in Postgres.
- Empty receiver groups → `{}`.

110
e2e/smtp_test_server.py Normal file
View File

@@ -0,0 +1,110 @@
"""In-process SMTP server for E2E email assertions."""
import asyncio
import socket
import threading
import time
from email import message_from_bytes, policy
from email.message import EmailMessage, Message
from aiosmtpd.controller import Controller
class _CaptureHandler:
"""
aiosmtpd handler that parses every incoming message into an ``EmailMessage``.
The default policy yields a ``Message`` instance, which loses structure
when wrapped into a new ``EmailMessage``. Parsing with ``policy.default``
keeps multipart payloads intact so tests can introspect the HTML body
via ``walk()``/``get_payload(decode=True)``.
"""
def __init__(self):
self.messages: list[EmailMessage | Message] = []
async def handle_DATA(self, server, session, envelope):
message = message_from_bytes(envelope.content, policy=policy.default)
self.messages.append(message)
return '250 OK'
def _reserve_port(host: str = '127.0.0.1') -> int:
"""Reserve a free TCP port on the given host."""
probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
probe.bind((host, 0))
port = probe.getsockname()[1]
probe.close()
return port
class SmtpTestServer:
"""
Wraps aiosmtpd Controller with a dedicated asyncio loop thread for pytest compatibility.
Attributes:
host: Bind host (127.0.0.1).
port: Listening port after start().
messages: Captured outbound messages.
"""
def __init__(self):
self.host = '127.0.0.1'
self.port: int | None = None
self._handler = _CaptureHandler()
self.messages: list[EmailMessage | Message] = self._handler.messages
self._controller: Controller | None = None
self._loop: asyncio.AbstractEventLoop | None = None
self._thread: threading.Thread | None = None
def start(self) -> None:
"""Start the SMTP controller on a reserved port in a background event loop."""
self.port = _reserve_port(self.host)
self._loop = asyncio.new_event_loop()
self._controller = Controller(
self._handler,
hostname=self.host,
port=self.port,
loop=self._loop,
ready_timeout=30,
)
def _run():
asyncio.set_event_loop(self._loop)
if self._controller is None:
raise RuntimeError('SMTP controller not initialized')
self._controller.start()
self._thread = threading.Thread(target=_run, name='e2e-smtp', daemon=True)
self._thread.start()
self._wait_until_ready()
def _wait_until_ready(self, timeout: float = 10.0) -> None:
"""Poll until the SMTP listener accepts TCP connections."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if self.port is None:
time.sleep(0.05)
continue
probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
if probe.connect_ex((self.host, self.port)) == 0:
return
finally:
probe.close()
time.sleep(0.05)
raise TimeoutError('SMTP test server did not become ready')
def stop(self) -> None:
"""Stop the controller and background event loop."""
if self._controller is not None:
self._controller.stop()
if self._thread is not None:
self._thread.join(timeout=5)
self._controller = None
self._loop = None
self._thread = None
def clear(self) -> None:
"""Remove all captured messages."""
self.messages.clear()

73
e2e/stub_workflows.py Normal file
View File

@@ -0,0 +1,73 @@
"""No-op Temporal workflows for managed scouter/laborious namespaces in E2E."""
from typing import Any
from temporalio import workflow
@workflow.defn(name='scouter')
class ScouterStub:
@workflow.run
async def run(self, _input_data: dict[str, Any]) -> None:
return None
@workflow.defn(name='pi_web_api_scouter')
class PiWebApiScouterStub:
@workflow.run
async def run(self, _input_data: dict[str, Any]) -> None:
return None
@workflow.defn(name='predictions_batch')
class PredictionsBatchStub:
@workflow.run
async def run(self, _input_data: dict[str, Any]) -> None:
return None
@workflow.defn(name='xgboost_predictions_batch')
class XgboostPredictionsBatchStub:
@workflow.run
async def run(self, _input_data: dict[str, Any]) -> None:
return None
@workflow.defn(name='drift')
class DriftStub:
@workflow.run
async def run(self, _input_data: dict[str, Any]) -> None:
return None
@workflow.defn(name='simple_metrics')
class SimpleMetricsStub:
@workflow.run
async def run(self, _input_data: dict[str, Any]) -> None:
return None
@workflow.defn(name='minimal_retrain')
class MinimalRetrainStub:
@workflow.run
async def run(self, _input_data: dict[str, Any]) -> None:
return None
@workflow.defn(name='xgboost_minimal_retrain')
class XgboostMinimalRetrainStub:
@workflow.run
async def run(self, _input_data: dict[str, Any]) -> None:
return None
STUB_WORKFLOW_CLASSES = [
ScouterStub,
PiWebApiScouterStub,
PredictionsBatchStub,
XgboostPredictionsBatchStub,
DriftStub,
SimpleMetricsStub,
MinimalRetrainStub,
XgboostMinimalRetrainStub,
]

116
e2e/test_alerts_main_workflow.py Normal file
View File

@@ -0,0 +1,116 @@
"""E2E tests for the Alerts main workflow."""
from datetime import UTC, datetime
import pytest
from redis import Redis
from temporalio.testing import WorkflowEnvironment
from e2e.conftest import E2E_DATABASE
from e2e.helpers import (
DATETIME_FORMAT_MS_WITH_TZ,
count_log_report_rows,
default_notification,
default_receiver_group,
fetch_log_report,
load_scenario_input,
make_workflow_id,
seed_notification_cache,
seed_notifications,
seed_receiver_groups,
start_and_await_workflow,
)
from orchestrator.workflows.alerts import Alerts
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_a_1_1_happy_path(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
smtp_server,
postgres_engine,
redis_client: Redis,
):
"""A.1.1: ERROR alert sends email, writes log_report, caches notification."""
seed_receiver_groups(mongo_uri, E2E_DATABASE, [default_receiver_group()])
seed_notifications(
mongo_uri,
E2E_DATABASE,
[default_notification(notification_id='alert-1', level='ERROR')],
)
input_data = load_scenario_input('alerts_happy_path.json')
await start_and_await_workflow(
temporal_env.client,
Alerts.run,
input_data,
make_workflow_id('alerts-happy'),
)
assert len(smtp_server.messages) == 1
assert count_log_report_rows(postgres_engine, 'Alerts') == 1
rows = fetch_log_report(postgres_engine, 'Alerts')
assert rows[0]['mail_type'] == 'Alerts'
assert redis_client.get('test-schedule:alert-1') is not None or redis_client.keys('*alert-1*')
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_a_1_2_duplicate_suppressed(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
smtp_server,
postgres_engine,
redis_client: Redis,
):
"""A.1.2: Cached notification is not emailed twice within sent_ttl."""
seed_receiver_groups(mongo_uri, E2E_DATABASE, [default_receiver_group()])
notif = default_notification(notification_id='dup-1', level='ERROR')
seed_notifications(mongo_uri, E2E_DATABASE, [notif])
seed_notification_cache(
redis_client,
notif['trigger'],
notif['notification_id'],
sent_at=datetime.now(UTC).strftime(DATETIME_FORMAT_MS_WITH_TZ),
ttl=600,
)
input_data = load_scenario_input('alerts_duplicate.json')
await start_and_await_workflow(
temporal_env.client,
Alerts.run,
input_data,
make_workflow_id('alerts-dup'),
)
assert len(smtp_server.messages) == 0
assert count_log_report_rows(postgres_engine, 'Alerts') == 0
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_a_3_1_empty_queue(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
smtp_server,
postgres_engine,
):
"""A.3.1: Empty notification queue short-circuits."""
seed_receiver_groups(mongo_uri, E2E_DATABASE, [default_receiver_group()])
await start_and_await_workflow(
temporal_env.client,
Alerts.run,
load_scenario_input('alerts_empty.json'),
make_workflow_id('alerts-empty'),
)
assert len(smtp_server.messages) == 0
assert count_log_report_rows(postgres_engine, 'Alerts') == 0

226
e2e/test_orchestrator_main_workflow.py Normal file
View File

@@ -0,0 +1,226 @@
"""E2E tests for the Orchestrator main workflow."""
import pytest
from pymongo import MongoClient
from redis import Redis
from sientia_do.temporal.worker.prepare_worker import build_queue_name
from temporalio.client import Client, Schedule, ScheduleActionStartWorkflow, ScheduleSpec
from temporalio.testing import WorkflowEnvironment
from e2e.conftest import E2E_DATABASE
from e2e.helpers import (
load_scenario_input,
make_workflow_id,
seed_active_ingestors,
seed_opc_servers,
seed_orchestrated_schedules,
seed_pipelines,
start_and_await_workflow,
)
from orchestrator.workflows.orchestrator import Orchestrator
async def _schedule_ids(host: str, namespace: str) -> list[str]:
"""Return the list of Temporal schedule ids in a given namespace."""
client = await Client.connect(host, namespace=namespace)
return [schedule.id async for schedule in await client.list_schedules()]
def _apply_orchestrator_seeds(
scenario: dict,
mongo_uri: str,
redis_client: Redis,
) -> None:
"""
Seed Mongo and Redis with the pipelines/opc_servers/ingestors declared in a scenario.
Args:
scenario: Scenario payload returned by ``load_scenario_input``.
mongo_uri: Mongo connection string for the test database.
redis_client: Redis client connected to the test instance.
"""
seed_pipelines(mongo_uri, E2E_DATABASE, scenario.get('pipelines', []))
seed_opc_servers(mongo_uri, E2E_DATABASE, scenario.get('opc_servers', []))
seed_active_ingestors(redis_client, scenario.get('active_ingestors', []))
if scenario.get('orchestrated_schedules'):
seed_orchestrated_schedules(
mongo_uri,
E2E_DATABASE,
scenario['orchestrated_schedules'],
)
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_1_1_1_happy_path(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
redis_client: Redis,
temporal_host: str,
):
"""1.1.1: Creates schedules, slots, and orchestrated_schedules entries."""
scenario = load_scenario_input('orchestrator_happy_path')
_apply_orchestrator_seeds(scenario, mongo_uri, redis_client)
await start_and_await_workflow(
temporal_env.client,
Orchestrator.run,
scenario['workflow_input'],
make_workflow_id('orchestrator-happy'),
)
mongo = MongoClient(mongo_uri)
try:
tracked = list(mongo[E2E_DATABASE]['orchestrated_schedules'].find())
names = {doc['schedule_name'] for doc in tracked}
assert names, f'Expected orchestrated_schedules rows, got {tracked}'
finally:
mongo.close()
laborious_schedules = await _schedule_ids(temporal_host, 'laborious')
assert 'pred-legacy' in laborious_schedules or 'drift-gpu' in laborious_schedules, (
f'Expected Temporal schedules in laborious, got {laborious_schedules}'
)
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_1_3_1_create_only(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
temporal_host: str,
redis_client: Redis,
):
"""1.3.1: New pipeline creates a Temporal schedule."""
scenario = load_scenario_input('orchestrator_create_only')
_apply_orchestrator_seeds(scenario, mongo_uri, redis_client)
await start_and_await_workflow(
temporal_env.client,
Orchestrator.run,
scenario['workflow_input'],
make_workflow_id('orchestrator-create'),
)
assert 'create-only-pred' in await _schedule_ids(temporal_host, 'laborious')
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_1_3_3_delete_only(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
temporal_host: str,
redis_client: Redis,
):
"""1.3.3: Removing pipeline deletes Temporal schedule."""
scenario = load_scenario_input('orchestrator_delete_only')
_apply_orchestrator_seeds(scenario, mongo_uri, redis_client)
await start_and_await_workflow(
temporal_env.client,
Orchestrator.run,
scenario['workflow_input'],
make_workflow_id('orchestrator-seed-delete'),
)
client = await Client.connect(temporal_host, namespace='laborious')
try:
handle = client.get_schedule_handle('delete-me')
await handle.describe()
schedule_exists = True
except Exception:
schedule_exists = False
assert schedule_exists
mongo = MongoClient(mongo_uri)
try:
mongo[E2E_DATABASE]['pipelines'].delete_many({})
finally:
mongo.close()
await start_and_await_workflow(
temporal_env.client,
Orchestrator.run,
scenario['workflow_input'],
make_workflow_id('orchestrator-delete'),
)
assert 'delete-me' not in await _schedule_ids(temporal_host, 'laborious')
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_1_5_1_empty_pipelines(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
temporal_host: str,
redis_client: Redis,
):
"""1.5.1: No pipelines → no orchestrated_schedules documents."""
scenario = load_scenario_input('orchestrator_empty')
_apply_orchestrator_seeds(scenario, mongo_uri, redis_client)
client = await Client.connect(temporal_host, namespace='laborious')
await client.create_schedule(
'orphan-schedule',
Schedule(
action=ScheduleActionStartWorkflow(
'drift',
{},
id='orphan-schedule-run',
task_queue=build_queue_name('drift', 'legacy'),
),
spec=ScheduleSpec(),
),
)
await start_and_await_workflow(
temporal_env.client,
Orchestrator.run,
scenario['workflow_input'],
make_workflow_id('orchestrator-empty'),
)
mongo = MongoClient(mongo_uri)
try:
assert mongo[E2E_DATABASE]['orchestrated_schedules'].count_documents({}) == 0
finally:
mongo.close()
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_1_6_1_ttl_index_bootstrap(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
redis_client: Redis,
):
"""1.6.1: Scouter pipeline triggers TTL index on notification_queue."""
scenario = load_scenario_input('orchestrator_ttl_index')
_apply_orchestrator_seeds(scenario, mongo_uri, redis_client)
await start_and_await_workflow(
temporal_env.client,
Orchestrator.run,
scenario['workflow_input'],
make_workflow_id('orchestrator-ttl'),
)
mongo = MongoClient(mongo_uri)
try:
indexes = mongo[E2E_DATABASE]['raw_scouter-ttl'].index_information()
assert any('expireAfterSeconds' in info for info in indexes.values())
finally:
mongo.close()

167
e2e/test_reports_main_workflow.py Normal file
View File

@@ -0,0 +1,167 @@
"""E2E tests for the Reports main workflow."""
from email.message import EmailMessage, Message
import pytest
from temporalio.testing import WorkflowEnvironment
from e2e.conftest import E2E_DATABASE
from e2e.helpers import (
count_log_report_rows,
default_notification,
default_receiver_group,
load_scenario_input,
make_workflow_id,
seed_notifications,
seed_receiver_groups,
start_and_await_workflow,
)
from orchestrator.workflows.reports import Reports
def _extract_html_body(message: EmailMessage | Message) -> str:
"""
Return the text/html portion of an email message as a decoded string.
Walks every part looking for the first text/html payload, decoding it
according to the part's transfer encoding and charset. Falls back to
the message's own ``get_content``/raw payload when no HTML part is
present so callers can still inspect plain-text reports.
Args:
message: Captured email message returned by the test SMTP server.
Return:
str: HTML body content, or an empty string when nothing decodable
is found.
"""
if message.is_multipart():
for part in message.walk():
if part.get_content_type() != 'text/html':
continue
payload = part.get_payload(decode=True)
if payload is None:
continue
charset = part.get_content_charset() or 'utf-8'
return payload.decode(charset, errors='replace')
payload = message.get_payload(decode=True)
if payload is not None:
charset = message.get_content_charset() or 'utf-8'
return payload.decode(charset, errors='replace')
try:
return message.get_content()
except (AttributeError, KeyError):
return str(message.get_payload() or '')
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_r_1_1_happy_path(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
smtp_server,
postgres_engine,
):
"""R.1.1: Mixed-level notifications produce one email with all sections."""
seed_receiver_groups(
mongo_uri,
E2E_DATABASE,
[default_receiver_group(contents=['reports'], levels=['ERROR', 'WARNING', 'INFO'])],
)
seed_notifications(
mongo_uri,
E2E_DATABASE,
[
default_notification(notification_id='r-err', level='ERROR', model_name='m1'),
default_notification(
notification_id='r-warn',
level='WARNING',
model_name='m2',
timestamp='2024-06-01 12:01:00+0000',
),
default_notification(
notification_id='r-info',
level='INFO',
model_name='m3',
timestamp='2024-06-01 12:02:00+0000',
),
],
)
await start_and_await_workflow(
temporal_env.client,
Reports.run,
load_scenario_input('reports_happy_path.json'),
make_workflow_id('reports-happy'),
)
assert len(smtp_server.messages) == 1
body = _extract_html_body(smtp_server.messages[0])
assert 'Errors detected:' in body
assert 'Warnings detected:' in body
assert 'Infos detected:' in body
# format_log_report writes one row per (notification_id, trigger) pair, so the
# three seeded notifications produce three rows even though a single email
# was sent to the receiver group.
assert count_log_report_rows(postgres_engine, 'Reports') == 3
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_r_1_2_error_section_only(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
smtp_server,
):
"""R.1.2: Only ERROR notifications → only Errors section in HTML."""
seed_receiver_groups(
mongo_uri,
E2E_DATABASE,
[default_receiver_group(contents=['reports'], levels=['ERROR'])],
)
seed_notifications(
mongo_uri,
E2E_DATABASE,
[default_notification(notification_id='only-err', level='ERROR')],
)
await start_and_await_workflow(
temporal_env.client,
Reports.run,
load_scenario_input('reports_multi_level.json'),
make_workflow_id('reports-error-only'),
)
assert smtp_server.messages, 'Expected at least one report email'
body = _extract_html_body(smtp_server.messages[0])
assert 'Errors detected:' in body
assert 'Warnings detected:' not in body
assert 'Infos detected:' not in body
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_r_2_1_empty_queue(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
smtp_server,
postgres_engine,
):
"""R.2.1: Empty queue → no email and no log_report row."""
seed_receiver_groups(mongo_uri, E2E_DATABASE, [default_receiver_group()])
await start_and_await_workflow(
temporal_env.client,
Reports.run,
load_scenario_input('reports_empty.json'),
make_workflow_id('reports-empty'),
)
assert len(smtp_server.messages) == 0
assert count_log_report_rows(postgres_engine, 'Reports') == 0

154
e2e/test_subworkflow_load_notification_package.py Normal file
View File

@@ -0,0 +1,154 @@
"""E2E tests for LoadNotificationPackage subworkflow."""
import pytest
from redis import Redis
from temporalio.testing import WorkflowEnvironment
from e2e.conftest import E2E_DATABASE
from e2e.helpers import (
default_notification,
default_receiver_group,
load_scenario_input,
make_workflow_id,
seed_last_timestamp,
seed_notifications,
seed_receiver_groups,
start_and_await_workflow,
)
from orchestrator.workflows.subworkflows.load_notification_package import LoadNotificationPackage
def _build_metadata(input_data: dict) -> None:
"""Attach the metadata block that the subworkflow expects."""
input_data['metadata'] = {
'metadata': {
'schedule_name': input_data['schedule_name'],
'workflow_name': 'load_notification_package',
'model_name': '-',
'model_id': '-',
}
}
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_load_package_without_prior_timestamp(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
redis_client: Redis,
):
"""
No Redis timestamp → returns notifications and stores max timestamp.
Notifications are seeded with the exact production format produced by
``sientia_do.notifications.models.Notification`` (string in
``DATETIME_FORMAT_WITH_TZ``, e.g. ``"2024-06-01 11:00:00+0000"``). The
cached "last timestamp" must mirror that representation.
"""
seed_receiver_groups(mongo_uri, E2E_DATABASE, [default_receiver_group()])
seed_notifications(
mongo_uri,
E2E_DATABASE,
[
default_notification(
notification_id='n1',
timestamp='2024-06-01 10:00:00+0000',
),
default_notification(
notification_id='n2',
timestamp='2024-06-01 11:00:00+0000',
),
],
)
input_data = load_scenario_input('subworkflow_load_notification_package.json')
_build_metadata(input_data)
result = await start_and_await_workflow(
temporal_env.client,
LoadNotificationPackage.run,
input_data,
make_workflow_id('load-pkg-none'),
)
assert len(result['notification_package']) == 2
stored = redis_client.get('notification_last_timestamp:Alerts') or ''
assert stored.strip('"') == '2024-06-01 11:00:00+0000'
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_load_package_with_prior_timestamp(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
redis_client: Redis,
):
"""
Existing timestamp → only newer notifications returned.
Both the seeded "last timestamp" (Redis) and the notification timestamps
(Mongo) follow the production format used by
``sientia_do.notifications.models.Notification`` (string in
``DATETIME_FORMAT_WITH_TZ``). ``load_latest_data`` will translate the
Redis value into a Python ``datetime`` and apply ``{$gt: <Date>}``
against the Mongo string timestamps; this exercise surfaces the real
BSON comparison semantics rather than a synthetic ideal.
"""
seed_receiver_groups(mongo_uri, E2E_DATABASE, [default_receiver_group()])
seed_last_timestamp(redis_client, 'Alerts', '2024-06-01 10:30:00+0000')
seed_notifications(
mongo_uri,
E2E_DATABASE,
[
default_notification(
notification_id='old',
timestamp='2024-06-01 10:00:00+0000',
),
default_notification(
notification_id='new',
timestamp='2024-06-01 11:00:00+0000',
),
],
)
input_data = load_scenario_input('subworkflow_load_notification_package.json')
_build_metadata(input_data)
result = await start_and_await_workflow(
temporal_env.client,
LoadNotificationPackage.run,
input_data,
make_workflow_id('load-pkg-ts'),
)
ids = {n['notification_id'] for n in result['notification_package']}
assert ids == {'new'}
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_load_package_empty_mongo_no_redis_write(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
redis_client: Redis,
):
"""Empty Mongo → no Redis timestamp write."""
seed_receiver_groups(mongo_uri, E2E_DATABASE, [default_receiver_group()])
input_data = load_scenario_input('subworkflow_load_notification_package.json')
_build_metadata(input_data)
await start_and_await_workflow(
temporal_env.client,
LoadNotificationPackage.run,
input_data,
make_workflow_id('load-pkg-empty'),
)
assert redis_client.get('notification_last_timestamp:Alerts') is None

94
e2e/test_subworkflow_process_notifications.py Normal file
View File

@@ -0,0 +1,94 @@
"""E2E tests for ProcessNotifications subworkflow."""
import pytest
from temporalio.testing import WorkflowEnvironment
from e2e.helpers import (
count_log_report_rows,
default_notification,
fetch_log_report,
load_scenario_input,
make_workflow_id,
start_and_await_workflow,
)
from orchestrator.workflows.subworkflows.process_notifications import ProcessNotifications
def _receiver_package():
notif = default_notification(notification_id='proc-1', level='ERROR')
return {
'admins': {
'group_name': 'admins',
'members': ['admin@example.com'],
'status': 'pending',
'notifications': [notif],
}
}
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_process_notifications_round_trip(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
smtp_server,
postgres_engine,
):
"""Email → SMTP → log_report persisted in Postgres."""
input_data = load_scenario_input('subworkflow_process_notifications.json')
input_data['metadata'] = {
'metadata': {
'schedule_name': input_data['schedule_name'],
'workflow_name': 'process_notifications',
'model_name': '-',
'model_id': '-',
}
}
input_data['notification_package'] = _receiver_package()
result = await start_and_await_workflow(
temporal_env.client,
ProcessNotifications.run,
input_data,
make_workflow_id('process-notif'),
)
assert result
assert len(smtp_server.messages) == 1
assert count_log_report_rows(postgres_engine, 'Alerts') == 1
rows = fetch_log_report(postgres_engine, 'Alerts')
assert rows[0]['notification_id'] == 'proc-1'
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_process_notifications_empty_package(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
smtp_server,
postgres_engine,
):
"""Empty receiver groups returns empty dict."""
input_data = load_scenario_input('subworkflow_process_notifications.json')
input_data['metadata'] = {
'metadata': {
'schedule_name': input_data['schedule_name'],
'workflow_name': 'process_notifications',
'model_name': '-',
'model_id': '-',
}
}
input_data['notification_package'] = {}
result = await start_and_await_workflow(
temporal_env.client,
ProcessNotifications.run,
input_data,
make_workflow_id('process-empty'),
)
assert result == {}
assert len(smtp_server.messages) == 0
assert count_log_report_rows(postgres_engine) == 0