SIENTIAPDE-1646
Update project configuration and dependencies - Added .mypy_cache and .cursor to .gitignore. - Changed asyncio_default_fixture_loop_scope and asyncio_default_test_loop_scope to "session" in pyproject.toml. - Updated e2e testing dependencies in requirements-dev.txt, replacing fakeredis and mongomock with pytest-httpserver. - Updated requirements.txt to use sientia_do instead of a specific git commit. - Modified sonar-project.properties to remove a file from coverage exclusions. - Enhanced E2E test fixtures in e2e/conftest.py for better container management. - Cleaned up e2e test files related to CoreScouter and PIWebAPIScouter workflows.
This commit is contained in:
vitor-aignosi
555
e2e/conftest.py
555
e2e/conftest.py
@@ -1,121 +1,177 @@
|
||||
"""
|
||||
Pytest configuration and fixtures for E2E tests.
|
||||
Pytest configuration and fixtures for production-faithful Scouter E2E tests.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from pandas import DataFrame
|
||||
from pymongo import MongoClient
|
||||
from redis import Redis
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from testcontainers.mongodb import MongoDbContainer
|
||||
from testcontainers.postgres import PostgresContainer
|
||||
from testcontainers.redis import RedisContainer
|
||||
from temporalio.testing import WorkflowEnvironment
|
||||
from temporalio.worker import Worker
|
||||
from temporalio.worker import PollerBehaviorAutoscaling, Worker
|
||||
|
||||
from e2e.helpers import SCOUTER_TASK_QUEUE, postgres_connection_parts
|
||||
from e2e.pi_web_api_test_server import PIWebAPITestServer
|
||||
from scouter.activities.activities import Activities
|
||||
from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter
|
||||
from scouter.workflow.scouter import Scouter
|
||||
from scouter.workflow.sub_workflows.core_scouter import CoreScouter
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
|
||||
from e2e.fixtures.fake_mongodb_repository import FakeMongoDBRepository
|
||||
from e2e.fixtures.fake_redis_repository import FakeRedisRepository
|
||||
E2E_DATABASE = 'scouter_e2e_test'
|
||||
E2E_NOTIFICATION_COLLECTION = 'notification_queue'
|
||||
DB_SCHEMA_PATH = Path(__file__).resolve().parent / 'db_schema.sql'
|
||||
|
||||
# Test constants
|
||||
TEST_MONGODB_CONNECTION_STRING = 'mongodb://localhost:27017'
|
||||
TEST_DATABASE_NAME = 'test_db'
|
||||
def _activity_list(activities: Activities) -> list:
|
||||
"""Return bound activity callables for the E2E worker."""
|
||||
return [
|
||||
activities.load_latest_data,
|
||||
activities.get_last_data_timestamp,
|
||||
activities.put_last_data_timestamp,
|
||||
activities.get_tag_values,
|
||||
activities.data_quality_gate,
|
||||
activities.aggregate_data,
|
||||
activities.group_and_hold_data,
|
||||
activities.export_data_to_postgres,
|
||||
activities.write_metrics,
|
||||
activities.store_data_package,
|
||||
]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope='session')
|
||||
@pytest.fixture(scope='session')
|
||||
def postgres_container():
|
||||
"""
|
||||
Create a PostgreSQL container using testcontainers.
|
||||
|
||||
This fixture creates a real PostgreSQL database in a Docker container
|
||||
that will be used for all tests in the session.
|
||||
Session-scoped PostgreSQL testcontainer.
|
||||
|
||||
Return:
|
||||
Running PostgresContainer instance
|
||||
"""
|
||||
postgres = PostgresContainer('postgres:15')
|
||||
postgres.start()
|
||||
yield postgres
|
||||
postgres.stop()
|
||||
container = PostgresContainer('postgres:15')
|
||||
container.start()
|
||||
yield container
|
||||
container.stop()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@pytest.fixture(scope='session')
|
||||
def mongo_container():
|
||||
"""
|
||||
Session-scoped MongoDB testcontainer.
|
||||
|
||||
Return:
|
||||
Running MongoDbContainer instance
|
||||
"""
|
||||
container = MongoDbContainer('mongo:7')
|
||||
container.start()
|
||||
yield container
|
||||
container.stop()
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def redis_container():
|
||||
"""
|
||||
Session-scoped Redis testcontainer.
|
||||
|
||||
Return:
|
||||
Running RedisContainer instance
|
||||
"""
|
||||
container = RedisContainer('redis:7')
|
||||
container.start()
|
||||
yield container
|
||||
container.stop()
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def postgres_engine(postgres_container):
|
||||
"""
|
||||
Create SQLAlchemy engine for PostgreSQL test database.
|
||||
|
||||
This fixture creates a connection to the PostgreSQL container
|
||||
created by the postgres_container fixture.
|
||||
SQLAlchemy engine bound to the Postgres testcontainer for the session.
|
||||
|
||||
Return:
|
||||
SQLAlchemy Engine
|
||||
"""
|
||||
# Get connection URL from container
|
||||
connection_string = postgres_container.get_connection_url()
|
||||
engine = create_engine(connection_string)
|
||||
|
||||
engine = create_engine(postgres_container.get_connection_url())
|
||||
yield engine
|
||||
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _create_schema_and_table(engine):
|
||||
@pytest.fixture(scope='session')
|
||||
def mongo_uri(mongo_container):
|
||||
"""
|
||||
Helper function to create schema and table in the given engine.
|
||||
|
||||
This is used by both the autouse fixture and test_activities to ensure
|
||||
the schema exists before Activities tries to use it.
|
||||
|
||||
Note: For tests, we create a non-partitioned table to avoid issues
|
||||
with pandas to_sql recognizing partitioned tables.
|
||||
MongoDB connection string for the testcontainer.
|
||||
|
||||
Return:
|
||||
Connection URI string
|
||||
"""
|
||||
schema_name = 'sientia_data'
|
||||
table_name = 'laborious_data'
|
||||
|
||||
# Use begin() to ensure transaction is properly committed
|
||||
with engine.begin() as conn:
|
||||
# Create schema
|
||||
conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {schema_name}"))
|
||||
|
||||
# Create table WITHOUT partitioning (simpler for tests)
|
||||
# Same structure as production, but without PARTITION BY RANGE
|
||||
create_table_sql = f"""
|
||||
CREATE TABLE IF NOT EXISTS {schema_name}.{table_name} (
|
||||
id SERIAL NOT NULL,
|
||||
model_id int4 NOT NULL,
|
||||
variable text NOT NULL,
|
||||
value numeric NULL,
|
||||
"timestamp" timestamptz NOT NULL,
|
||||
created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
PRIMARY KEY (id, created_at)
|
||||
);
|
||||
"""
|
||||
|
||||
conn.execute(text(create_table_sql))
|
||||
# Transaction is automatically committed when exiting the 'with' block
|
||||
return mongo_container.get_connection_url()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(autouse=True)
|
||||
@pytest.fixture(scope='session')
|
||||
def redis_client(redis_container):
|
||||
"""
|
||||
Redis client connected to the testcontainer.
|
||||
|
||||
Return:
|
||||
redis.Redis client with decode_responses=True
|
||||
"""
|
||||
host = redis_container.get_container_host_ip()
|
||||
port = int(redis_container.get_exposed_port(6379))
|
||||
client = Redis(host=host, port=port, decode_responses=True)
|
||||
yield client
|
||||
client.close()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_postgres_schema_and_table(postgres_engine):
|
||||
"""
|
||||
Automatically create necessary schema and table before each test.
|
||||
|
||||
This fixture runs automatically (autouse=True) and ensures
|
||||
that the sientia_data schema and laborious_data table exist
|
||||
with the correct structure before tests execute.
|
||||
|
||||
Note: For tests, we use a non-partitioned table which is simpler and avoids issues
|
||||
with pandas to_sql recognizing partitioned tables.
|
||||
Apply db_schema.sql before each test so laborious_data is empty and current.
|
||||
"""
|
||||
_create_schema_and_table(postgres_engine)
|
||||
sql = DB_SCHEMA_PATH.read_text(encoding='utf-8')
|
||||
with postgres_engine.begin() as conn:
|
||||
conn.exec_driver_sql(sql)
|
||||
yield
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_mongo_collections(mongo_uri):
|
||||
"""
|
||||
Drop all collections in the E2E Mongo database between tests.
|
||||
"""
|
||||
client = MongoClient(mongo_uri)
|
||||
try:
|
||||
db = client[E2E_DATABASE]
|
||||
for name in db.list_collection_names():
|
||||
db.drop_collection(name)
|
||||
finally:
|
||||
client.close()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_redis(redis_client):
|
||||
"""
|
||||
Flush the Redis testcontainer between tests.
|
||||
"""
|
||||
redis_client.flushdb()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_logger():
|
||||
"""Mock logger for testing."""
|
||||
"""
|
||||
Logger stand-in (only permitted MagicMock in the E2E harness).
|
||||
|
||||
Return:
|
||||
MagicMock with Logger spec
|
||||
"""
|
||||
logger = MagicMock(spec=Logger)
|
||||
logger.info = MagicMock()
|
||||
logger.debug = MagicMock()
|
||||
@@ -125,238 +181,139 @@ def mock_logger():
|
||||
return logger
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def mock_mongo_client():
|
||||
@pytest.fixture
|
||||
def notification_handler(mock_logger, mongo_uri):
|
||||
"""
|
||||
Mock MongoDB client to avoid real connections.
|
||||
|
||||
This fixture mocks the pymongo.MongoClient used by CoreNotificationHandler,
|
||||
allowing us to use a real NotificationHandler instance without connecting to MongoDB.
|
||||
Real CoreNotificationHandler backed by the Mongo testcontainer.
|
||||
|
||||
Return:
|
||||
CoreNotificationHandler instance
|
||||
"""
|
||||
mock_client = MagicMock()
|
||||
mock_db = MagicMock()
|
||||
mock_collection = MagicMock()
|
||||
|
||||
# Configure the mock chain: client[database] -> db[collection] -> collection
|
||||
mock_client.__getitem__.return_value = mock_db
|
||||
mock_db.__getitem__.return_value = mock_collection
|
||||
|
||||
# Mock server_info() to avoid connection attempts
|
||||
mock_client.server_info = MagicMock()
|
||||
|
||||
# Mock insert_one for notifications
|
||||
mock_collection.insert_one = MagicMock()
|
||||
|
||||
return mock_client
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def notification_handler(mock_logger, mock_mongo_client):
|
||||
"""
|
||||
Create a real NotificationHandler instance with mocked MongoDB client.
|
||||
|
||||
This fixture creates a real CoreNotificationHandler instance but mocks
|
||||
the underlying MongoDB connection to avoid real database connections.
|
||||
"""
|
||||
# Patch MongoClient where it's imported in the handlers module
|
||||
with patch('sientia_do.notifications.handlers.MongoClient', return_value=mock_mongo_client):
|
||||
handler = CoreNotificationHandler(
|
||||
connection_string=TEST_MONGODB_CONNECTION_STRING,
|
||||
database=TEST_DATABASE_NAME,
|
||||
logger=mock_logger,
|
||||
project_name='scouter',
|
||||
)
|
||||
yield handler
|
||||
handler.shutdown()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def metrics_controller(mock_logger):
|
||||
"""
|
||||
Create a real MetricsController instance.
|
||||
|
||||
MetricsController doesn't require external services, so we can use
|
||||
a real instance without mocking anything.
|
||||
"""
|
||||
controller = MetricsController(logger=mock_logger)
|
||||
yield controller
|
||||
# MetricsController might have cleanup, but it's optional
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def mock_pi_web_api_client():
|
||||
"""Mock PI Web API client."""
|
||||
mock_client = MagicMock()
|
||||
|
||||
# Mock DataFrame response similar to real API
|
||||
# The real PIWebAPIClient returns timestamp as datetime, so we need to match that
|
||||
mock_df = DataFrame({
|
||||
'timestamp': [
|
||||
'2024-01-01 12:00:00+0000',
|
||||
'2024-01-01 12:01:00+0000',
|
||||
'2024-01-01 12:02:00+0000',
|
||||
],
|
||||
'name': ['tag1', 'tag2', 'tag3'],
|
||||
'value': [10.5, 20.3, 30.7],
|
||||
'tag': ['webid1', 'webid2', 'webid3'],
|
||||
})
|
||||
|
||||
# Convert timestamp to datetime (UTC, floored to seconds) to match real client behavior
|
||||
mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp'], utc=True).dt.floor('s')
|
||||
|
||||
mock_client.get_latest_values_df = AsyncMock(return_value=mock_df)
|
||||
mock_client.close = MagicMock()
|
||||
return mock_client
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope='function')
|
||||
def test_activities(
|
||||
postgres_engine,
|
||||
postgres_container,
|
||||
mock_logger,
|
||||
notification_handler,
|
||||
metrics_controller,
|
||||
mock_pi_web_api_client,
|
||||
):
|
||||
"""
|
||||
Create Activities instance with test dependencies.
|
||||
|
||||
This fixture creates a real Activities instance with:
|
||||
- PostgreSQL database (via testcontainers)
|
||||
- FakeRedis instead of real Redis
|
||||
- FakeMongoDB instead of real MongoDB
|
||||
- Mocked PI Web API client
|
||||
- Real NotificationHandler and MetricsController (with mocked underlying services)
|
||||
"""
|
||||
# Get connection details from container
|
||||
connection_string = postgres_container.get_connection_url()
|
||||
|
||||
# Parse connection string to get individual components
|
||||
# Format: postgresql://testuser:testpass@localhost:5432/test
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(connection_string)
|
||||
|
||||
# Ensure schema and table exist BEFORE creating Activities
|
||||
# This ensures the schema exists when Activities initializes its engine
|
||||
_create_schema_and_table(postgres_engine)
|
||||
|
||||
# Create fake repositories that will be used from the start
|
||||
fake_redis_repo = FakeRedisRepository(
|
||||
host='localhost',
|
||||
port=6379,
|
||||
username='',
|
||||
password='',
|
||||
handler = CoreNotificationHandler(
|
||||
connection_string=mongo_uri,
|
||||
database=E2E_DATABASE,
|
||||
logger=mock_logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
project_name='scouter-e2e',
|
||||
notification_topic=E2E_NOTIFICATION_COLLECTION,
|
||||
)
|
||||
|
||||
fake_mongo_repo = FakeMongoDBRepository(
|
||||
connection_string=TEST_MONGODB_CONNECTION_STRING,
|
||||
database_name=TEST_DATABASE_NAME,
|
||||
logger=mock_logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
# Patch create_engine to return our postgres_engine instead of creating a new one
|
||||
# This ensures Activities uses the same engine from the start
|
||||
original_create_engine = create_engine
|
||||
|
||||
def patched_create_engine(connection_string, *args, **kwargs):
|
||||
# Check if this is the connection string that Activities would create
|
||||
# Activities creates: postgresql://user:password@host:port/dbname
|
||||
expected_conn_str = f"postgresql://{parsed.username or 'test'}:{parsed.password or 'test'}@localhost:{postgres_container.get_exposed_port(5432)}/{parsed.path.lstrip('/') if parsed.path else 'test'}"
|
||||
|
||||
# If it matches our test container connection, return our engine
|
||||
if connection_string == expected_conn_str:
|
||||
return postgres_engine
|
||||
# Otherwise, use the original create_engine
|
||||
return original_create_engine(connection_string, *args, **kwargs)
|
||||
|
||||
# Patch RedisRepository to return our fake repository from the start
|
||||
def patched_redis_repository(*args, **kwargs):
|
||||
return fake_redis_repo
|
||||
|
||||
# Patch MongoDBRepository to return our fake repository from the start
|
||||
def patched_mongodb_repository(*args, **kwargs):
|
||||
return fake_mongo_repo
|
||||
|
||||
# Patch PIWebAPIClient to return our mock from the start
|
||||
def patched_pi_web_api_client(*args, **kwargs):
|
||||
return mock_pi_web_api_client
|
||||
|
||||
# Create Activities with test configurations
|
||||
# All patches ensure it uses our test instances from the start
|
||||
with patch('sientia_do.temporal.activities.postgres.create_engine', new=patched_create_engine), \
|
||||
patch('scouter.activities.redis.RedisRepository', new=patched_redis_repository), \
|
||||
patch('scouter.activities.mongodb.MongoDBRepository', new=patched_mongodb_repository), \
|
||||
patch('scouter.activities.api.PIWebAPIClient', new=patched_pi_web_api_client):
|
||||
|
||||
activities = Activities(
|
||||
postgres_config={
|
||||
'host': 'localhost', # Container exposes to localhost
|
||||
'port': postgres_container.get_exposed_port(5432),
|
||||
'user': parsed.username or 'test',
|
||||
'password': parsed.password or 'test',
|
||||
'dbname': parsed.path.lstrip('/') if parsed.path else 'test',
|
||||
'min_connections': 1,
|
||||
'max_connections': 5,
|
||||
},
|
||||
redis_config={
|
||||
'host': 'localhost',
|
||||
'port': 6379,
|
||||
'username': '',
|
||||
'password': '',
|
||||
},
|
||||
mongodb_config={
|
||||
'connection_string': TEST_MONGODB_CONNECTION_STRING,
|
||||
'database_name': TEST_DATABASE_NAME,
|
||||
},
|
||||
api_config={
|
||||
'base_url': 'http://localhost:8080',
|
||||
'auth_type': 'bearer',
|
||||
'auth_token': 'test_token',
|
||||
},
|
||||
logger=mock_logger,
|
||||
notification_handler=notification_handler,
|
||||
)
|
||||
|
||||
# Verify that Activities is using our instances from the start
|
||||
assert activities.engine is postgres_engine, "Activities should use the same engine as postgres_engine"
|
||||
assert activities.redis_repository is fake_redis_repo, "Activities should use the same fake Redis repository"
|
||||
assert activities.mongodb_repository is fake_mongo_repo, "Activities should use the same fake MongoDB repository"
|
||||
assert activities.pi_web_api_client is mock_pi_web_api_client, "Activities should use the same mock PI Web API client"
|
||||
|
||||
yield activities
|
||||
|
||||
# Cleanup
|
||||
activities.shutdown()
|
||||
yield handler
|
||||
handler.shutdown()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope='function')
|
||||
async def temporal_test_env():
|
||||
"""Create Temporal test environment."""
|
||||
env = await WorkflowEnvironment.start_time_skipping()
|
||||
async with env:
|
||||
@pytest.fixture
|
||||
def notification_inserts(notification_handler):
|
||||
"""
|
||||
Spy wrapper around notification collection insert_one (still writes to Mongo).
|
||||
|
||||
Return:
|
||||
MagicMock wrapping insert_one
|
||||
"""
|
||||
collection = notification_handler.mongo_collection
|
||||
spy = MagicMock(wraps=collection.insert_one)
|
||||
collection.insert_one = spy
|
||||
return spy
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def pi_web_api_server():
|
||||
"""
|
||||
Session-scoped in-process PI Web API HTTP stub.
|
||||
|
||||
Return:
|
||||
Started PIWebAPITestServer instance
|
||||
"""
|
||||
server = PIWebAPITestServer()
|
||||
server.start()
|
||||
yield server
|
||||
server.stop()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cleanup_pi_web_api_server(pi_web_api_server):
|
||||
"""
|
||||
Reset PI Web API stub state between tests.
|
||||
"""
|
||||
pi_web_api_server.clear()
|
||||
yield
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope='session')
|
||||
async def temporal_env():
|
||||
"""
|
||||
Session-scoped WorkflowEnvironment using the real local Temporal dev server.
|
||||
|
||||
Return:
|
||||
WorkflowEnvironment from start_local()
|
||||
"""
|
||||
async with await WorkflowEnvironment.start_local() as env:
|
||||
yield env
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope='function')
|
||||
async def temporal_worker(temporal_test_env, test_activities):
|
||||
"""Create Temporal worker with test activities."""
|
||||
@pytest.fixture
|
||||
def test_activities(
|
||||
postgres_container,
|
||||
mongo_uri,
|
||||
redis_container,
|
||||
pi_web_api_server,
|
||||
mock_logger,
|
||||
notification_handler,
|
||||
):
|
||||
"""
|
||||
Production Activities wired to testcontainers and the PI Web API stub.
|
||||
|
||||
Return:
|
||||
Live Activities instance (no unittest.mock.patch)
|
||||
"""
|
||||
pg_parts = postgres_connection_parts(postgres_container.get_connection_url())
|
||||
redis_host = redis_container.get_container_host_ip()
|
||||
redis_port = int(redis_container.get_exposed_port(6379))
|
||||
|
||||
activities = Activities(
|
||||
postgres_config={
|
||||
**pg_parts,
|
||||
'min_connections': 1,
|
||||
'max_connections': 5,
|
||||
},
|
||||
redis_config={
|
||||
'host': redis_host,
|
||||
'port': redis_port,
|
||||
'username': '',
|
||||
'password': '',
|
||||
},
|
||||
mongodb_config={
|
||||
'connection_string': mongo_uri,
|
||||
'database_name': E2E_DATABASE,
|
||||
},
|
||||
api_config={
|
||||
'base_url': pi_web_api_server.base_url,
|
||||
'auth_type': 'bearer',
|
||||
'auth_token': 'e2e-test-token',
|
||||
},
|
||||
logger=mock_logger,
|
||||
notification_handler=notification_handler,
|
||||
)
|
||||
yield activities
|
||||
activities.shutdown()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def temporal_worker(temporal_env, test_activities):
|
||||
"""
|
||||
Temporal worker registering all Scouter workflows and activities on the E2E queue.
|
||||
|
||||
Return:
|
||||
Running temporalio.worker.Worker
|
||||
"""
|
||||
async with Worker(
|
||||
temporal_test_env.client,
|
||||
task_queue='test-queue',
|
||||
workflows=[PIWebAPIScouter, CoreScouter],
|
||||
activities=[
|
||||
test_activities.get_tag_values,
|
||||
test_activities.data_quality_gate,
|
||||
test_activities.aggregate_data,
|
||||
test_activities.group_and_hold_data,
|
||||
test_activities.export_data_to_postgres,
|
||||
test_activities.write_metrics,
|
||||
test_activities.store_data_package,
|
||||
],
|
||||
temporal_env.client,
|
||||
task_queue=SCOUTER_TASK_QUEUE,
|
||||
workflows=[Scouter, PIWebAPIScouter, CoreScouter],
|
||||
activities=_activity_list(test_activities),
|
||||
activity_executor=ThreadPoolExecutor(max_workers=50, thread_name_prefix='e2e-activity'),
|
||||
max_concurrent_workflow_tasks=50,
|
||||
max_concurrent_activities=50,
|
||||
max_concurrent_local_activities=50,
|
||||
workflow_task_poller_behavior=PollerBehaviorAutoscaling(minimum=1, initial=2, maximum=10),
|
||||
activity_task_poller_behavior=PollerBehaviorAutoscaling(minimum=1, initial=2, maximum=10),
|
||||
) as worker:
|
||||
yield worker
|
||||
|
||||
Reference in New Issue
Block a user