From cf7f9fb63cf001d06005cbe192616551e1205cc6 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 8 Jan 2026 12:41:53 -0300 Subject: [PATCH 01/21] SIENTIAPDE-1478 Refactor PIWebAPIClient and Update Configuration Imports - Removed outdated PostgreSQL, Redis, MongoDB, and API configuration functions from connectors_config.py. - Deleted the pi_web_api_client.py file as part of the refactor. - Updated imports in worker.py and api.py to use the new repository structure. - Cleaned up scenarios.md by removing obsolete scenarios related to unique constraint violations. - Commented out the specific version of the sientia-dataops-library in requirements.txt for flexibility. --- e2e/scenarios.md | 20 -- requirements.txt | 3 +- scouter/activities/api.py | 3 +- scouter/metrics.py | 22 +- scouter/utils/clients/pi_web_api_client.py | 340 ------------------ scouter/utils/connectors_config.py | 96 ----- scouter/worker/worker.py | 2 +- .../workflow/sub_workflows/core_scouter.py | 4 +- tests/utils/clients/__init__.py | 0 9 files changed, 6 insertions(+), 484 deletions(-) delete mode 100644 scouter/utils/clients/pi_web_api_client.py delete mode 100644 tests/utils/clients/__init__.py diff --git a/e2e/scenarios.md b/e2e/scenarios.md index c5a9260..253c19e 100644 --- a/e2e/scenarios.md +++ b/e2e/scenarios.md @@ -338,25 +338,6 @@ The `pi_web_api_scouter` workflow: --- -#### Scenario 2.3.2: PostgreSQL Unique Constraint Violation -**Description**: Duplicate data violates unique constraint - -**Input**: -- Data with duplicate `model_id`, `timestamp`, `variable` combination -- `on_conflict: 'ignore'` configured - -**Expected Behavior**: -- PostgreSQL handles conflict with `ON CONFLICT DO NOTHING` -- `affected_rows` may be 0 for duplicates -- Workflow continues normally - -**Assertions**: -- No exception raised -- Duplicates ignored -- Workflow continues - ---- - ## 3. Activity-Specific Scenarios > **Note**: Activity-specific scenarios are better suited for unit tests rather than e2e tests. @@ -457,7 +438,6 @@ For each scenario, verify: 2. Scenario 2.1.2: Quality Filters 3. Scenario 2.1.3: Different Aggregations 4. Scenario 3.3.5: Invalid Aggregation -5. Scenario 3.5.2: Conflict Ignore ### Low Priority (Nice to Have) 1. Scenario 4.1.2: Retry Success diff --git a/requirements.txt b/requirements.txt index 541df15..3ffda95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ psycopg2-binary sqlalchemy redis pymongo -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.7.1 +#git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.7.1 +/home/grezewave/Documents/projects/sientia/sientia-dataops-library/ prometheus-client pycurl \ No newline at end of file diff --git a/scouter/activities/api.py b/scouter/activities/api.py index fb65850..1b1c282 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -10,8 +10,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.metrics_controller import MetricsController from sientia_do.observability.sientia_monitoring import SientiaMonitoring from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ - - from scouter.utils.clients.pi_web_api_client import PIWebAPIClient + from sientia_do.repository.pi_web_api_client import PIWebAPIClient class API(SientiaMonitoring): diff --git a/scouter/metrics.py b/scouter/metrics.py index 3ef2288..a89be12 100644 --- a/scouter/metrics.py +++ b/scouter/metrics.py @@ -23,24 +23,4 @@ TAG_CHANGES_MONITOR = Gauge( 'scouter_tag_changes_monitor', 'Current value change of each tag', [*CORE_LABELS, 'tag_name'], -) - - -# Generic REST client metrics -GENERIC_REST_CLIENT_LAG = Histogram( - 'scouter_generic_rest_client_lag', - 'Lag time for a request to a generic REST client', - SIENTIA_CORE_LABELS, -) - -GENERIC_REST_READ_COUNT = Counter( - 'scouter_generic_rest_client_read_count', - 'Number of reads from a generic REST client', - SIENTIA_CORE_LABELS, -) - -GENERIC_REST_READ_ERROR_COUNT = Counter( - 'scouter_generic_rest_client_read_error_count', - 'Number of read errors from a generic REST client', - SIENTIA_CORE_LABELS, -) +) \ No newline at end of file diff --git a/scouter/utils/clients/pi_web_api_client.py b/scouter/utils/clients/pi_web_api_client.py deleted file mode 100644 index a8ba79a..0000000 --- a/scouter/utils/clients/pi_web_api_client.py +++ /dev/null @@ -1,340 +0,0 @@ -import io -import json -import time -import warnings -from typing import Any -from urllib.parse import urlencode - -import pandas as pd -import pycurl -from sientia_do.notifications.handlers import NotificationHandler -from sientia_do.observability.logger import Logger -from sientia_do.observability.metrics_controller import MetricsController -from sientia_do.observability.sientia_monitoring import SientiaMonitoring - -from scouter import metrics - -warnings.simplefilter('ignore') # Ignore warnings such as 'verify=False' - - -class PIMSRequestError(Exception): - """Generic error for failed requests to PI Web API using pycurl.""" - - pass - - -class PIWebAPIClient(SientiaMonitoring): - """ - Client for interacting with the PI Web API. - - This class provides a robust interface for querying historical and real-time - data from OSIsoft PI systems through the PI Web API. It implements: - - Asynchronous HTTP requests using pycurl - - Authentication support (Basic and Bearer) - - Automatic data normalization and timestamp handling - - Comprehensive error handling and monitoring - - Metrics collection for observability - - The client is designed for high-performance data retrieval with proper - connection management and error recovery mechanisms. - """ - - def __init__( - self, - base_url: str, - auth_config: dict[str, Any], - logger: Logger, - notification_handler: NotificationHandler, - metrics_controller: MetricsController, - headers_config: dict[str, Any] | None = None, - ) -> None: - """ - Initialize the PI Web API client with connection parameters. - - Args: - base_url (str): Base URL of the PI Web API server - auth_config (dict[str, Any]): Authentication configuration. - Required fields: - - type (str): Authentication type ('basic' or 'bearer') - - token (str): Authentication token - logger (Logger): Logger instance for operation logging - notification_handler (NotificationHandler): Handler for system notifications - metrics_controller (MetricsController): Controller for metrics collection - headers_config (dict[str, Any], optional): HTTP headers configuration. - Default headers include content-type, accept, and x-requested-with - max_concurrency (int, optional): Maximum number of concurrent requests. Defaults to 8 - """ - if headers_config is None: - headers_config = { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - 'x-requested-with': 'XMLHttpRequest', - } - SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) - - self.base_url = base_url.rstrip('/') - - # auth_config spec: - # 'type': 'basic' or 'bearer', - # 'token': 'token', - self.auth_config = auth_config - self.auth_config['type'] = self.auth_config['type'].lower() - self.headers: dict[str, str] = headers_config - - self.authenticate() - - def close(self) -> None: - """ - Close the client and shutdown monitoring services. - """ - SientiaMonitoring.shutdown(self) - - def _to_clean_timestamp(self, series: pd.Series) -> pd.Series: - """ - Convert a Series of timestamps to datetime, UTC, and round to the nearest second. - - Args: - series (pd.Series): Series containing timestamp values - - Returns: - pd.Series: Cleaned timestamp series in UTC, floored to seconds - """ - series = pd.to_datetime(series, utc=True, errors='coerce') - return series.dt.floor('s') - - def _extract_numeric(self, value: Any) -> float | None: - """ - Normalize a value (potentially nested) to float. - - This method handles PI Web API response values that may be nested - in dictionaries or other structures, extracting the numeric value. - - Args: - value (Any): Value to extract and normalize - - Returns: - float | None: Numeric value as float, or None if conversion fails - """ - if isinstance(value, dict): - value = value.get('Value', value) - return pd.to_numeric(value, errors='coerce') - - def authenticate(self): - """ - Configure authentication headers based on auth_config. - - This method sets up the Authorization header using either Basic or Bearer - authentication based on the configured authentication type. - - Raises: - ValueError: If authentication type is not 'basic' or 'bearer' - """ - self.logger.info(f'Authenticating with {self.auth_config["type"]} authentication') - - if self.auth_config['type'] == 'basic': - self.headers['Authorization'] = f'Basic {self.auth_config["token"]}' - elif self.auth_config['type'] == 'bearer': - self.headers['Authorization'] = f'Bearer {self.auth_config["token"]}' - else: - raise ValueError(f'Invalid authentication type: {self.auth_config["type"]}') - - async def _curl_get_json( - self, - url: str, - params: list[tuple[str, str]] | None = None, - timeout: int = 30, - verify: bool = True, - metadata: dict[str, Any] | None = None, - ) -> dict[str, Any]: - """ - Perform a GET request using pycurl and return the decoded JSON response. - - This method executes an asynchronous HTTP GET request with proper error handling, - metrics collection, and timeout management. It automatically tracks request - latency and emits monitoring metrics. - - Args: - url (str): Target URL for the GET request - params (list[tuple[str, str]], optional): Query parameters as list of tuples. - Each tuple contains (parameter_name, parameter_value) - timeout (int, optional): Request timeout in seconds. Defaults to 30 - verify (bool, optional): Verify SSL certificates. Defaults to True - metadata (dict[str, Any], optional): Workflow execution metadata for tracking - - Returns: - dict[str, Any]: Parsed JSON response body - - Raises: - PIMSRequestError: If HTTP error, connection error, or JSON parsing error occurs - """ - if metadata is None: - metadata = {} - - buffer = io.BytesIO() - c = pycurl.Curl() - - core_labels = self.get_core_labels(metadata=metadata, operation_type='get_json') - - try: - if params: - query_string = urlencode(params, doseq=True) - full_url = f'{url}?{query_string}' - else: - full_url = url - - c.setopt(pycurl.URL, full_url.encode('utf-8')) - c.setopt(pycurl.WRITEDATA, buffer) - - # Configure HTTP headers - header_list = [f'{k}: {v}' for k, v in self.headers.items()] - if header_list: - c.setopt(pycurl.HTTPHEADER, header_list) - - # Set request timeout - c.setopt(pycurl.TIMEOUT, timeout) - - # Configure SSL verification - if not verify: - c.setopt(pycurl.SSL_VERIFYPEER, 0) - c.setopt(pycurl.SSL_VERIFYHOST, 0) - - start_time = time.time() - try: - c.perform() - except Exception as e: - await self.emit_metric( - metric_object=metrics.GENERIC_REST_READ_ERROR_COUNT, - tags=core_labels, - ) - raise e - await self.observe_lag( - start_time=start_time, - metric_object=metrics.GENERIC_REST_CLIENT_LAG, - tags=core_labels, - ) - - status_code = c.getinfo(pycurl.RESPONSE_CODE) - body = buffer.getvalue().decode('utf-8', errors='replace') - - if status_code >= 400: - await self.emit_metric( - metric_object=metrics.GENERIC_REST_READ_ERROR_COUNT, - tags=core_labels, - ) - raise PIMSRequestError(f"HTTP {status_code} calling '{full_url}': {body[:200]}") - - await self.emit_metric( - metric_object=metrics.GENERIC_REST_READ_COUNT, - tags=core_labels, - ) - try: - return json.loads(body) - except json.JSONDecodeError as e: - raise PIMSRequestError( - f"Error decoding JSON response from '{full_url}': {e}; body: {body[:200]}" - ) from e - - except pycurl.error as e: - raise PIMSRequestError(f"Connection error calling '{url}': {e}") from e - finally: - c.close() - - async def get_latest_values_df( - self, - web_ids: dict[str, dict[str, str]], - endpoint: str, - timeout: int = 30, - start_time: str = '*-1d', - end_time: str = '*', - max_count: int | None = 1, - metadata: dict[str, Any] | None = None, - ) -> pd.DataFrame: - """ - Retrieve historical values for multiple WebIds using PI Web API streamsets. - - This method queries the PI Web API's /streamsets/recorded endpoint to fetch - historical data for multiple tags simultaneously. It returns a normalized - DataFrame with timestamps, tag names, values, and WebIds. - - Args: - web_ids (dict[str, str | None]): Dictionary mapping tag names to their WebIds. - None values are filtered out before querying - endpoint (str): PI Web API endpoint path (e.g., '/streamsets/recorded') - timeout (int, optional): Request timeout in seconds. Defaults to 30 - start_time (str, optional): Start time in PI Web API format (e.g., "*-50d"). - Defaults to "*-1d" (1 day ago) - end_time (str, optional): End time in PI Web API format (e.g., "*"). - Defaults to "*" (current time) - max_count (int, optional): Maximum number of data points per series. - Defaults to 1. If None, maxCount parameter is not sent - metadata (dict[str, Any], optional): Workflow execution metadata for tracking - - Returns: - pd.DataFrame: DataFrame with columns: - - timestamp: Cleaned timestamp (UTC, floored to seconds) - - name: Tag name - - value: Numeric value (normalized) - - tag: WebId of the tag - Returns empty DataFrame if no data is found - - Raises: - PIMSRequestError: If API request fails or returns invalid data - """ - if metadata is None: - metadata = {} - - url = f'{self.base_url}{endpoint}' - - # Build query parameters with WebIds (filtering out None values) - params: list[tuple[str, str]] = [('webid', web_id['webid']) for web_id in web_ids.values()] - # Invert startTime and endTime to get descending order (most recent first) - # PI Web API returns descending order when endTime < startTime - params.extend( - [ - ('startTime', end_time), # Use end_time as startTime (inverted) - ('endtime', start_time), # Use start_time as endTime (inverted) - ('selectedFields', 'Items.Name;Items.Items.Timestamp;Items.Items.Value'), - ] - ) - - if max_count is not None: - params.append(('maxCount', str(max_count))) - - data = await self._curl_get_json( - url=url, - params=params, - timeout=timeout, - verify=False, - metadata=metadata, - ) - - self.debug(f'Raw data from PI Web API: {data}', metadata=metadata) - - raw_data = data.get('Items', []) - - records = [] - for entry in raw_data: - tag_name = entry.get('Name') - series_items = entry.get('Items', []) - for it in series_items: - if isinstance(it, dict) and 'Timestamp' in it and 'Value' in it: - ts = it.get('Timestamp') - val = it.get('Value') - web_id = web_ids[tag_name]['webid'] - if ts is not None: - records.append( - { - 'timestamp': ts, - 'name': tag_name, - 'value': self._extract_numeric(val), - 'tag': web_id, - } - ) - - if not records: - return pd.DataFrame() - - df = pd.DataFrame.from_records(records) - df['timestamp'] = self._to_clean_timestamp(df['timestamp']) - - return df diff --git a/scouter/utils/connectors_config.py b/scouter/utils/connectors_config.py index da712d9..64737df 100644 --- a/scouter/utils/connectors_config.py +++ b/scouter/utils/connectors_config.py @@ -2,31 +2,6 @@ from os import getenv from typing import Any -def build_postgres_config() -> dict[str, Any]: - """ - Build PostgreSQL connection configuration from environment variables. - - Returns: - dict[str, Any]: PostgreSQL configuration dictionary with keys: - - host: Database hostname (default: localhost) - - port: Database port (default: 5432) - - user: Database username (default: sientia) - - password: Database password (default: sientia) - - dbname: Database name (default: sientia) - - min_connections: Minimum connection pool size (default: 5) - - max_connections: Maximum connection pool size (default: 20) - """ - return { - 'host': getenv('POSTGRES_HOST', 'localhost'), - 'port': int(getenv('POSTGRES_PORT', '5432')), - 'user': getenv('POSTGRES_USER', 'sientia'), - 'password': getenv('POSTGRES_PASSWORD', 'sientia'), - 'dbname': getenv('POSTGRES_DBNAME', 'sientia'), - 'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')), - 'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20')), - } - - def build_kafka_config() -> dict[str, Any]: """ Build Kafka configuration from environment variables. @@ -42,74 +17,3 @@ def build_kafka_config() -> dict[str, Any]: 'polling_time': int(getenv('KAFKA_POLLING_TIME', '1000')), 'group_id': 'scouter-group', } - - -def build_redis_config() -> dict[str, Any]: - """ - Build Redis connection configuration from environment variables. - - Returns: - dict[str, Any]: Redis configuration dictionary with keys: - - host: Redis server hostname (default: localhost) - - port: Redis server port (default: 6379) - - username: Redis authentication username (default: None) - - password: Redis authentication password (default: None) - """ - return { - 'host': getenv('REDIS_HOST', 'localhost'), - 'port': int(getenv('REDIS_PORT', '6379')), - 'username': getenv('REDIS_USERNAME', None), - 'password': getenv('REDIS_PASSWORD', None), - } - - -def build_mongodb_config() -> dict[str, Any]: - """ - Build MongoDB connection configuration from environment variables. - - Returns: - dict[str, Any]: MongoDB configuration dictionary with keys: - - connection_string: Complete MongoDB connection URI - - database_name: Target database name (default: sientia) - """ - username = getenv('MONGODB_USERNAME', 'sientia') - password = getenv('MONGODB_PASSWORD', 'sientia') - uri = getenv('MONGODB_URL', 'localhost:27017') - - connection_string = f'mongodb://{username}:{password}@{uri}' - return { - 'connection_string': connection_string, - 'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'), - } - - -def build_api_config() -> dict[str, Any]: - """ - Build API connection configuration from environment variables. - - Returns: - dict[str, Any]: API configuration dictionary with keys: - - base_url: API base URL (default: https://pi.example.com) - - auth_type: API authentication type (default: basic) - - auth_token: API authentication token (default: None) - """ - return { - 'base_url': getenv('PI_WEB_API_BASE_URL', 'https://pi.example.com'), - 'auth_type': getenv('PI_WEB_API_AUTH_TYPE', 'basic'), - 'auth_token': getenv('PI_WEB_API_AUTH_TOKEN', None), - } - - -def build_druid_config() -> dict[str, Any]: - """ - Build Apache Druid connection configuration from environment variables. - - Returns: - dict[str, Any]: Druid configuration dictionary with keys: - - host: Druid server hostname (default: localhost) - - port: Druid server port (default: 8082) - """ - return { - 'host': getenv('DRUID_HOST', 'localhost'), - 'port': int(getenv('DRUID_PORT', '8082')), - } diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 4de701a..d518f66 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -14,7 +14,7 @@ with workflow.unsafe.imports_passed_through(): from scouter import metrics from scouter.activities.activities import Activities - from scouter.utils.connectors_config import ( + from sientia_do.connectors_config import ( build_api_config, build_mongodb_config, build_postgres_config, diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py index 2e9b1c1..13b7942 100644 --- a/scouter/workflow/sub_workflows/core_scouter.py +++ b/scouter/workflow/sub_workflows/core_scouter.py @@ -109,9 +109,7 @@ class CoreScouter: 'schema': input_data['schema'], 'table_name': input_data['table_name'], 'data': held_data, - 'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ}, - 'on_conflict': 'ignore', - 'unique_columns': ['model_id', 'timestamp', 'variable'], + 'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ} }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=60), diff --git a/tests/utils/clients/__init__.py b/tests/utils/clients/__init__.py deleted file mode 100644 index e69de29..0000000 From 7958afbadf858a5b16da4b85d9b0e0c660f8a7e7 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 8 Jan 2026 16:13:59 -0300 Subject: [PATCH 02/21] SIENTIAPDE-1478 Update pytest_asyncio fixture scopes in conftest.py for improved test isolation and add asyncio_default_fixture_loop_scope in pyproject.toml. Remove outdated scenarios from scenarios.md and delete unused test files for cleaner codebase. --- e2e/conftest.py | 27 +- e2e/scenarios.md | 19 - e2e/test_core_scouter_early_exit.py | 97 --- e2e/test_core_scouter_errors.py | 99 --- e2e/test_pi_web_api_scouter.py | 106 ---- e2e/test_pi_web_api_scouter_errors.py | 2 +- pyproject.toml | 1 + tests/utils/clients/test_pi_web_api_client.py | 588 ------------------ tests/utils/test_connectors_config.py | 160 ----- 9 files changed, 14 insertions(+), 1085 deletions(-) delete mode 100644 e2e/test_pi_web_api_scouter.py delete mode 100644 tests/utils/clients/test_pi_web_api_client.py diff --git a/e2e/conftest.py b/e2e/conftest.py index 9fe947b..34042fa 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -30,7 +30,7 @@ TEST_MONGODB_CONNECTION_STRING = 'mongodb://localhost:27017' TEST_DATABASE_NAME = 'test_db' -@pytest.fixture(scope='session') +@pytest_asyncio.fixture(scope='session') def postgres_container(): """ Create a PostgreSQL container using testcontainers. @@ -44,7 +44,7 @@ def postgres_container(): postgres.stop() -@pytest.fixture +@pytest_asyncio.fixture def postgres_engine(postgres_container): """ Create SQLAlchemy engine for PostgreSQL test database. @@ -81,7 +81,6 @@ def _create_schema_and_table(engine): # Create table WITHOUT partitioning (simpler for tests) # Same structure as production, but without PARTITION BY RANGE - # Use UNIQUE constraint directly since table is not partitioned create_table_sql = f""" CREATE TABLE IF NOT EXISTS {schema_name}.{table_name} ( id SERIAL NOT NULL, @@ -90,8 +89,7 @@ def _create_schema_and_table(engine): value numeric NULL, "timestamp" timestamptz NOT NULL, created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL, - PRIMARY KEY (id, created_at), - UNIQUE (model_id, timestamp, variable) + PRIMARY KEY (id, created_at) ); """ @@ -99,7 +97,7 @@ def _create_schema_and_table(engine): # Transaction is automatically committed when exiting the 'with' block -@pytest.fixture(autouse=True) +@pytest_asyncio.fixture(autouse=True) def setup_postgres_schema_and_table(postgres_engine): """ Automatically create necessary schema and table before each test. @@ -108,15 +106,14 @@ def setup_postgres_schema_and_table(postgres_engine): that the sientia_data schema and laborious_data table exist with the correct structure before tests execute. - Note: For tests, we use a non-partitioned table with a UNIQUE constraint - directly in the table definition, which is simpler and avoids issues + Note: For tests, we use a non-partitioned table which is simpler and avoids issues with pandas to_sql recognizing partitioned tables. """ _create_schema_and_table(postgres_engine) yield -@pytest.fixture +@pytest_asyncio.fixture def mock_logger(): """Mock logger for testing.""" logger = MagicMock(spec=Logger) @@ -128,7 +125,7 @@ def mock_logger(): return logger -@pytest.fixture +@pytest_asyncio.fixture def mock_mongo_client(): """ Mock MongoDB client to avoid real connections. @@ -153,7 +150,7 @@ def mock_mongo_client(): return mock_client -@pytest.fixture +@pytest_asyncio.fixture def notification_handler(mock_logger, mock_mongo_client): """ Create a real NotificationHandler instance with mocked MongoDB client. @@ -173,7 +170,7 @@ def notification_handler(mock_logger, mock_mongo_client): handler.shutdown() -@pytest.fixture +@pytest_asyncio.fixture def metrics_controller(mock_logger): """ Create a real MetricsController instance. @@ -186,7 +183,7 @@ def metrics_controller(mock_logger): # MetricsController might have cleanup, but it's optional -@pytest.fixture +@pytest_asyncio.fixture def mock_pi_web_api_client(): """Mock PI Web API client.""" mock_client = MagicMock() @@ -212,8 +209,8 @@ def mock_pi_web_api_client(): return mock_client -@pytest_asyncio.fixture -async def test_activities( +@pytest_asyncio.fixture(scope='function') +def test_activities( postgres_engine, postgres_container, mock_logger, diff --git a/e2e/scenarios.md b/e2e/scenarios.md index 253c19e..edf97b7 100644 --- a/e2e/scenarios.md +++ b/e2e/scenarios.md @@ -299,25 +299,6 @@ The `pi_web_api_scouter` workflow: --- -#### Scenario 2.2.2: Zero Affected Rows After Export -**Description**: PostgreSQL export returns zero affected rows - -**Input**: -- Data that results in `affected_rows: 0` from export - -**Expected Behavior**: -- `export_data_to_postgres` returns `{'affected_rows': 0}` -- Workflow checks `if data_exported.get('affected_rows', 0) <= 0:` and returns early -- `write_metrics` NOT called -- `store_data_package` NOT called - -**Assertions**: -- Early return after export -- No metrics written -- Workflow completes without error - ---- - ### 2.3 Error Scenarios #### Scenario 2.3.1: Redis Connection Error diff --git a/e2e/test_core_scouter_early_exit.py b/e2e/test_core_scouter_early_exit.py index ca2e47f..2de7704 100644 --- a/e2e/test_core_scouter_early_exit.py +++ b/e2e/test_core_scouter_early_exit.py @@ -94,100 +94,3 @@ async def test_scenario_2_2_1_empty_data_after_grouping( # Should have 0 rows since export_data_to_postgres was not called assert row_count == 0, f"Expected no data in PostgreSQL, got {row_count} rows" - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_2_2_2_zero_affected_rows_after_export( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - postgres_engine, -): - """ - Scenario 2.2.2: Zero Affected Rows After Export - - PostgreSQL export returns zero affected rows, workflow exits early. - - Note: This scenario is hard to test directly in e2e because we'd need to - simulate a conflict or other condition that results in 0 affected rows. - We'll test by inserting duplicate data first, then running the workflow again. - """ - client = temporal_test_env.client - - # First, insert some data directly to create a conflict scenario - test_data = [ - { - 'timestamp': '2024-01-01 12:00:00+0000', - 'name': 'tag1', - 'value': 10.5, - 'tag': 'webid1', - }, - ] - - # Insert data directly into PostgreSQL to create duplicates - schema_name = 'sientia_data' - table_name = 'laborious_data' - full_table_name = f"{schema_name}.{table_name}" - - with postgres_engine.connect() as conn: - conn.execute( - text(f""" - INSERT INTO {full_table_name} (model_id, variable, value, timestamp) - VALUES (1, 'tag1', 10.5, '2024-01-01 12:00:00+00:00') - ON CONFLICT (model_id, timestamp, variable) DO NOTHING - """) - ) - conn.commit() - - # Prepare input data with the same data (will result in conflict) - input_data = { - 'metadata': { - 'metadata': { - 'model_id': '1', - 'model_name': 'Test Model', - 'schedule_name': 'test-schedule', - 'workflow_name': 'pi_web_api_scouter', - } - }, - 'workflow_name': 'pi_web_api_scouter', - 'schedule_name': 'test-schedule', - 'model_name': 'Test Model', - 'model_id': '1', - 'data': test_data, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - } - - # Start workflow - handle = await client.start_workflow( - CoreScouter.run, - input_data, - id=f'test-core-scouter-zero-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion (should complete without error) - await handle.result() - - # Verify the count didn't increase (conflict handled, 0 affected rows) - with postgres_engine.connect() as conn: - result = conn.execute( - text(f"SELECT COUNT(*) FROM {full_table_name} WHERE model_id = 1") - ) - row_count = result.scalar() - - # Should still have 1 row (the original one, duplicate was ignored) - assert row_count == 1, f"Expected 1 row (duplicate ignored), got {row_count}" - diff --git a/e2e/test_core_scouter_errors.py b/e2e/test_core_scouter_errors.py index 6f723ef..e7d60f5 100644 --- a/e2e/test_core_scouter_errors.py +++ b/e2e/test_core_scouter_errors.py @@ -88,102 +88,3 @@ async def test_scenario_2_3_1_redis_connection_error( # Restore original method test_activities.redis_repository.get = original_get - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_scenario_2_3_2_postgresql_unique_constraint_violation( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - postgres_engine, -): - """ - Scenario 2.3.2: PostgreSQL Unique Constraint Violation - - Duplicate data violates unique constraint, handled gracefully with ON CONFLICT DO NOTHING. - """ - client = temporal_test_env.client - - # Generate unique model_id to avoid conflicts with other tests - unique_id = int(datetime.now().timestamp() * 1000) % 1000000 - - # First, insert data directly to create a duplicate - schema_name = 'sientia_data' - table_name = 'laborious_data' - full_table_name = f"{schema_name}.{table_name}" - - with postgres_engine.connect() as conn: - conn.execute( - text(f""" - INSERT INTO {full_table_name} (model_id, variable, value, timestamp) - VALUES (:model_id, 'tag1', 10.5, '2024-01-01 12:00:00+00:00') - ON CONFLICT (model_id, timestamp, variable) DO NOTHING - """), - {'model_id': unique_id} - ) - conn.commit() - - # Prepare the same data to trigger conflict - test_data = [ - { - 'timestamp': '2024-01-01 12:00:00+0000', - 'name': 'tag1', - 'value': 10.5, - 'tag': 'webid1', - }, - ] - - input_data = { - 'metadata': { - 'metadata': { - 'model_id': str(unique_id), - 'model_name': 'Test Model', - 'schedule_name': 'test-schedule', - 'workflow_name': 'pi_web_api_scouter', - } - }, - 'workflow_name': 'pi_web_api_scouter', - 'schedule_name': 'test-schedule', - 'model_name': 'Test Model', - 'model_id': str(unique_id), - 'data': test_data, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - } - - # Start workflow - handle = await client.start_workflow( - CoreScouter.run, - input_data, - id=f'test-core-scouter-conflict-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - should complete without error - # (conflict is handled gracefully with ON CONFLICT DO NOTHING) - await handle.result() - - # Verify no exception was raised and workflow completed - # The duplicate should be ignored (0 affected rows), but workflow should complete - with postgres_engine.connect() as conn: - result = conn.execute( - text(f"SELECT COUNT(*) FROM {full_table_name} WHERE model_id = :model_id"), - {'model_id': unique_id} - ) - row_count = result.scalar() - - # Should still have 1 row (duplicate was ignored) - assert row_count == 1, f"Expected 1 row (duplicate ignored), got {row_count}" - diff --git a/e2e/test_pi_web_api_scouter.py b/e2e/test_pi_web_api_scouter.py deleted file mode 100644 index 71f002f..0000000 --- a/e2e/test_pi_web_api_scouter.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -End-to-end tests for PI Web API Scouter workflow. -""" - -from datetime import datetime - -import pytest -from sqlalchemy import inspect, text -from temporalio.testing import WorkflowEnvironment -from temporalio.worker import Worker - -from scouter.activities.activities import Activities -from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter - - -@pytest.mark.asyncio -@pytest.mark.integration -async def test_pi_web_api_scouter_e2e( - temporal_test_env: WorkflowEnvironment, - temporal_worker: Worker, - test_activities: Activities, - mock_pi_web_api_client, - postgres_engine, -): - """ - End-to-end test for PI Web API Scouter workflow. - - This test: - 1. Starts the workflow with test data - 2. Verifies PI Web API is called - 3. Verifies data flows through CoreScouter - 4. Verifies data is stored in PostgreSQL (schema: sientia_data, table: laborious_data) - 5. Verifies data is cached in Redis - """ - client = temporal_test_env.client - - # Prepare test input - input_data = { - 'model_name': 'PI Web API Scouter Test Model', - 'model_id': '1', - 'schedule_name': 'pi-web-api-scouter-test', - 'model_tags': { - 'tag1': { - 'webid': 'webid1', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - 'tag2': { - 'webid': 'webid2', - 'aggr_function': 'avg', - 'data_range': [0, 100], - 'frequency': 60000, - }, - }, - 'trigger_laborious': False, - 'filters': {}, - 'schema': 'sientia_data', - 'table_name': 'laborious_data', - 'retention_time': 3600, - 'fill_missing_tags': False, - 'pi_web_api_query': { - 'endpoint': '/streamsets/recorded', - 'period': '*-1d', - 'max_count': 10, - 'api_timeout': 30, - }, - } - - # Start workflow - handle = await client.start_workflow( - PIWebAPIScouter.run, - input_data, - id=f'test-workflow-{datetime.now().timestamp()}', - task_queue='test-queue', - ) - - # Wait for workflow completion - await handle.result() - - # Verify PI Web API was called - mock_pi_web_api_client.get_latest_values_df.assert_called_once() - - # Verify data was stored in PostgreSQL - inspector = inspect(postgres_engine) - - # Schema and table are created by the setup_postgres_schema_and_table fixture - schema_name = 'sientia_data' - table_name = 'laborious_data' - full_table_name = f"{schema_name}.{table_name}" - - # Check if table exists in the schema - table_exists = inspector.has_table(table_name, schema=schema_name) - - assert table_exists, f"Expected table {full_table_name} to exist in PostgreSQL" - - # Verify data was inserted - with postgres_engine.connect() as conn: - result = conn.execute(text(f"SELECT COUNT(*) FROM {full_table_name}")) - row_count = result.scalar() - - assert row_count > 0, f"Expected data in PostgreSQL table {full_table_name}, got {row_count} rows" - - # Verify data was cached in Redis - keys = await test_activities.redis_repository.keys('*') - assert len(keys) > 0, "Expected data in Redis" diff --git a/e2e/test_pi_web_api_scouter_errors.py b/e2e/test_pi_web_api_scouter_errors.py index cb4e512..f1088be 100644 --- a/e2e/test_pi_web_api_scouter_errors.py +++ b/e2e/test_pi_web_api_scouter_errors.py @@ -11,7 +11,7 @@ from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker from scouter.activities.activities import Activities -from scouter.utils.clients.pi_web_api_client import PIMSRequestError +from sientia_do.repository.pi_web_api_client import PIMSRequestError from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter diff --git a/pyproject.toml b/pyproject.toml index 25e2eab..f77f9f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,7 @@ addopts = [ "--strict-markers", ] asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" markers = [ "asyncio: marks tests as async", "integration: marks tests as integration tests", diff --git a/tests/utils/clients/test_pi_web_api_client.py b/tests/utils/clients/test_pi_web_api_client.py deleted file mode 100644 index f4f8870..0000000 --- a/tests/utils/clients/test_pi_web_api_client.py +++ /dev/null @@ -1,588 +0,0 @@ -import json -from unittest.mock import AsyncMock, MagicMock, patch - -import pandas as pd -import pycurl -import pytest - -from scouter.utils.clients.pi_web_api_client import PIMSRequestError, PIWebAPIClient - - -@pytest.fixture -def mock_logger(): - return MagicMock() - - -@pytest.fixture -def mock_notification_handler(): - return AsyncMock() - - -@pytest.fixture -def mock_metrics_controller(): - return AsyncMock() - - -@pytest.fixture -def auth_config_basic(): - return {'type': 'basic', 'token': 'test_token_123'} - - -@pytest.fixture -def auth_config_bearer(): - return {'type': 'bearer', 'token': 'bearer_token_456'} - - -@pytest.fixture -def pi_client(mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_basic): - return PIWebAPIClient( - base_url='https://pi.example.com', - auth_config=auth_config_basic, - logger=mock_logger, - notification_handler=mock_notification_handler, - metrics_controller=mock_metrics_controller, - ) - - -def test_init_with_basic_auth( - mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_basic -): - """Test initialization with basic authentication""" - client = PIWebAPIClient( - base_url='https://pi.example.com/', - auth_config=auth_config_basic, - logger=mock_logger, - notification_handler=mock_notification_handler, - metrics_controller=mock_metrics_controller, - ) - - assert client.base_url == 'https://pi.example.com' - assert client.auth_config['type'] == 'basic' - assert client.headers['Authorization'] == 'Basic test_token_123' - assert client.headers['Content-Type'] == 'application/json' - assert client.headers['Accept'] == 'application/json' - mock_logger.info.assert_called_with('Authenticating with basic authentication') - - -def test_init_with_bearer_auth( - mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_bearer -): - """Test initialization with bearer authentication""" - client = PIWebAPIClient( - base_url='https://pi.example.com', - auth_config=auth_config_bearer, - logger=mock_logger, - notification_handler=mock_notification_handler, - metrics_controller=mock_metrics_controller, - ) - - assert client.base_url == 'https://pi.example.com' - assert client.auth_config['type'] == 'bearer' - assert client.headers['Authorization'] == 'Bearer bearer_token_456' - mock_logger.info.assert_called_with('Authenticating with bearer authentication') - - -def test_init_with_custom_headers( - mock_logger, mock_notification_handler, mock_metrics_controller, auth_config_basic -): - """Test initialization with custom headers""" - custom_headers = { - 'Content-Type': 'application/xml', - 'Custom-Header': 'custom_value', - } - - client = PIWebAPIClient( - base_url='https://pi.example.com', - auth_config=auth_config_basic, - logger=mock_logger, - notification_handler=mock_notification_handler, - metrics_controller=mock_metrics_controller, - headers_config=custom_headers, - ) - - assert client.headers['Content-Type'] == 'application/xml' - assert client.headers['Custom-Header'] == 'custom_value' - assert client.headers['Authorization'] == 'Basic test_token_123' - - -def test_authenticate_invalid_type(mock_logger, mock_notification_handler, mock_metrics_controller): - """Test that invalid authentication type raises ValueError""" - invalid_auth_config = {'type': 'invalid', 'token': 'test_token'} - - with pytest.raises(ValueError) as exc_info: - PIWebAPIClient( - base_url='https://pi.example.com', - auth_config=invalid_auth_config, - logger=mock_logger, - notification_handler=mock_notification_handler, - metrics_controller=mock_metrics_controller, - ) - - assert 'Invalid authentication type: invalid' in str(exc_info.value) - - -@patch('scouter.utils.clients.pi_web_api_client.SientiaMonitoring.shutdown') -def test_close(mock_shutdown, pi_client): - """Test close method calls shutdown""" - pi_client.close() - - mock_shutdown.assert_called_once() - - -def test_to_clean_timestamp(pi_client): - """Test timestamp cleaning and normalization""" - timestamps = pd.Series( - [ - '2025-01-15T10:30:45.123456Z', - '2025-01-15T10:30:46.789012Z', - '2025-01-15T10:30:47.999999Z', - ] - ) - - result = pi_client._to_clean_timestamp(timestamps) - - assert isinstance(result, pd.Series) - assert result.dtype == 'datetime64[ns, UTC]' - # Verify microseconds are floored to seconds - assert result[0] == pd.Timestamp('2025-01-15T10:30:45Z') - assert result[1] == pd.Timestamp('2025-01-15T10:30:46Z') - assert result[2] == pd.Timestamp('2025-01-15T10:30:47Z') - - -def test_to_clean_timestamp_with_invalid_values(pi_client): - """Test timestamp cleaning with invalid values returns NaT""" - timestamps = pd.Series(['invalid', 'not_a_date', '2025-01-15T10:30:45Z']) - - result = pi_client._to_clean_timestamp(timestamps) - - assert pd.isna(result[0]) - assert pd.isna(result[1]) - assert result[2] == pd.Timestamp('2025-01-15T10:30:45Z') - - -def test_extract_numeric_with_float(pi_client): - """Test extracting numeric value from float""" - result = pi_client._extract_numeric(42.5) - - assert result == pytest.approx(42.5) - - -def test_extract_numeric_with_int(pi_client): - """Test extracting numeric value from int""" - result = pi_client._extract_numeric(42) - - assert result == pytest.approx(42.0) - - -def test_extract_numeric_with_string(pi_client): - """Test extracting numeric value from string""" - result = pi_client._extract_numeric('123.45') - - assert result == pytest.approx(123.45) - - -def test_extract_numeric_with_dict(pi_client): - """Test extracting numeric value from dictionary""" - result = pi_client._extract_numeric({'Value': 99.9}) - - assert result == pytest.approx(99.9) - - -def test_extract_numeric_with_invalid_value(pi_client): - """Test extracting numeric value from invalid value returns None/NaN""" - result = pi_client._extract_numeric('invalid_number') - - assert pd.isna(result) - - -@pytest.mark.asyncio -@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl') -async def test_curl_get_json_success(mock_curl_class, pi_client): - """Test successful GET request with JSON response""" - mock_curl = MagicMock() - mock_curl_class.return_value = mock_curl - - response_data = {'status': 'success', 'data': [1, 2, 3]} - response_json = json.dumps(response_data).encode('utf-8') - - def mock_perform(): - buffer = mock_curl.setopt.call_args_list[1][0][1] - buffer.write(response_json) - - mock_curl.perform.side_effect = mock_perform - mock_curl.getinfo.return_value = 200 - - result = await pi_client._curl_get_json('https://pi.example.com/api/test') - - assert result == response_data - mock_curl.setopt.assert_any_call(pycurl.TIMEOUT, 30) - mock_curl.perform.assert_called_once() - mock_curl.close.assert_called_once() - - -@pytest.mark.asyncio -@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl') -async def test_curl_get_json_with_params(mock_curl_class, pi_client): - """Test GET request with query parameters""" - mock_curl = MagicMock() - mock_curl_class.return_value = mock_curl - - response_data = {'result': 'ok'} - response_json = json.dumps(response_data).encode('utf-8') - - def mock_perform(): - buffer = mock_curl.setopt.call_args_list[1][0][1] - buffer.write(response_json) - - mock_curl.perform.side_effect = mock_perform - mock_curl.getinfo.return_value = 200 - - params = [('key1', 'value1'), ('key2', 'value2')] - result = await pi_client._curl_get_json('https://pi.example.com/api', params=params) - - assert result == response_data - # Verify URL includes query parameters - set_url_call = [call for call in mock_curl.setopt.call_args_list if call[0][0] == pycurl.URL][0] - assert b'key1=value1' in set_url_call[0][1] - assert b'key2=value2' in set_url_call[0][1] - - -@pytest.mark.asyncio -@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl') -async def test_curl_get_json_http_error(mock_curl_class, pi_client): - """Test GET request with HTTP error response""" - mock_curl = MagicMock() - mock_curl_class.return_value = mock_curl - - error_response = b'{"error": "Not found"}' - - def mock_perform(): - buffer = mock_curl.setopt.call_args_list[1][0][1] - buffer.write(error_response) - - mock_curl.perform.side_effect = mock_perform - mock_curl.getinfo.return_value = 404 - - with pytest.raises(PIMSRequestError) as exc_info: - await pi_client._curl_get_json('https://pi.example.com/api/notfound') - - assert 'HTTP 404' in str(exc_info.value) - mock_curl.close.assert_called_once() - - -@pytest.mark.asyncio -@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl') -async def test_curl_get_json_connection_error(mock_curl_class, pi_client): - """Test GET request with connection error""" - mock_curl = MagicMock() - mock_curl_class.return_value = mock_curl - - mock_curl.perform.side_effect = pycurl.error('Connection failed') - - with pytest.raises(PIMSRequestError) as exc_info: - await pi_client._curl_get_json('https://pi.example.com/api/test') - - assert 'Connection error' in str(exc_info.value) - mock_curl.close.assert_called_once() - - -@pytest.mark.asyncio -@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl') -async def test_curl_get_json_invalid_json(mock_curl_class, pi_client): - """Test GET request with invalid JSON response""" - mock_curl = MagicMock() - mock_curl_class.return_value = mock_curl - - invalid_json = b'This is not valid JSON' - - def mock_perform(): - buffer = mock_curl.setopt.call_args_list[1][0][1] - buffer.write(invalid_json) - - mock_curl.perform.side_effect = mock_perform - mock_curl.getinfo.return_value = 200 - - with pytest.raises(PIMSRequestError) as exc_info: - await pi_client._curl_get_json('https://pi.example.com/api/test') - - assert 'Error decoding JSON response' in str(exc_info.value) - mock_curl.close.assert_called_once() - - -@pytest.mark.asyncio -@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl') -async def test_curl_get_json_with_custom_timeout(mock_curl_class, pi_client): - """Test GET request with custom timeout""" - mock_curl = MagicMock() - mock_curl_class.return_value = mock_curl - - response_data = {'status': 'ok'} - response_json = json.dumps(response_data).encode('utf-8') - - def mock_perform(): - buffer = mock_curl.setopt.call_args_list[1][0][1] - buffer.write(response_json) - - mock_curl.perform.side_effect = mock_perform - mock_curl.getinfo.return_value = 200 - - await pi_client._curl_get_json('https://pi.example.com/api/test', timeout=60) - - mock_curl.setopt.assert_any_call(pycurl.TIMEOUT, 60) - - -@pytest.mark.asyncio -@patch('scouter.utils.clients.pi_web_api_client.pycurl.Curl') -async def test_curl_get_json_without_ssl_verify(mock_curl_class, pi_client): - """Test GET request with SSL verification disabled""" - mock_curl = MagicMock() - mock_curl_class.return_value = mock_curl - - response_data = {'status': 'ok'} - response_json = json.dumps(response_data).encode('utf-8') - - def mock_perform(): - buffer = mock_curl.setopt.call_args_list[1][0][1] - buffer.write(response_json) - - mock_curl.perform.side_effect = mock_perform - mock_curl.getinfo.return_value = 200 - - await pi_client._curl_get_json('https://pi.example.com/api/test', verify=False) - - mock_curl.setopt.assert_any_call(pycurl.SSL_VERIFYPEER, 0) - mock_curl.setopt.assert_any_call(pycurl.SSL_VERIFYHOST, 0) - - -@pytest.mark.asyncio -@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock) -async def test_get_latest_values_df_success(mock_curl_get_json, pi_client): - """Test successful retrieval of latest values""" - mock_curl_get_json.return_value = { - 'Items': [ - { - 'Name': 'tag1', - 'Items': [ - {'Timestamp': '2025-01-15T10:30:00Z', 'Value': 42.5}, - {'Timestamp': '2025-01-15T10:31:00Z', 'Value': 43.0}, - ], - }, - { - 'Name': 'tag2', - 'Items': [ - {'Timestamp': '2025-01-15T10:30:00Z', 'Value': 100.0}, - ], - }, - ] - } - - web_ids = { - 'tag1': {'webid': 'webid1'}, - 'tag2': {'webid': 'webid2'}, - } - - result = await pi_client.get_latest_values_df( - web_ids=web_ids, - endpoint='/streamsets/recorded', - start_time='*-1d', - end_time='*', - max_count=10, - ) - - assert isinstance(result, pd.DataFrame) - assert len(result) == 3 - assert list(result.columns) == ['timestamp', 'name', 'value', 'tag'] - assert result['name'].tolist() == ['tag1', 'tag1', 'tag2'] - assert result['value'].tolist() == [42.5, 43.0, 100.0] - - mock_curl_get_json.assert_called_once() - call_args = mock_curl_get_json.call_args - assert call_args[1]['url'] == 'https://pi.example.com/streamsets/recorded' - assert call_args[1]['timeout'] == 30 - - -@pytest.mark.asyncio -@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock) -async def test_get_latest_values_df_with_custom_params(mock_curl_get_json, pi_client): - """Test get_latest_values_df with custom parameters""" - mock_curl_get_json.return_value = { - 'Items': [ - { - 'Name': 'tag1', - 'Items': [ - {'Timestamp': '2025-01-15T10:30:00Z', 'Value': 42.5}, - ], - } - ] - } - - web_ids = {'tag1': {'webid': 'webid1'}} - metadata = {'model_id': 'test_model'} - - result = await pi_client.get_latest_values_df( - web_ids=web_ids, - endpoint='/streamsets/recorded', - start_time='*-7d', - end_time='*-1d', - max_count=100, - timeout=60, - metadata=metadata, - ) - - assert isinstance(result, pd.DataFrame) - assert len(result) == 1 - - mock_curl_get_json.assert_called_once() - call_args = mock_curl_get_json.call_args - params = call_args[1]['params'] - - # Verify parameters (inverted: startTime uses end_time, endTime uses start_time) - assert ('startTime', '*-1d') in params - assert ('endtime', '*-7d') in params - assert ('maxCount', '100') in params - assert call_args[1]['timeout'] == 60 - assert call_args[1]['metadata'] == metadata - - -@pytest.mark.asyncio -@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock) -async def test_get_latest_values_df_empty_response(mock_curl_get_json, pi_client): - """Test get_latest_values_df with empty response""" - mock_curl_get_json.return_value = {'Items': []} - - web_ids = {'tag1': {'webid': 'webid1'}} - - result = await pi_client.get_latest_values_df( - web_ids=web_ids, - endpoint='/streamsets/recorded', - ) - - assert isinstance(result, pd.DataFrame) - assert len(result) == 0 - - -@pytest.mark.asyncio -@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock) -async def test_get_latest_values_df_no_items_in_tag(mock_curl_get_json, pi_client): - """Test get_latest_values_df when tag has no items""" - mock_curl_get_json.return_value = { - 'Items': [ - { - 'Name': 'tag1', - 'Items': [], - } - ] - } - - web_ids = {'tag1': {'webid': 'webid1'}} - - result = await pi_client.get_latest_values_df( - web_ids=web_ids, - endpoint='/streamsets/recorded', - ) - - assert isinstance(result, pd.DataFrame) - assert len(result) == 0 - - -@pytest.mark.asyncio -@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock) -async def test_get_latest_values_df_with_missing_timestamp(mock_curl_get_json, pi_client): - """Test get_latest_values_df filters out items with missing timestamp""" - mock_curl_get_json.return_value = { - 'Items': [ - { - 'Name': 'tag1', - 'Items': [ - {'Timestamp': '2025-01-15T10:30:00Z', 'Value': 42.5}, - {'Value': 43.0}, # Missing Timestamp - {'Timestamp': None, 'Value': 44.0}, # None Timestamp - ], - } - ] - } - - web_ids = {'tag1': {'webid': 'webid1'}} - - result = await pi_client.get_latest_values_df( - web_ids=web_ids, - endpoint='/streamsets/recorded', - ) - - assert isinstance(result, pd.DataFrame) - assert len(result) == 1 # Only the first item should be included - assert result['value'].tolist() == [42.5] - - -@pytest.mark.asyncio -@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock) -async def test_get_latest_values_df_with_nested_value(mock_curl_get_json, pi_client): - """Test get_latest_values_df with nested value extraction""" - mock_curl_get_json.return_value = { - 'Items': [ - { - 'Name': 'tag1', - 'Items': [ - {'Timestamp': '2025-01-15T10:30:00Z', 'Value': {'Value': 42.5}}, - ], - } - ] - } - - web_ids = {'tag1': {'webid': 'webid1'}} - - result = await pi_client.get_latest_values_df( - web_ids=web_ids, - endpoint='/streamsets/recorded', - ) - - assert isinstance(result, pd.DataFrame) - assert len(result) == 1 - assert result['value'].tolist() == [42.5] - - -@pytest.mark.asyncio -@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock) -async def test_get_latest_values_df_default_max_count(mock_curl_get_json, pi_client): - """Test get_latest_values_df uses default max_count of 1""" - mock_curl_get_json.return_value = {'Items': []} - - web_ids = {'tag1': {'webid': 'webid1'}} - - await pi_client.get_latest_values_df( - web_ids=web_ids, - endpoint='/streamsets/recorded', - ) - - call_args = mock_curl_get_json.call_args - params = call_args[1]['params'] - - assert ('maxCount', '1') in params - # Verify default time parameters are inverted (startTime uses end_time default, endTime uses start_time default) - assert ('startTime', '*') in params # Default end_time - assert ('endtime', '*-1d') in params # Default start_time - - -@pytest.mark.asyncio -@patch.object(PIWebAPIClient, '_curl_get_json', new_callable=AsyncMock) -async def test_get_latest_values_df_with_none_max_count(mock_curl_get_json, pi_client): - """Test get_latest_values_df does not send maxCount parameter when max_count is None""" - mock_curl_get_json.return_value = {'Items': []} - - web_ids = {'tag1': {'webid': 'webid1'}} - - await pi_client.get_latest_values_df( - web_ids=web_ids, - endpoint='/streamsets/recorded', - max_count=None, - ) - - call_args = mock_curl_get_json.call_args - params = call_args[1]['params'] - - # Verify maxCount parameter is not present when max_count is None - assert ('maxCount', '1') not in params - assert ('maxCount', None) not in params - # Verify time parameters are still present - assert ('startTime', '*') in params - assert ('endtime', '*-1d') in params diff --git a/tests/utils/test_connectors_config.py b/tests/utils/test_connectors_config.py index be7ab70..fe24347 100644 --- a/tests/utils/test_connectors_config.py +++ b/tests/utils/test_connectors_config.py @@ -4,12 +4,7 @@ from unittest.mock import patch import pytest from scouter.utils.connectors_config import ( - build_api_config, - build_druid_config, build_kafka_config, - build_mongodb_config, - build_postgres_config, - build_redis_config, ) @@ -19,50 +14,6 @@ def mock_env_vars(): yield -@pytest.mark.usefixtures('mock_env_vars') -def test_build_postgres_config_defaults(): - """Test that build_postgres_config returns default values when no env vars are set""" - config = build_postgres_config() - - assert config == { - 'host': 'localhost', - 'port': 5432, - 'user': 'sientia', - 'password': 'sientia', - 'dbname': 'sientia', - 'min_connections': 5, - 'max_connections': 20, - } - - -@pytest.mark.usefixtures('mock_env_vars') -def test_build_postgres_config_with_env_vars(): - """Test that build_postgres_config uses env vars when set""" - with patch.dict( - os.environ, - { - 'POSTGRES_HOST': 'db.example.com', - 'POSTGRES_PORT': '5433', - 'POSTGRES_USER': 'admin', - 'POSTGRES_PASSWORD': 'secret', - 'POSTGRES_DBNAME': 'test_db', - 'POSTGRES_MIN_CONNECTIONS': '3', - 'POSTGRES_MAX_CONNECTIONS': '15', - }, - ): - config = build_postgres_config() - - assert config == { - 'host': 'db.example.com', - 'port': 5433, - 'user': 'admin', - 'password': 'secret', - 'dbname': 'test_db', - 'min_connections': 3, - 'max_connections': 15, - } - - @pytest.mark.usefixtures('mock_env_vars') def test_build_kafka_config_defaults(): """Test that build_kafka_config returns default values when no env vars are set""" @@ -89,114 +40,3 @@ def test_build_kafka_config_with_env_vars(): 'polling_time': 5000, 'group_id': 'scouter-group', } - - -@pytest.mark.usefixtures('mock_env_vars') -def test_build_redis_config_defaults(): - """Test that build_redis_config returns default values when no env vars are set""" - config = build_redis_config() - - assert config == {'host': 'localhost', 'port': 6379, 'username': None, 'password': None} - - -@pytest.mark.usefixtures('mock_env_vars') -def test_build_redis_config_with_env_vars(): - """Test that build_redis_config uses env vars when set""" - with patch.dict( - os.environ, - { - 'REDIS_HOST': 'redis.example.com', - 'REDIS_PORT': '6380', - 'REDIS_USERNAME': 'test', - 'REDIS_PASSWORD': 'test', - }, - ): - config = build_redis_config() - - assert config == { - 'host': 'redis.example.com', - 'port': 6380, - 'username': 'test', - 'password': 'test', - } - - -def test_build_mongodb_config_defaults(): - """Test that build_mongodb_config returns default values when no env vars are set""" - os.environ['MONGODB_URL'] = 'localhost:27017' - os.environ['MONGODB_DATABASE_NAME'] = 'sientia' - os.environ['MONGODB_USERNAME'] = 'sientia' - os.environ['MONGODB_PASSWORD'] = 'sientia' - - config = build_mongodb_config() - - assert config == { - 'connection_string': 'mongodb://sientia:sientia@localhost:27017', # NOSONAR - 'database_name': 'sientia', - } - - -def test_build_mongodb_config_with_env_vars(): - """Test that build_mongodb_config uses env vars when set""" - with patch.dict( - os.environ, - { - 'MONGODB_URL': 'mongodb.example.com:27017', - 'MONGODB_DATABASE_NAME': 'test_db', - 'MONGODB_USERNAME': 'test', - 'MONGODB_PASSWORD': 'test', - }, - ): - config = build_mongodb_config() - - assert config == { - 'connection_string': 'mongodb://test:test@mongodb.example.com:27017', - 'database_name': 'test_db', - } - - -@pytest.mark.usefixtures('mock_env_vars') -def test_build_api_config_defaults(): - """Test that build_api_config returns default values when no env vars are set""" - config = build_api_config() - - assert config == { - 'base_url': 'https://pi.example.com', - 'auth_type': 'basic', - 'auth_token': None, - } - - -@pytest.mark.usefixtures('mock_env_vars') -def test_build_api_config_with_env_vars(): - """Test that build_api_config uses env vars when set""" - with patch.dict( - os.environ, - { - 'PI_WEB_API_BASE_URL': 'https://api.production.com', - 'PI_WEB_API_AUTH_TYPE': 'bearer', - 'PI_WEB_API_AUTH_TOKEN': 'secret_token_123', - }, - ): - config = build_api_config() - - assert config == { - 'base_url': 'https://api.production.com', - 'auth_type': 'bearer', - 'auth_token': 'secret_token_123', - } - - -def test_build_druid_config_defaults(): - """Test that build_druid_config returns default values when no env vars are set""" - config = build_druid_config() - - assert config == {'host': 'localhost', 'port': 8082} - - -def test_build_druid_config_with_env_vars(): - """Test that build_druid_config uses env vars when set""" - with patch.dict(os.environ, {'DRUID_HOST': 'druid.example.com', 'DRUID_PORT': '8083'}): - config = build_druid_config() - - assert config == {'host': 'druid.example.com', 'port': 8083} From f2b648ead272ea66908462fd8ae3336b8ca644f4 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 8 Jan 2026 16:20:10 -0300 Subject: [PATCH 03/21] SIENTIAPDE-1478 Refactor metrics and imports across multiple files - Removed unused import of CORE_LABELS in metrics.py. - Updated import statements in api.py for consistency. - Cleaned up import order in worker.py for better readability. - Added missing newline at the end of metrics.py. - Adjusted formatting in core_scouter.py to ensure proper syntax. - Removed redundant 'on_conflict' and 'unique_columns' keys from test cases in test_core_scouter.py for clarity. --- scouter/activities/api.py | 2 +- scouter/metrics.py | 5 ++--- scouter/worker/worker.py | 10 +++++----- scouter/workflow/sub_workflows/core_scouter.py | 2 +- tests/workflow/sub_workflows/test_core_scouter.py | 6 ------ 5 files changed, 9 insertions(+), 16 deletions(-) diff --git a/scouter/activities/api.py b/scouter/activities/api.py index 1b1c282..045b415 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -9,8 +9,8 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.logger import Logger from sientia_do.observability.metrics_controller import MetricsController from sientia_do.observability.sientia_monitoring import SientiaMonitoring - from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ from sientia_do.repository.pi_web_api_client import PIWebAPIClient + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ class API(SientiaMonitoring): diff --git a/scouter/metrics.py b/scouter/metrics.py index a89be12..b25fa46 100644 --- a/scouter/metrics.py +++ b/scouter/metrics.py @@ -1,5 +1,4 @@ -from prometheus_client import Counter, Gauge, Histogram -from sientia_do.observability.metrics import CORE_LABELS as SIENTIA_CORE_LABELS +from prometheus_client import Counter, Gauge # Application health and status metrics APP_UP = Gauge( @@ -23,4 +22,4 @@ TAG_CHANGES_MONITOR = Gauge( 'scouter_tag_changes_monitor', 'Current value change of each tag', [*CORE_LABELS, 'tag_name'], -) \ No newline at end of file +) diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index d518f66..2ee4ee6 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -9,17 +9,17 @@ with workflow.unsafe.imports_passed_through(): import sys from prometheus_client import start_http_server - from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler - from sientia_do.observability.logger import get_logger - - from scouter import metrics - from scouter.activities.activities import Activities from sientia_do.connectors_config import ( build_api_config, build_mongodb_config, build_postgres_config, build_redis_config, ) + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.observability.logger import get_logger + + from scouter import metrics + from scouter.activities.activities import Activities from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter from scouter.workflow.scouter import Scouter from scouter.workflow.sub_workflows.core_scouter import CoreScouter diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py index 13b7942..e395979 100644 --- a/scouter/workflow/sub_workflows/core_scouter.py +++ b/scouter/workflow/sub_workflows/core_scouter.py @@ -109,7 +109,7 @@ class CoreScouter: 'schema': input_data['schema'], 'table_name': input_data['table_name'], 'data': held_data, - 'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ} + 'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ}, }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=60), diff --git a/tests/workflow/sub_workflows/test_core_scouter.py b/tests/workflow/sub_workflows/test_core_scouter.py index 608985e..91e0fdd 100644 --- a/tests/workflow/sub_workflows/test_core_scouter.py +++ b/tests/workflow/sub_workflows/test_core_scouter.py @@ -118,8 +118,6 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter): 'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ, }, - 'on_conflict': 'ignore', - 'unique_columns': ['model_id', 'timestamp', 'variable'], }, retry_policy=ANY, start_to_close_timeout=ANY, @@ -305,8 +303,6 @@ async def test_core_scouter_workflow_with_zero_affected_rows(mock_workflow, core 'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ, }, - 'on_conflict': 'ignore', - 'unique_columns': ['model_id', 'timestamp', 'variable'], }, retry_policy=ANY, start_to_close_timeout=ANY, @@ -377,8 +373,6 @@ async def test_core_scouter_workflow_without_debug_data_package(mock_workflow, c 'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ, }, - 'on_conflict': 'ignore', - 'unique_columns': ['model_id', 'timestamp', 'variable'], }, retry_policy=ANY, start_to_close_timeout=ANY, From f94ee12324785d9c212dd9f19ef23ddb7329b9ba Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 8 Jan 2026 16:25:07 -0300 Subject: [PATCH 04/21] SIENTIAPDE-1478 Update import path for connectors_config in worker.py to reflect new repository structure --- scouter/worker/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 2ee4ee6..0b90965 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -9,7 +9,7 @@ with workflow.unsafe.imports_passed_through(): import sys from prometheus_client import start_http_server - from sientia_do.connectors_config import ( + from sientia_do.utils.connectors_config import ( build_api_config, build_mongodb_config, build_postgres_config, From b75f4994362e92aeb569339b5352c10e6ebd31b9 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 8 Jan 2026 16:25:40 -0300 Subject: [PATCH 05/21] SIENTIAPDE-1478 Update sientia-dataops-library version in requirements.txt from 1.7.1 to 1.8.0 for improved functionality. --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3ffda95..d18dd00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ psycopg2-binary sqlalchemy redis pymongo -#git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.7.1 -/home/grezewave/Documents/projects/sientia/sientia-dataops-library/ +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.8.0 prometheus-client pycurl \ No newline at end of file From 4352c67f3a13d37e5255d81319f89bb7e2bffaa3 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 9 Jan 2026 09:49:55 -0300 Subject: [PATCH 06/21] SIENTIAPDE-1478 Enhance PI Web API Scouter Workflow and Documentation - Added support for PI Web API data ingestion, including real-time and historical data retrieval. - Implemented timestamp normalization to ensure consistency across records. - Updated README.md to reflect new features and detailed workflow execution flow. - Enhanced API class with comprehensive error handling and cleanup operations. - Improved CoreScouter workflow with early exit conditions and detailed processing steps. - Updated configuration parameters for better clarity and usability. --- README.md | 112 +++++++++++++++++- scouter/activities/api.py | 11 +- scouter/workflow/pi_web_api_scouter.py | 21 ++-- scouter/workflow/scouter.py | 8 +- .../workflow/sub_workflows/core_scouter.py | 13 +- 5 files changed, 150 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 0cad971..6e3a682 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A high-performance, scalable data processing and ML model orchestration system b ## Features ### Core Functionality -- **Multi-Source Data Ingestion**: Support for Kafka topics, direct OPC server access, and real-time triggers +- **Multi-Source Data Ingestion**: Support for Kafka topics, direct OPC server access, PI Web API endpoints, and real-time triggers - **Temporal Workflow Orchestration**: Robust workflow management with automatic retry policies and fault tolerance - **Data Quality Gates**: Configurable filtering for null values, out-of-bounds data, and custom validation rules - **Time-Series Aggregation**: Flexible aggregation functions (average, median, max, min, latest) with configurable parameters @@ -102,9 +102,14 @@ The **CoreScouter** workflow implements the core data processing pipeline for in 1. **Data Quality Gate**: Applies configured filters (null values, out-of-bounds, custom rules) 2. **Data Aggregation**: Groups data by tag and name, applies aggregation functions 3. **Data Grouping**: Organizes data and stores temporarily in Redis with TTL -4. **Data Export**: Persists processed data to PostgreSQL database +4. **Data Export**: Persists processed data to PostgreSQL database with timestamp conversion 5. **Metrics Recording**: Writes processing metrics for operational visibility +**Note**: The data export step uses timestamp conversion to ensure consistent datetime +formatting. The export operation receives the schema, table name, data, and timestamp +conversion configuration. Conflict resolution and unique column constraints are handled +by the underlying PostgreSQL activity implementation. + #### Aggregation Functions - **`lts`**: Latest value (most recent data point) - **`avg`**: Average of all values in the group @@ -171,6 +176,102 @@ When `debug_data_package` is set to `true`, the workflow stores both raw and pro - Validating data transformations - Auditing data quality gate decisions + +### 3. PI Web API Scouter Workflow (`pi_web_api_scouter.py`) + +The **PI Web API Scouter** workflow serves as the entry point for PI Web API data processing pipelines. Unlike the standard Scouter workflow that loads data from MongoDB collections, this workflow directly queries PI Web API endpoints to retrieve tag values and processes them for downstream use. + +#### Purpose +- **Direct API Ingestion**: Retrieves data directly from PI Web API endpoints +- **Real-time Data Processing**: Supports real-time and historical data retrieval +- **Data Normalization**: Normalizes timestamps to ensure consistency across records +- **Workflow Orchestration**: Delegates data processing to the CoreScouter workflow +- **Error Handling**: Comprehensive error handling with retry policies + +#### Execution Flow +1. **Tag Value Retrieval**: Retrieves tag values from PI Web API using configured WebIds and time periods +2. **Data Normalization**: Normalizes timestamps to ensure all records in a batch share the same timestamp value +3. **Data Validation**: Validates retrieved data and handles empty responses +4. **Data Processing**: Delegates data processing to the CoreScouter child workflow + +**Note**: The timestamp normalization process converts all timestamps to string format and then sets all records to the maximum timestamp value (lexicographically) found in the dataset. This ensures consistency across all records in a single batch. + +#### Key Features +- **Configurable Time Periods**: Supports flexible time period configurations (e.g., '*-1d', '*-1h') +- **Data Point Limits**: Configurable maximum data points per tag via `max_count` parameter +- **Timeout Management**: Configurable API request timeouts for reliable operation +- **Empty Data Handling**: Gracefully handles empty responses without processing +- **Standardized Processing**: Uses CoreScouter for consistent data quality and export operations + +#### Input Parameters +```json +{ + "model_name": "pi_sensors", + "model_id": "pi_001", + "schedule_name": "hourly_pi_collection", + "pi_web_api_query": { + "endpoint": "/streamsets/recorded", + "period": "*-1d", + "max_count": 10, + "api_timeout": 30 + }, + "model_tags": { + "Temperature": { + "webid": "F1AbCdEfGhIjKlMnOpQrStUvWxYz", + "aggr_function": "avg", + "data_range": [-50, 150] + }, + "Pressure": { + "webid": "F2AbCdEfGhIjKlMnOpQrStUvWxYz", + "aggr_function": "max", + "data_range": [0, 100] + } + }, + "trigger_laborious": false, + "filters": { + "OUT_OF_BOUNDS_FILTER": {"policy": "DISCARD"}, + "NULL_VALUES_FILTER": {"policy": "DISCARD"} + }, + "schema": "sensor_data", + "table_name": "pi_readings", + "retention_time": 3600, + "fill_missing_tags": false, + "debug_data_package": false +} +``` + +**PI Web API Query Parameters:** +- `endpoint` (str): PI Web API endpoint path (e.g., '/streamsets/recorded') +- `period` (str): Time period configuration (e.g., '*-1d' for last day, '*-1h' for last hour) +- `max_count` (int, optional): Maximum data points per tag. Defaults to 1 +- `api_timeout` (int): Request timeout in seconds for PI Web API calls + +**Model Tags Configuration:** +- `webid` (str): PI Web API WebId for the tag +- `aggr_function` (str): Aggregation method (avg, mdn, max, min, lts) +- `data_range` (list[int]): [min, max] values for data validation + +#### Architecture + +```mermaid +flowchart LR + A[1. get_tag_values] --> B{data empty?} + B -->|yes| C[Exit] + B -->|no| D[2. core_scouter 🔃] + + A -.-> PI_API[(PI Web API)] + D -.-> CoreScouter[CoreScouter Workflow] +``` + +#### Data Normalization + +The `get_tag_values` activity normalizes timestamps to ensure consistency: +1. Converts all timestamps to string format using the configured datetime format +2. Identifies the maximum timestamp value (lexicographically) in the dataset +3. Sets all records to use this normalized timestamp value + +This normalization ensures that all records in a single batch share the same timestamp, which is useful for batch processing and data consistency in downstream operations. + ## 📋 Prerequisites - Python 3.11+ @@ -179,6 +280,7 @@ When `debug_data_package` is set to `true`, the workflow stores both raw and pro - Redis server - MongoDB server - Kafka cluster (for data ingestion) +- PI Web API server (for PI Web API Scouter workflow) **Note**: External dependencies must be available either through: - Kubernetes cluster deployment @@ -365,6 +467,9 @@ The Scouter system exposes comprehensive Prometheus metrics: | `REDIS_PORT` | Redis port | `6379` | Yes | | `MONGODB_URL` | MongoDB connection URI | `localhost:27017` | Yes | | `KAFKA_BOOTSTRAP_SERVERS` | Kafka broker addresses | `localhost:9092` | No | +| `PI_WEB_API_BASE_URL` | PI Web API base URL | - | Yes (for PI Web API Scouter) | +| `PI_WEB_API_AUTH_TYPE` | PI Web API authentication type ('basic' or 'bearer') | - | Yes (for PI Web API Scouter) | +| `PI_WEB_API_AUTH_TOKEN` | PI Web API authentication token | - | Yes (for PI Web API Scouter) | | `HTTP_METRICS_PORT` | Prometheus metrics port | `9090` | No | | `HTTP_SDK_METRICS_PORT` | Temporal SDK metrics port | `9091` | No | | `PROJECT_NAME` | Project identifier for notifications | `scouter` | No | @@ -460,11 +565,13 @@ MongoDB pipeline configuration: scouter/ ├── activities/ # Temporal activity implementations │ ├── activities.py # Main activities orchestrator +│ ├── api.py # PI Web API operations (tag value retrieval) │ ├── redis.py # Redis operations (caching, timestamps) │ ├── gates.py # Data quality gates and filtering │ └── mongodb.py # MongoDB operations (data loading) ├── workflow/ # Temporal workflow definitions │ ├── scouter.py # Main data ingestion workflow +│ ├── pi_web_api_scouter.py # PI Web API data ingestion workflow │ └── sub_workflows/ # Sub-workflow implementations │ └── core_scouter.py # Core data processing workflow ├── worker/ # Worker implementation @@ -484,6 +591,7 @@ The Activities class combines multiple service classes through multiple inherita - **Redis**: Timestamp management, data caching, and temporary storage - **Gates**: Data quality validation and filtering logic - **MongoDB**: Data loading from raw collections +- **API**: PI Web API tag value retrieval and data normalization All activities support: - Comprehensive logging and error handling diff --git a/scouter/activities/api.py b/scouter/activities/api.py index 045b415..eb2806a 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -63,6 +63,10 @@ class API(SientiaMonitoring): def close(self) -> None: """ Close the PI Web API client and shutdown monitoring services. + + This method performs cleanup operations: + - Closes the PI Web API client connection + - Shuts down SientiaMonitoring services (metrics, notifications) """ self.pi_web_api_client.close() SientiaMonitoring.shutdown(self) @@ -76,6 +80,11 @@ class API(SientiaMonitoring): for a set of configured tags. It returns the data as a list of dictionaries suitable for further processing in the workflow. + The timestamps are normalized to ensure consistency across all records in the + response. After converting timestamps to string format, all timestamps are + set to the maximum timestamp value (lexicographically) found in the dataset. + This ensures all records in a single batch share the same timestamp value. + Args: input_data (dict[str, Any]): Activity input parameters. Required fields: @@ -88,7 +97,7 @@ class API(SientiaMonitoring): Returns: list[dict]: List of data records, each containing: - - timestamp: Data point timestamp + - timestamp: Normalized timestamp string (all records share the same value) - name: Tag name - value: Numeric value - tag: WebId diff --git a/scouter/workflow/pi_web_api_scouter.py b/scouter/workflow/pi_web_api_scouter.py index 42cbce6..acaa1dc 100644 --- a/scouter/workflow/pi_web_api_scouter.py +++ b/scouter/workflow/pi_web_api_scouter.py @@ -33,26 +33,31 @@ class PIWebAPIScouter: This method orchestrates the complete data ingestion process from PI Web API: 1. Retrieves tag values from PI Web API using configured WebIds - 2. Validates and normalizes the retrieved data + 2. Validates and normalizes the retrieved data (timestamps are normalized) 3. Delegates data processing to the CoreScouter workflow + If no data is retrieved from the PI Web API, the workflow exits early without + invoking the CoreScouter workflow. + Args: input_data (dict[str, Any]): Configuration and parameters for the workflow execution. Required fields: + - schedule_name (str): Unique identifier for the data collection schedule - model_name (str): Name of the data model being processed - model_id (str): Unique identifier for the data model - - schedule_name (str): Unique identifier for the data collection schedule - - endpoint (str): PI Web API endpoint path (e.g., '/streamsets/recorded') - - web_ids (dict[str, str | None]): Mapping of tag names to WebIds - - period (dict[str, str]): Time period configuration with 'start_time' - - api_timeout (int): Request timeout in seconds for PI Web API calls - - max_count (int, optional): Maximum data points per tag. Defaults to 1 + - pi_web_api_query (dict[str, Any]): PI Web API query configuration containing: + - endpoint (str): PI Web API endpoint path (e.g., '/streamsets/recorded') + - period (str): Time period configuration (e.g., '*-1d', '*-1h') + - api_timeout (int): Request timeout in seconds for PI Web API calls + - max_count (int, optional): Maximum data points per tag. Defaults to 1 - trigger_laborious (bool): Flag to enable intensive data processing - filters (dict[str, str]): Data quality filters configuration - schema (str): Target database schema for data export - table_name (str): Target table name for data export - retention_time (int): Data retention period in Redis (seconds) - - model_tags (dict[str, Any]): Tag-specific configuration including: + - model_tags (dict[str, Any]): Tag-specific configuration mapping tag names + to WebIds and processing rules, including: + - webid (str): PI Web API WebId for the tag - data_range: [min, max] values for data validation - aggr_function: Aggregation method (avg, mdn, max, min, lts) - frequency: Data collection frequency in milliseconds diff --git a/scouter/workflow/scouter.py b/scouter/workflow/scouter.py index 52393e2..3ca3bbe 100644 --- a/scouter/workflow/scouter.py +++ b/scouter/workflow/scouter.py @@ -32,10 +32,14 @@ class Scouter: This method orchestrates the complete data ingestion process: 1. Retrieves the last processed timestamp from Redis - 2. Loads new data from MongoDB since the last timestamp - 3. Updates the last processed timestamp + 2. Loads new data from MongoDB since the last timestamp using collection name + format: `raw_{schedule_name}` + 3. Updates the last processed timestamp with the most recent data point 4. Delegates data processing to the CoreScouter workflow + If no new data is found in MongoDB, the workflow exits early without updating + the timestamp or invoking the CoreScouter workflow. + Args: input_data (dict[str, Any]): Configuration and parameters for the workflow execution. Required fields: diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py index e395979..3aac96b 100644 --- a/scouter/workflow/sub_workflows/core_scouter.py +++ b/scouter/workflow/sub_workflows/core_scouter.py @@ -23,7 +23,8 @@ class CoreScouter: - Metrics collection and monitoring The workflow is designed for high-throughput data processing with configurable - quality gates and aggregation strategies. + quality gates and aggregation strategies. It is typically invoked as a child + workflow by parent workflows such as Scouter or PIWebAPIScouter. """ @workflow.run @@ -35,9 +36,14 @@ class CoreScouter: 1. Data Quality Gate: Applies configurable filters for data validation 2. Data Aggregation: Groups and aggregates data using specified functions 3. Data Grouping: Organizes data by tags and applies retention policies - 4. Data Export: Persists processed data to PostgreSQL + 4. Data Export: Persists processed data to PostgreSQL with timestamp conversion 5. Metrics Collection: Records processing metrics for monitoring + The workflow implements early exit conditions: + - If held_data is empty after grouping, the workflow exits without exporting + - If data export results in zero or negative affected_rows, the workflow exits + without writing metrics or storing debug packages + Args: input_data (dict[str, Any]): Complete workflow configuration and data. Required fields: @@ -53,6 +59,9 @@ class CoreScouter: - table_name (str): Target database table - retention_time (int): Redis data retention period (seconds) - model_tags (dict[str, Any]): Tag-specific processing rules + - fill_missing_tags (bool): Enable filling of missing tag values + - debug_data_package (bool, optional): Store data packages for debugging. + When True, stores both raw and processed data in MongoDB for debugging Returns: None: This workflow processes data but doesn't return results From 18293077044f52db8e7852051a0f6d09dfd6e1e7 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 9 Jan 2026 12:35:39 -0300 Subject: [PATCH 07/21] SIENTIAPDE-1478 Refactor import statements in worker.py for improved readability and consistency --- scouter/worker/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 0b90965..19974c9 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -1,13 +1,13 @@ from temporalio import client, workflow from temporalio.runtime import PrometheusConfig, Runtime, TelemetryConfig -from scouter.worker.prepare_worker import prepare_worker - with workflow.unsafe.imports_passed_through(): import asyncio import os import sys + from scouter.worker.prepare_worker import prepare_worker + from prometheus_client import start_http_server from sientia_do.utils.connectors_config import ( build_api_config, From ea081ca348bf08aa344af896b07a76d0acd559ee Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 9 Jan 2026 12:38:46 -0300 Subject: [PATCH 08/21] SIENTIAPDE-1478 Refactor import statements in worker.py to enhance readability and maintain consistency with the new repository structure. --- scouter/worker/worker.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 19974c9..33fe54a 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -6,20 +6,19 @@ with workflow.unsafe.imports_passed_through(): import os import sys - from scouter.worker.prepare_worker import prepare_worker - from prometheus_client import start_http_server + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.observability.logger import get_logger from sientia_do.utils.connectors_config import ( build_api_config, build_mongodb_config, build_postgres_config, build_redis_config, ) - from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler - from sientia_do.observability.logger import get_logger from scouter import metrics from scouter.activities.activities import Activities + from scouter.worker.prepare_worker import prepare_worker from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter from scouter.workflow.scouter import Scouter from scouter.workflow.sub_workflows.core_scouter import CoreScouter From f9fdc3f95af1803dd8d4a2e2ca033318009ab04e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 9 Jan 2026 12:52:32 -0300 Subject: [PATCH 09/21] SIENTIAPDE-1478 Update GITHUB_BRANCH in values.yaml to reflect the new fix/SIENTIAPDE-1478 branch. --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index 567c70c..47bfd0b 100644 --- a/values.yaml +++ b/values.yaml @@ -163,7 +163,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git" - name: GITHUB_BRANCH - value: "fix/SIENTIAPDE-1445" + value: "fix/SIENTIAPDE-1478" - name: PYTHON_APP value: "scouter.worker.worker" From 9ca7c43b2bfcdc4ab0a52239db4efd791162b53a Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 12 Jan 2026 12:27:26 -0300 Subject: [PATCH 10/21] SIENTIAPDE-1478 Refactor API class to rename 'timeout' parameter to 'request_timeout' for improved clarity and consistency. --- scouter/activities/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scouter/activities/api.py b/scouter/activities/api.py index eb2806a..9f75d88 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -122,7 +122,7 @@ class API(SientiaMonitoring): start_time=period, max_count=max_count, metadata=metadata, - timeout=api_timeout, + request_timeout=api_timeout, ) except Exception as e: From 0bc8b55a3dd80d2b98b517fc0745927acd12f479 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 12 Jan 2026 12:34:28 -0300 Subject: [PATCH 11/21] SIENTIAPDE-1478 Enhance API class logging by adding debug statement for latest values retrieval --- scouter/activities/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scouter/activities/api.py b/scouter/activities/api.py index 9f75d88..120d3d1 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -138,10 +138,12 @@ class API(SientiaMonitoring): latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ) + self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata) + # Normalize the package timestamp latest_values['timestamp'] = latest_values['timestamp'].max() - self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata) + self.info(f'Gathered {len(latest_values)} tag values', metadata=metadata) return latest_values.to_dict(orient='records') From efeb29414e981db32904acca9a020cc2e08f19f5 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 12 Jan 2026 12:38:59 -0300 Subject: [PATCH 12/21] SIENTIAPDE-1478 Refactor timestamp normalization in API class to handle NaN values correctly by dropping them before calculating the maximum timestamp. --- scouter/activities/api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scouter/activities/api.py b/scouter/activities/api.py index 120d3d1..7d2cf47 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -136,13 +136,14 @@ class API(SientiaMonitoring): ) raise e + latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ) self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata) # Normalize the package timestamp - latest_values['timestamp'] = latest_values['timestamp'].max() - + valid_timestamp_values = latest_values['timestamp'].dropna() + latest_values['timestamp'] = valid_timestamp_values.max() self.info(f'Gathered {len(latest_values)} tag values', metadata=metadata) From dc38309348365adfcc0f4d60b5402b0402fdda6f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 13 Jan 2026 15:53:25 -0300 Subject: [PATCH 13/21] SIENTIAPDE-1478 Update tests.ipynb and prepare_worker.py - Reset execution counts in tests.ipynb for reproducibility. - Removed error outputs and added new tags in TAG_NAMES for enhanced data retrieval. - Introduced JSON export of web_ids to 'web_ids.json' for better data management. - Added comments in prepare_worker.py to clarify worker configuration parameters. --- scouter/worker/prepare_worker.py | 2 + scouter/worker/worker_parameters.md | 113 +++++++++++++++++++++++++++ tests.ipynb | 117 ++++++++++++++++------------ 3 files changed, 181 insertions(+), 51 deletions(-) create mode 100644 scouter/worker/worker_parameters.md diff --git a/scouter/worker/prepare_worker.py b/scouter/worker/prepare_worker.py index c8b68f2..09eb902 100644 --- a/scouter/worker/prepare_worker.py +++ b/scouter/worker/prepare_worker.py @@ -7,6 +7,8 @@ from sientia_do.observability.logger import Logger from temporalio.client import Client from temporalio.worker import PollerBehaviorAutoscaling, Worker +# Worker configuration parameters with default values +# See worker_parameters.md for detailed documentation parameters = [ ('MAX_CONCURRENT_WORKFLOW_TASKS', '200'), ('MAX_CONCURRENT_ACTIVITIES', '200'), diff --git a/scouter/worker/worker_parameters.md b/scouter/worker/worker_parameters.md new file mode 100644 index 0000000..ca2c7cd --- /dev/null +++ b/scouter/worker/worker_parameters.md @@ -0,0 +1,113 @@ +# Worker Parameters Documentation + +This document explains each configuration parameter used in the `prepare_worker.py` file for configuring Temporal workers. + +## Overview + +All parameters can be configured via environment variables using the pattern: `{WORKFLOW_NAME}_{PARAMETER_NAME}`. If not set, default values are used as specified below. + +## Concurrency Parameters + +### MAX_CONCURRENT_WORKFLOW_TASKS +- **Default**: `200` +- **Description**: Maximum number of concurrent workflow tasks that can be processed simultaneously by the worker. This controls how many workflow executions can be actively running at the same time. +- **Usage**: Set via `max_concurrent_workflow_tasks` in the Worker configuration. +- **Impact**: Higher values allow more workflows to run concurrently but consume more resources. Lower values provide better resource control but may limit throughput. + +### MAX_CONCURRENT_ACTIVITIES +- **Default**: `200` +- **Description**: Maximum number of concurrent activity tasks that can be executed simultaneously by the worker. Activities are the actual work units that perform business logic. +- **Usage**: Set via `max_concurrent_activities` in the Worker configuration. +- **Impact**: Controls the parallelism of activity execution. Higher values increase throughput but require more system resources (CPU, memory, network connections). + +### MAX_CONCURRENT_LOCAL_ACTIVITIES +- **Default**: `200` +- **Description**: Maximum number of concurrent local activity tasks that can be executed simultaneously. Local activities run in the same process as the workflow, without requiring a separate activity worker. +- **Usage**: Set via `max_concurrent_local_activities` in the Worker configuration. +- **Impact**: Similar to regular activities, but local activities have lower latency and overhead since they don't require network round-trips. Useful for lightweight operations. + +## Caching Parameters + +### MAX_CACHED_WORKFLOWS +- **Default**: `200` +- **Description**: Maximum number of workflow instances that can be cached in memory by the worker. Cached workflows allow faster resumption of execution without reloading state. +- **Usage**: Set via `max_cached_workflows` in the Worker configuration. +- **Impact**: Higher values improve performance for frequently accessed workflows but consume more memory. Lower values reduce memory usage but may require more frequent state reloads. + +## Understanding Pollers in Temporal + +**Pollers** are components of Temporal Workers that continuously request tasks from the Temporal service's Task Queues via synchronous RPCs. There are separate pollers for workflow tasks and activity tasks. + +### How Pollers Work + +Pollers send requests to the Temporal service to retrieve tasks from Task Queues. When a task is available, the poller retrieves it and the Worker processes it using registered Workflow or Activity handlers. This architecture provides: +- **Load Balancing**: Workers only poll when they have capacity, distributing load across multiple processes +- **Fault Tolerance**: Tasks persist in queues if a Worker fails, allowing recovery +- **Task Routing**: Tasks can be routed to specific Worker processes + +### Autoscaling Poller Behavior + +Temporal supports autoscaling that dynamically adjusts the number of concurrent pollers based on workload. The system scales up during high load and down during low load, maintaining a baseline for responsiveness. Autoscaling is configured with `minimum`, `initial`, and `maximum` parameters that define the scaling bounds. + +## Workflow Poller Behavior (Autoscaling) + +These parameters control the autoscaling behavior of the workflow task poller, which retrieves workflow tasks from the Temporal server. + +### WORKFLOW_POLLER_BEHAVIUR_MINIMUM +- **Default**: `10` +- **Description**: Minimum number of concurrent pollers for workflow tasks. The poller count will never go below this value. +- **Usage**: Set via `minimum` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`. +- **Impact**: Ensures a baseline level of polling activity even during low load periods. + +### WORKFLOW_POLLER_BEHAVIUR_INITIAL +- **Default**: `100` +- **Description**: Initial number of concurrent pollers for workflow tasks when the worker starts. +- **Usage**: Set via `initial` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`. +- **Impact**: Determines the starting point for poller scaling. Higher values provide faster initial task acquisition but consume more resources. + +### WORKFLOW_POLLER_BEHAVIUR_MAXIMUM +- **Default**: `200` +- **Description**: Maximum number of concurrent pollers allowed for workflow tasks. The poller count will not exceed this value even under high load. +- **Usage**: Set via `maximum` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`. +- **Impact**: Caps the resource consumption for workflow task polling. Prevents excessive polling that could overwhelm the Temporal server or worker. + +## Activity Poller Behavior (Autoscaling) + +These parameters control the autoscaling behavior of the activity task poller, which retrieves activity tasks from the Temporal server. + +### ACTIVITY_POLLER_BEHAVIUR_MINIMUM +- **Default**: `10` +- **Description**: Minimum number of concurrent pollers for activity tasks. The poller count will never go below this value. +- **Usage**: Set via `minimum` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`. +- **Impact**: Ensures a baseline level of polling activity even during low load periods. + +### ACTIVITY_POLLER_BEHAVIUR_INITIAL +- **Default**: `100` +- **Description**: Initial number of concurrent pollers for activity tasks when the worker starts. +- **Usage**: Set via `initial` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`. +- **Impact**: Determines the starting point for poller scaling. Higher values provide faster initial task acquisition but consume more resources. + +### ACTIVITY_POLLER_BEHAVIUR_MAXIMUM +- **Default**: `200` +- **Description**: Maximum number of concurrent pollers allowed for activity tasks. The poller count will not exceed this value even under high load. +- **Usage**: Set via `maximum` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`. +- **Impact**: Caps the resource consumption for activity task polling. Prevents excessive polling that could overwhelm the Temporal server or worker. + +## Configuration Example + +To override these parameters, set environment variables using the pattern: +``` +{WORKFLOW_NAME}_{PARAMETER_NAME}={value} +``` + +For example, if your workflow is named `ScouterWorkflow`: +```bash +SCOUTERWORKFLOW_MAX_CONCURRENT_ACTIVITIES=500 +SCOUTERWORKFLOW_WORKFLOW_POLLER_BEHAVIUR_MAXIMUM=300 +``` + +## Notes + +- All parameter values are converted to integers before use. +- The autoscaling poller behavior dynamically adjusts the number of pollers between the minimum and maximum values based on workload. +- These parameters should be tuned based on your specific workload characteristics, available resources, and performance requirements. diff --git a/tests.ipynb b/tests.ipynb index 79f937d..9df4097 100644 --- a/tests.ipynb +++ b/tests.ipynb @@ -168,57 +168,67 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 1, "id": "72af4236", "metadata": {}, - "outputs": [ - { - "ename": "KeyError", - "evalue": "'Items'", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mKeyError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[17]\u001b[39m\u001b[32m, line 42\u001b[39m\n\u001b[32m 39\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m tag \u001b[38;5;129;01min\u001b[39;00m TAG_NAMES:\n\u001b[32m 40\u001b[39m response = requests.get(url.replace(\u001b[33m'\u001b[39m\u001b[38;5;132;01m{tag}\u001b[39;00m\u001b[33m'\u001b[39m, tag), headers=headers).json()\n\u001b[32m 41\u001b[39m web_ids[tag] = {\n\u001b[32m---> \u001b[39m\u001b[32m42\u001b[39m \u001b[33m'\u001b[39m\u001b[33mwebid\u001b[39m\u001b[33m'\u001b[39m: \u001b[43mresponse\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mItems\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m]\u001b[49m[\u001b[32m0\u001b[39m][\u001b[33m'\u001b[39m\u001b[33mWebId\u001b[39m\u001b[33m'\u001b[39m],\n\u001b[32m 43\u001b[39m \u001b[33m'\u001b[39m\u001b[33maggr_func\u001b[39m\u001b[33m'\u001b[39m: \u001b[33m'\u001b[39m\u001b[33mlts\u001b[39m\u001b[33m'\u001b[39m,\n\u001b[32m 44\u001b[39m \u001b[33m'\u001b[39m\u001b[33mdata_range\u001b[39m\u001b[33m'\u001b[39m: [-\u001b[32m100000\u001b[39m, \u001b[32m100000\u001b[39m],\n\u001b[32m 45\u001b[39m }\n\u001b[32m 46\u001b[39m sleep(\u001b[32m0.5\u001b[39m)\n", - "\u001b[31mKeyError\u001b[39m: 'Items'" - ] - } - ], + "outputs": [], "source": [ "import requests\n", "from time import sleep\n", "\n", "# Obter web id das seguintes tags:\n", - "TAG_NAMES = [\n", + "TAG_NAMES = [\n", " \"CI-W3A05F1\",\n", - " \"CI-W3W03S1\", \"CI-W3W03I1\", \"CI-W3K01T1\", \"CI-W3W01A3\",\n", - " \"CI-W3W01A2\", \"CI-W3W01A1\", \"CI-J3P01T1A\", \"CI-W3A50T1\", \"CI-W3A55T1\",\n", - " \"CI-W3A55P1\", \"CI-W3V33P1\", \"CI-W3E01F1\", \"CI-W3A50A3\", \"CI-W3A50A2\",\n", - " \"CI-W3A50A1\", \"CI-W3A50P1\", \"CI-W3W01P1\", \"CI-W3A71P1\", \"CI-W3W01P2\",\n", - " \"CI-W3A71P2\", \"CI-W3A71P3\", \"CI-J3J01S1\", \"CI-W3P17S1\", \"CI-J3P03S1\",\n", - " \"CI-W3K01S1\", \"CI-W3K14P1\", \"CI-W3K01T4\", \"CI-W3K01T2\", \n", - "\n", - " \"CI-W3FARCI_FSC\",\n", - " \"CI-W3FARCI_MA\",\n", - " \"CI-W3FARCI_MS\",\n", - " \"CI-W3FARCI_P100\",\n", - " \"CI-W3FARCI_p170\",\n", - "\n", - " \"CI-W3CLK_C3S\",\n", - " \"CI-W3CLK_C3S_EXP\",\n", - " \"CI-W3CLK_C3S_MD_EXP\",\n", - " \"CI-W3CLK_C3S_MD_PETRO\",\n", - " \"CI-W3V04P3\", \"CI-W3V04P1\",\n", - " \"CI-W3W01G1\"\n", + " \"CI-W3W03S1\",\n", + " \"CI-W3W03I1\",\n", + " \"CI-W3K01T1\",\n", + " \"CI-W3W01A3\",\n", + " \"CI-W3W01A2\",\n", + " \"CI-W3W01A1\",\n", + " \"CI-J3P01T1A\",\n", + " \"CI-W3A50T1\",\n", + " \"CI-W3A55T1\",\n", + " \"CI-W3A55P1\",\n", + " \"CI-W3V33P1\",\n", + " \"CI-W3E01F1\",\n", + " \"CI-W3A50A3\",\n", + " \"CI-W3A50A2\",\n", + " \"CI-W3A50A1\",\n", + " \"CI-W3A50P1\",\n", + " \"CI-W3W01P1\",\n", + " \"CI-W3A71P1\",\n", + " \"CI-W3W01P2\",\n", + " \"CI-W3A71P2\",\n", + " \"CI-W3A71P3\",\n", + " \"CI-J3J01S1\",\n", + " \"CI-W3P17S1\",\n", + " \"CI-J3P03S1\",\n", + " \"CI-W3K01S1\",\n", + " \"CI-W3K14P1\",\n", + " \"CI-W3K01T4\",\n", + " \"CI-W3K01T2\",\n", + " \"CI-W3A65_SO3\",\n", + " \"CI-W3A65_CL\",\n", + " \"CI-W3_C3S\",\n", + " \"CI-W3_MS\",\n", + " \"CI-W3_MA\",\n", + " \"CI-W3_PL\",\n", + " \"CI-W3_CAO\",\n", + " \"CI-W3V04P3\",\n", + " \"CI-W3V04P1\",\n", + " \"CI-W3W01G1\",\n", + " \"CI-W3V21F1\",\n", + " \"CI-W3V21P1\",\n", + " \"CI-W3V30F1\",\n", + " \"CI-W3V33P1\"\n", "]\n", - "\n", "url = 'https://pivision.votorantimcimentos.com/piwebapi/dataservers/F1DS-7fYgsRTtUOa7V9NIwSujAUElIQVZD/points?namefilter={tag}'\n", "\n", "headers = {\n", " 'Content-Type': 'application/json',\n", " 'Accept': 'application/json',\n", " 'X-Requested-With': 'piwebapistreams', # Header recomendado pelo PI Web API\n", - " 'Authorization': \"\"\n", + " 'Authorization': \"Basic dmlkX3ZjbmV0XHN2Yy5waW9zaS5wcmQud2ViYXBpOlN2Y1ByRFdlQkBQaQ==\"\n", "}\n", "\n", "web_ids = {}\n", @@ -235,7 +245,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "55793801", "metadata": {}, "outputs": [ @@ -329,31 +339,25 @@ " 'CI-W3K01T2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAlVQAAAUElIQVZDXENJLVczSzAxVDI',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3FARCI_FSC': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAgVQAAAUElIQVZDXENJLVczRkFSQ0lfRlND',\n", + " 'CI-W3A65_SO3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAjFQAAAUElIQVZDXENJLVczQTY1X1NPMw',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3FARCI_MA': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAg1QAAAUElIQVZDXENJLVczRkFSQ0lfTUE',\n", + " 'CI-W3A65_CL': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAf1QAAAUElIQVZDXENJLVczQTY1X0NM',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3FARCI_MS': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAhVQAAAUElIQVZDXENJLVczRkFSQ0lfTVM',\n", + " 'CI-W3_C3S': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA5FMAAAUElIQVZDXENJLVczX0MzUw',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3FARCI_P100': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAiFQAAAUElIQVZDXENJLVczRkFSQ0lfUDEwMA',\n", + " 'CI-W3_MS': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujABlQAAAUElIQVZDXENJLVczX01T',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3FARCI_p170': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAiVQAAAUElIQVZDXENJLVczRkFSQ0lfUDE3MA',\n", + " 'CI-W3_MA': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA_VMAAAUElIQVZDXENJLVczX01B',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3CLK_C3S': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA4lMAAAUElIQVZDXENJLVczQ0xLX0MzUw',\n", + " 'CI-W3_PL': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAFVQAAAUElIQVZDXENJLVczX1BM',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3CLK_C3S_EXP': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA41MAAAUElIQVZDXENJLVczQ0xLX0MzU19FWFA',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3CLK_C3S_MD_EXP': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA5VMAAAUElIQVZDXENJLVczQ0xLX0MzU19NRF9FWFA',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3CLK_C3S_MD_PETRO': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA5lMAAAUElIQVZDXENJLVczQ0xLX0MzU19NRF9QRVRSTw',\n", + " 'CI-W3_CAO': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA61MAAAUElIQVZDXENJLVczX0NBTw',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", " 'CI-W3V04P3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAElUAAAUElIQVZDXENJLVczVjA0UDM',\n", @@ -364,15 +368,26 @@ " 'data_range': [-100000, 100000]},\n", " 'CI-W3W01G1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAOlUAAAUElIQVZDXENJLVczVzAxRzE',\n", " 'aggr_func': 'lts',\n", + " 'data_range': [-100000, 100000]},\n", + " 'CI-W3V21F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAQnYEAAUElIQVZDXENJLVczVjIxRjE',\n", + " 'aggr_func': 'lts',\n", + " 'data_range': [-100000, 100000]},\n", + " 'CI-W3V21P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAQHYEAAUElIQVZDXENJLVczVjIxUDE',\n", + " 'aggr_func': 'lts',\n", + " 'data_range': [-100000, 100000]},\n", + " 'CI-W3V30F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAIFUAAAUElIQVZDXENJLVczVjMwRjE',\n", + " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]}}" ] }, - "execution_count": 23, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "import json\n", + "json.dump(web_ids, open('web_ids.json', 'w'), indent=4)\n", "web_ids" ] }, From 9dab8da5b3fc5b351b7262f396e74fb74626cf70 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 19 Jan 2026 10:11:36 -0300 Subject: [PATCH 14/21] SIENTIAPDE-1478 Refactor API class to remove unnecessary blank line and improve code readability. Update test cases to rename 'timeout' parameter to 'request_timeout' for consistency. --- scouter/activities/api.py | 3 +-- tests/activities/test_api.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scouter/activities/api.py b/scouter/activities/api.py index 7d2cf47..4f8e4fd 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -136,7 +136,6 @@ class API(SientiaMonitoring): ) raise e - latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ) self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata) @@ -144,7 +143,7 @@ class API(SientiaMonitoring): # Normalize the package timestamp valid_timestamp_values = latest_values['timestamp'].dropna() latest_values['timestamp'] = valid_timestamp_values.max() - + self.info(f'Gathered {len(latest_values)} tag values', metadata=metadata) return latest_values.to_dict(orient='records') diff --git a/tests/activities/test_api.py b/tests/activities/test_api.py index 8bd21e8..977cd0d 100644 --- a/tests/activities/test_api.py +++ b/tests/activities/test_api.py @@ -141,7 +141,7 @@ async def test_get_tag_values_success(api_activity): start_time='*-1d', max_count=10, metadata=metadata['metadata'], - timeout=30, + request_timeout=30, ) assert len(result) == 3 @@ -195,7 +195,7 @@ async def test_get_tag_values_with_default_max_count(api_activity): start_time='*-1h', max_count=1, metadata=metadata['metadata'], - timeout=15, + request_timeout=15, ) assert len(result) == 1 From e64af1aa7c5552b31153a304d12b50a96e63aae4 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 21 Jan 2026 15:38:22 -0300 Subject: [PATCH 15/21] SIENTIAPDE-1478 Update tests.ipynb and API class in scouter module - Reset execution counts in tests.ipynb for reproducibility. - Removed unnecessary outputs and adjusted execution counts for clarity. - Enhanced API class to accept 'end_time' parameter for improved data retrieval flexibility. - Commented out timestamp formatting and normalization for future adjustments. --- README.md | 12 +- scouter/activities/api.py | 12 +- scouter/worker/prepare_worker.py | 24 +- scouter/worker/worker.py | 12 +- scouter/worker/worker_parameters.md | 14 +- tests.ipynb | 2712 ++++++++++++++++++++------- values.yaml | 12 +- 7 files changed, 2117 insertions(+), 681 deletions(-) diff --git a/README.md b/README.md index 6e3a682..739c2bd 100644 --- a/README.md +++ b/README.md @@ -492,16 +492,16 @@ The worker implements aggressive autoscaling policies for workflow and activity **Workflow Poller Behavior:** | Variable | Description | Default | |----------|-------------|---------| -| `WORKFLOW_POLLER_BEHAVIUR_MINIMUM` | Minimum workflow pollers | `10` | -| `WORKFLOW_POLLER_BEHAVIUR_INITIAL` | Initial workflow pollers | `100` | -| `WORKFLOW_POLLER_BEHAVIUR_MAXIMUM` | Maximum workflow pollers | `200` | +| `WORKFLOW_POLLER_BEHAVIOUR_MINIMUM` | Minimum workflow pollers | `10` | +| `WORKFLOW_POLLER_BEHAVIOUR_INITIAL` | Initial workflow pollers | `100` | +| `WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM` | Maximum workflow pollers | `200` | **Activity Poller Behavior:** | Variable | Description | Default | |----------|-------------|---------| -| `ACTIVITY_POLLER_BEHAVIUR_MINIMUM` | Minimum activity pollers | `10` | -| `ACTIVITY_POLLER_BEHAVIUR_INITIAL` | Initial activity pollers | `100` | -| `ACTIVITY_POLLER_BEHAVIUR_MAXIMUM` | Maximum activity pollers | `200` | +| `ACTIVITY_POLLER_BEHAVIOUR_MINIMUM` | Minimum activity pollers | `10` | +| `ACTIVITY_POLLER_BEHAVIOUR_INITIAL` | Initial activity pollers | `100` | +| `ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM` | Maximum activity pollers | `200` | ### Workflow Configuration diff --git a/scouter/activities/api.py b/scouter/activities/api.py index 4f8e4fd..530ff44 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -110,6 +110,7 @@ class API(SientiaMonitoring): endpoint = input_data['endpoint'] web_ids = input_data['web_ids'] period = input_data['period'] + end_time = input_data.get('end_time', '*') max_count = input_data.get('max_count', 1) api_timeout = input_data['api_timeout'] @@ -120,6 +121,7 @@ class API(SientiaMonitoring): endpoint=endpoint, web_ids=web_ids, start_time=period, + end_time=end_time, max_count=max_count, metadata=metadata, request_timeout=api_timeout, @@ -136,13 +138,13 @@ class API(SientiaMonitoring): ) raise e - latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ) + # latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ) - self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata) + # self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata) - # Normalize the package timestamp - valid_timestamp_values = latest_values['timestamp'].dropna() - latest_values['timestamp'] = valid_timestamp_values.max() + # # Normalize the package timestamp + # valid_timestamp_values = latest_values['timestamp'].dropna() + # latest_values['timestamp'] = valid_timestamp_values.max() self.info(f'Gathered {len(latest_values)} tag values', metadata=metadata) diff --git a/scouter/worker/prepare_worker.py b/scouter/worker/prepare_worker.py index 09eb902..b497c5d 100644 --- a/scouter/worker/prepare_worker.py +++ b/scouter/worker/prepare_worker.py @@ -14,12 +14,12 @@ parameters = [ ('MAX_CONCURRENT_ACTIVITIES', '200'), ('MAX_CONCURRENT_LOCAL_ACTIVITIES', '200'), ('MAX_CACHED_WORKFLOWS', '200'), - ('WORKFLOW_POLLER_BEHAVIUR_MINIMUM', '10'), - ('WORKFLOW_POLLER_BEHAVIUR_INITIAL', '100'), - ('WORKFLOW_POLLER_BEHAVIUR_MAXIMUM', '200'), - ('ACTIVITY_POLLER_BEHAVIUR_MINIMUM', '10'), - ('ACTIVITY_POLLER_BEHAVIUR_INITIAL', '100'), - ('ACTIVITY_POLLER_BEHAVIUR_MAXIMUM', '200'), + ('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10'), + ('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100'), + ('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200'), + ('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10'), + ('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100'), + ('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200'), ] @@ -62,13 +62,13 @@ def prepare_worker( ], max_cached_workflows=local_workflow_parameters['MAX_CACHED_WORKFLOWS'], workflow_task_poller_behavior=PollerBehaviorAutoscaling( - minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIUR_MINIMUM'], - initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIUR_INITIAL'], - maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIUR_MAXIMUM'], + minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MINIMUM'], + initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_INITIAL'], + maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM'], ), activity_task_poller_behavior=PollerBehaviorAutoscaling( - minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIUR_MINIMUM'], - initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIUR_INITIAL'], - maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIUR_MAXIMUM'], + minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MINIMUM'], + initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_INITIAL'], + maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM'], ), ) diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 33fe54a..379e627 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -39,13 +39,13 @@ MAX_CACHED_WORKFLOWS = int(os.getenv('MAX_CACHED_WORKFLOWS', '200')) # Temporal docs also recommends an autoscaling policy, with agrresive limits to prioritize latency over throughput. -WORKFLOW_POLLER_BEHAVIUR_MINIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIUR_MINIMUM', '10')) -WORKFLOW_POLLER_BEHAVIUR_INITIAL = int(os.getenv('WORKFLOW_POLLER_BEHAVIUR_INITIAL', '100')) -WORKFLOW_POLLER_BEHAVIUR_MAXIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIUR_MAXIMUM', '200')) +WORKFLOW_POLLER_BEHAVIOUR_MINIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10')) +WORKFLOW_POLLER_BEHAVIOUR_INITIAL = int(os.getenv('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100')) +WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200')) -ACTIVITY_POLLER_BEHAVIUR_MINIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIUR_MINIMUM', '10')) -ACTIVITY_POLLER_BEHAVIUR_INITIAL = int(os.getenv('ACTIVITY_POLLER_BEHAVIUR_INITIAL', '100')) -ACTIVITY_POLLER_BEHAVIUR_MAXIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIUR_MAXIMUM', '200')) +ACTIVITY_POLLER_BEHAVIOUR_MINIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10')) +ACTIVITY_POLLER_BEHAVIOUR_INITIAL = int(os.getenv('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100')) +ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200')) async def main(): diff --git a/scouter/worker/worker_parameters.md b/scouter/worker/worker_parameters.md index ca2c7cd..8b18148 100644 --- a/scouter/worker/worker_parameters.md +++ b/scouter/worker/worker_parameters.md @@ -53,19 +53,19 @@ Temporal supports autoscaling that dynamically adjusts the number of concurrent These parameters control the autoscaling behavior of the workflow task poller, which retrieves workflow tasks from the Temporal server. -### WORKFLOW_POLLER_BEHAVIUR_MINIMUM +### WORKFLOW_POLLER_BEHAVIOUR_MINIMUM - **Default**: `10` - **Description**: Minimum number of concurrent pollers for workflow tasks. The poller count will never go below this value. - **Usage**: Set via `minimum` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`. - **Impact**: Ensures a baseline level of polling activity even during low load periods. -### WORKFLOW_POLLER_BEHAVIUR_INITIAL +### WORKFLOW_POLLER_BEHAVIOUR_INITIAL - **Default**: `100` - **Description**: Initial number of concurrent pollers for workflow tasks when the worker starts. - **Usage**: Set via `initial` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`. - **Impact**: Determines the starting point for poller scaling. Higher values provide faster initial task acquisition but consume more resources. -### WORKFLOW_POLLER_BEHAVIUR_MAXIMUM +### WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM - **Default**: `200` - **Description**: Maximum number of concurrent pollers allowed for workflow tasks. The poller count will not exceed this value even under high load. - **Usage**: Set via `maximum` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`. @@ -75,19 +75,19 @@ These parameters control the autoscaling behavior of the workflow task poller, w These parameters control the autoscaling behavior of the activity task poller, which retrieves activity tasks from the Temporal server. -### ACTIVITY_POLLER_BEHAVIUR_MINIMUM +### ACTIVITY_POLLER_BEHAVIOUR_MINIMUM - **Default**: `10` - **Description**: Minimum number of concurrent pollers for activity tasks. The poller count will never go below this value. - **Usage**: Set via `minimum` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`. - **Impact**: Ensures a baseline level of polling activity even during low load periods. -### ACTIVITY_POLLER_BEHAVIUR_INITIAL +### ACTIVITY_POLLER_BEHAVIOUR_INITIAL - **Default**: `100` - **Description**: Initial number of concurrent pollers for activity tasks when the worker starts. - **Usage**: Set via `initial` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`. - **Impact**: Determines the starting point for poller scaling. Higher values provide faster initial task acquisition but consume more resources. -### ACTIVITY_POLLER_BEHAVIUR_MAXIMUM +### ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM - **Default**: `200` - **Description**: Maximum number of concurrent pollers allowed for activity tasks. The poller count will not exceed this value even under high load. - **Usage**: Set via `maximum` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`. @@ -103,7 +103,7 @@ To override these parameters, set environment variables using the pattern: For example, if your workflow is named `ScouterWorkflow`: ```bash SCOUTERWORKFLOW_MAX_CONCURRENT_ACTIVITIES=500 -SCOUTERWORKFLOW_WORKFLOW_POLLER_BEHAVIUR_MAXIMUM=300 +SCOUTERWORKFLOW_WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM=300 ``` ## Notes diff --git a/tests.ipynb b/tests.ipynb index 9df4097..7831532 100644 --- a/tests.ipynb +++ b/tests.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "9d16b24a", "metadata": {}, "outputs": [], @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "5e344fb0", "metadata": {}, "outputs": [], @@ -103,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "9350bff3", "metadata": {}, "outputs": [], @@ -118,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "45712d7a", "metadata": {}, "outputs": [], @@ -143,18 +143,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "d065d0de", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "pi-web-api-scouter\n" - ] - } - ], + "outputs": [], "source": [ "import re\n", "def camel_to_kebab(text: str) -> str:\n", @@ -168,7 +160,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "72af4236", "metadata": {}, "outputs": [], @@ -208,7 +200,7 @@ " \"CI-W3K01T4\",\n", " \"CI-W3K01T2\",\n", " \"CI-W3A65_SO3\",\n", - " \"CI-W3A65_CL\",\n", + " \"CI-W3A65_Cl\",\n", " \"CI-W3_C3S\",\n", " \"CI-W3_MS\",\n", " \"CI-W3_MA\",\n", @@ -220,7 +212,9 @@ " \"CI-W3V21F1\",\n", " \"CI-W3V21P1\",\n", " \"CI-W3V30F1\",\n", - " \"CI-W3V33P1\"\n", + " \"CI-W3V33P1\",\n", + " \"CI-W3W01A1_AI\",\n", + " \"CI-W3W01A2_AI\"\n", "]\n", "url = 'https://pivision.votorantimcimentos.com/piwebapi/dataservers/F1DS-7fYgsRTtUOa7V9NIwSujAUElIQVZD/points?namefilter={tag}'\n", "\n", @@ -245,146 +239,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "55793801", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'CI-W3A05F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3W03S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujATFUAAAUElIQVZDXENJLVczVzAzUzE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3W03I1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAS1UAAAUElIQVZDXENJLVczVzAzSTE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3K01T1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAklQAAAUElIQVZDXENJLVczSzAxVDE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3W01A3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAOFUAAAUElIQVZDXENJLVczVzAxQTM',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3W01A2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAN1UAAAUElIQVZDXENJLVczVzAxQTI',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3W01A1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujANlUAAAUElIQVZDXENJLVczVzAxQTE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-J3P01T1A': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAYUUAAAUElIQVZDXENJLUozUDAxVDFB',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3A50T1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujArFMAAAUElIQVZDXENJLVczQTUwVDE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3A55T1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAxFMAAAUElIQVZDXENJLVczQTU1VDE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3A55P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAwVMAAAUElIQVZDXENJLVczQTU1UDE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3V33P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAJVUAAAUElIQVZDXENJLVczVjMzUDE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3E01F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAzY0CAAUElIQVZDXENJLVczRTAxRjE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3A50A3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAqVMAAAUElIQVZDXENJLVczQTUwQTM',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3A50A2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAqFMAAAUElIQVZDXENJLVczQTUwQTI',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3A50A1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAp1MAAAUElIQVZDXENJLVczQTUwQTE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3A50P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAq1MAAAUElIQVZDXENJLVczQTUwUDE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3W01P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAPlUAAAUElIQVZDXENJLVczVzAxUDE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3A71P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAy1MAAAUElIQVZDXENJLVczQTcxUDE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3W01P2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAP1UAAAUElIQVZDXENJLVczVzAxUDI',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3A71P2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAzFMAAAUElIQVZDXENJLVczQTcxUDI',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3A71P3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAzVMAAAUElIQVZDXENJLVczQTcxUDM',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-J3J01S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAMUUAAAUElIQVZDXENJLUozSjAxUzE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3P17S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA7VQAAAUElIQVZDXENJLVczUDE3UzE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-J3P03S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAa0UAAAUElIQVZDXENJLUozUDAzUzE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3K01S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAkFQAAAUElIQVZDXENJLVczSzAxUzE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3K14P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAuFQAAAUElIQVZDXENJLVczSzE0UDE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3K01T4': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAl1QAAAUElIQVZDXENJLVczSzAxVDQ',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3K01T2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAlVQAAAUElIQVZDXENJLVczSzAxVDI',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3A65_SO3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAjFQAAAUElIQVZDXENJLVczQTY1X1NPMw',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3A65_CL': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAf1QAAAUElIQVZDXENJLVczQTY1X0NM',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3_C3S': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA5FMAAAUElIQVZDXENJLVczX0MzUw',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3_MS': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujABlQAAAUElIQVZDXENJLVczX01T',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3_MA': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA_VMAAAUElIQVZDXENJLVczX01B',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3_PL': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAFVQAAAUElIQVZDXENJLVczX1BM',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3_CAO': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA61MAAAUElIQVZDXENJLVczX0NBTw',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3V04P3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAElUAAAUElIQVZDXENJLVczVjA0UDM',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3V04P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAEFUAAAUElIQVZDXENJLVczVjA0UDE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3W01G1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAOlUAAAUElIQVZDXENJLVczVzAxRzE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3V21F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAQnYEAAUElIQVZDXENJLVczVjIxRjE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3V21P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAQHYEAAUElIQVZDXENJLVczVjIxUDE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3V30F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAIFUAAAUElIQVZDXENJLVczVjMwRjE',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]}}" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import json\n", "json.dump(web_ids, open('web_ids.json', 'w'), indent=4)\n", @@ -394,32 +252,1951 @@ { "cell_type": "code", "execution_count": null, - "id": "2e0e3d5a", + "id": "a83b3cb4", + "metadata": {}, + "outputs": [], + "source": [ + "from scouter.activities.api import API\n", + "from unittest.mock import MagicMock, AsyncMock\n", + "from pandas import DataFrame, concat\n", + "import json\n", + "from time import sleep\n", + "\n", + "web_ids = json.load(open('web_ids.json'))\n", + "\n", + "api = API(\n", + " base_url='https://pivision.votorantimcimentos.com/piwebapi',\n", + " auth_type='basic',\n", + " auth_token='dmlkX3ZjbmV0XHN2Yy5waW9zaS5wcmQud2ViYXBpOlN2Y1ByRFdlQkBQaQ==',\n", + " logger=MagicMock(),\n", + " notification_handler=AsyncMock(),\n", + " metrics_controller=AsyncMock(),\n", + ")\n", + "\n", + "start_time = 1800\n", + "pace = 30\n", + "\n", + "data = DataFrame()\n", + "\n", + "for i in range(start_time, 0, -pace):\n", + " j = i - pace\n", + " print(f'Getting data for chunk -{i} to -{j} days')\n", + " try:\n", + " chunk = DataFrame(await api.get_tag_values(\n", + " input_data={\n", + " 'endpoint': '/streamsets/recorded',\n", + " 'web_ids': web_ids,\n", + " 'period': f'*-{i}d',\n", + " 'end_time': f'*-{j}d'.replace('-0d', ''),\n", + " 'max_count': None,\n", + " 'api_timeout': 5,\n", + " 'metadata': {}\n", + " }\n", + " ))\n", + " except Exception as e:\n", + " chunk = DataFrame(await api.get_tag_values(\n", + " input_data={\n", + " 'endpoint': '/streamsets/recorded',\n", + " 'web_ids': web_ids,\n", + " 'period': f'*-{i}d',\n", + " 'end_time': f'*-{j}d'.replace('-0d', ''),\n", + " 'max_count': None,\n", + " 'api_timeout': 5,\n", + " 'metadata': {}\n", + " }\n", + " ))\n", + " sleep(1)\n", + "\n", + " data = concat([data, chunk])\n", + "\n", + "base_data = data\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e2ef295", + "metadata": {}, + "outputs": [], + "source": [ + "trunc_spec = 's'\n", + "\n", + "base_data['truncated_timestamp'] = base_data['timestamp'].dt.floor(trunc_spec)\n", + "\n", + "# drop timestamp NaT\n", + "base_data = base_data[base_data['timestamp'].notna()]\n", + "base_data.sort_values(by='truncated_timestamp', inplace=True)\n", + "\n", + "display(base_data)\n", + "\n", + "base_data.drop_duplicates(subset=['truncated_timestamp', 'name'], keep='last', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae7b5889", + "metadata": {}, + "outputs": [], + "source": [ + "data = base_data.pivot(index='truncated_timestamp', columns='name', values='value')\n", + "\n", + "data.sort_index(inplace=True)\n", + "\n", + "# replace Nan with upper row value\n", + "data.ffill(inplace=True)\n", + "#data.bfill(inplace=True)\n", + "data.dropna(inplace=True)\n", + "\n", + "data['timestamp'] = data.index\n", + "\n", + "data.reset_index(drop=True, inplace=True)\n", + "\n", + "display(data)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4782290f", + "metadata": {}, + "outputs": [], + "source": [ + "to_insert_data = data.melt(id_vars=['timestamp'], var_name='variable', value_name='value')\n", + "to_insert_data['model_id'] = '111'\n", + "to_insert_data.drop_duplicates(subset=['timestamp', 'variable'], keep='last', inplace=True)\n", + "to_insert_data = to_insert_data.sort_values(by='timestamp', ascending=False)\n", + "to_insert_data.to_csv('data.csv', index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "a36d48d1", + "metadata": {}, + "outputs": [], + "source": [ + "from sientia_do.temporal.activities.postgres import Postgres\n", + "from unittest.mock import MagicMock, AsyncMock\n", + "\n", + "postgres_interface = Postgres(\n", + " host='localhost',\n", + " port=5432,\n", + " dbname='sientia',\n", + " user='sientia',\n", + " password='sientia',\n", + " min_connections=1,\n", + " max_connections=200,\n", + " logger=MagicMock(),\n", + " notification_handler=AsyncMock(),\n", + " metrics_controller=AsyncMock(),\n", + ")\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "713ecb2e", + "metadata": {}, + "outputs": [], + "source": [ + "from pandas import read_csv\n", + "from time import sleep\n", + "to_insert_data = read_csv('data.csv')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f269b8e7", "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "{'Links': {},\n", - " 'Items': [{'WebId': 'F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE',\n", - " 'Name': 'CI-W3A05F1',\n", - " 'Path': '\\\\\\\\PIHAVC\\\\CI-W3A05F1',\n", - " 'Links': {'Source': 'https://pivision.votorantimcimentos.com/piwebapi/points/F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE'},\n", - " 'Items': [{'Timestamp': '2025-12-17T18:12:49.2170104Z',\n", - " 'Value': 273.3339,\n", - " 'UnitsAbbreviation': '',\n", - " 'Good': True,\n", - " 'Questionable': False,\n", - " 'Substituted': False,\n", - " 'Annotated': False}],\n", - " 'UnitsAbbreviation': ''}]}" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 0 to 100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 100000 to 200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 200000 to 300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 300000 to 400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 400000 to 500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 500000 to 600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 600000 to 700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 700000 to 800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 800000 to 900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 900000 to 1000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1000000 to 1100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1100000 to 1200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1200000 to 1300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1300000 to 1400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1400000 to 1500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1500000 to 1600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1600000 to 1700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1700000 to 1800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1800000 to 1900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 1900000 to 2000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2000000 to 2100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2100000 to 2200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2200000 to 2300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2300000 to 2400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2400000 to 2500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2500000 to 2600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2600000 to 2700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2700000 to 2800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2800000 to 2900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 2900000 to 3000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3000000 to 3100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3100000 to 3200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3200000 to 3300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3300000 to 3400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3400000 to 3500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3500000 to 3600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3600000 to 3700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3700000 to 3800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3800000 to 3900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 3900000 to 4000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4000000 to 4100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4100000 to 4200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4200000 to 4300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4300000 to 4400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4400000 to 4500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4500000 to 4600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4600000 to 4700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4700000 to 4800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4800000 to 4900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 4900000 to 5000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5000000 to 5100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5100000 to 5200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5200000 to 5300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5300000 to 5400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5400000 to 5500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5500000 to 5600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5600000 to 5700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5700000 to 5800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5800000 to 5900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 5900000 to 6000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6000000 to 6100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6100000 to 6200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6200000 to 6300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6300000 to 6400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6400000 to 6500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6500000 to 6600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6600000 to 6700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6700000 to 6800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6800000 to 6900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 6900000 to 7000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7000000 to 7100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7100000 to 7200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7200000 to 7300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7300000 to 7400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7400000 to 7500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7500000 to 7600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7600000 to 7700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7700000 to 7800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7800000 to 7900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 7900000 to 8000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8000000 to 8100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8100000 to 8200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8200000 to 8300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8300000 to 8400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8400000 to 8500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8500000 to 8600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8600000 to 8700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8700000 to 8800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8800000 to 8900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 8900000 to 9000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9000000 to 9100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9100000 to 9200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9200000 to 9300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9300000 to 9400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9400000 to 9500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9500000 to 9600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9600000 to 9700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9700000 to 9800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9800000 to 9900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 9900000 to 10000000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10000000 to 10100000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10100000 to 10200000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10200000 to 10300000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10300000 to 10400000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10400000 to 10500000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10500000 to 10600000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10600000 to 10700000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10700000 to 10800000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10800000 to 10900000\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inserting chunk 10900000 to 11000000\n" + ] } ], + "source": [ + "\n", + "chunk_start = 10900000\n", + "chunk_end = 20000000\n", + "\n", + "chunk_size = 100000\n", + "\n", + "chunk_pace = chunk_start\n", + "while chunk_pace < chunk_end:\n", + " print(f'Inserting chunk {chunk_pace} to {chunk_pace+chunk_size}')\n", + " chunk = to_insert_data.iloc[chunk_pace:chunk_pace+chunk_size]\n", + " await postgres_interface.export_data_to_postgres(\n", + " input_data={\n", + " 'data': chunk,\n", + " 'table_name': 'laborious_data',\n", + " 'schema': 'sientia_data',\n", + " 'unique_columns': ['timestamp', 'variable', 'model_id'],\n", + " 'on_conflict': 'ignore',\n", + " 'metadata': {}\n", + " }\n", + " )\n", + " sleep(10)\n", + " chunk_pace += chunk_size" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e0e3d5a", + "metadata": {}, + "outputs": [], "source": [ "item = web_ids['CI-W3A05F1']['webid']\n", "\n", @@ -437,9 +2214,34 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "f8c425e2", "metadata": {}, + "outputs": [], + "source": [ + "from pandas import read_csv, to_datetime\n", + "\n", + "df = read_csv('/home/grezewave/Downloads/laborious_data_202512220821.csv')\n", + "data = df.pivot(index='timestamp', columns='variable', values='value')\n", + "data['timestamp'] = data.index\n", + "\n", + "#Remove tz from timestamp\n", + "data['timestamp'] = to_datetime(data['timestamp'])\n", + "data['timestamp'] = data['timestamp'].dt.tz_localize(None)\n", + "\n", + "# Back to string and add \"\"\n", + "data['timestamp'] = data['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')\n", + "data.reset_index(drop=True, inplace=True)\n", + "data.to_csv('VC-model-data.csv', index=False)\n", + "\n", + "display(data)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "2bd5569d", + "metadata": {}, "outputs": [ { "data": { @@ -461,494 +2263,136 @@ "\n", " \n", " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", " \n", + " \n", " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
model_idvariableCI-J3J01S1CI-J3P01T1ACI-J3P03S1CI-W3A05F1CI-W3A50A1CI-W3A50A2CI-W3A50A3CI-W3A50P1CI-W3A50T1CI-W3A55P1...CI-W3V33P1CI-W3W01A1CI-W3W01A2CI-W3W01A3CI-W3W01G1CI-W3W01P1CI-W3W01P2CI-W3W03I1CI-W3W03S1valuetimestampcreated_at
087.999020238.000000100.442688272.8691000.0833382.636171589.246400-0.923784383.722400-18.167961...259.5655000.0082204.190174376.848267600.466900-3.1095980.22179863.6058301545.502932025-12-17 22:12:42111CI-W3V30F18.5148512026-01-20 19:41:23+00002026-01-20 23:28:18+0000
187.999020238.000000100.442688272.6492000.0833382.267340618.462100-0.771080381.468872-17.495136...263.0998540.0082202.611867435.797668600.466900-3.0301560.15352765.7857061545.502932025-12-17 22:41:54111CI-W3A71P3-2.3086902026-01-20 19:41:23+00002026-01-20 23:28:18+0000
286.997925228.200012100.442688264.2253720.0823583.254068444.800200-0.789923389.007800-16.590466...223.3463130.0859279.447197343.596000600.000000-2.4367670.35028969.8865201616.154912025-12-18 19:08:08111CI-J3P01T1A228.9000002026-01-20 19:41:23+00002026-01-20 23:28:18+0000
386.997925228.200012100.442688266.7141720.0823612.444070481.586060-0.551620390.612854-16.791473...240.6614690.0859279.324739462.062256600.000000-3.0301560.20861664.6597301628.228762025-12-18 19:23:08111CI-W3W03I175.3674242026-01-20 19:41:23+00002026-01-20 23:28:18+0000
491.991210139.299988100.442688280.3923340.0823611.835279436.144100-0.486019387.727000-17.495136...249.9351650.0596025.359922367.250300600.000000-3.2863160.27637259.9493371691.234622025-12-18 19:31:54111CI-J3J01S187.9990202026-01-20 19:41:23+00002026-01-20 23:28:18+0000
590.001220230.000000100.442688268.8931580.0836232.572147410.169100-0.929219395.898200-16.390797..................241.3099670.0020004.262301340.296700880.778200-3.6710640.24807565.1953901609.201542025-12-18 19:34:28
690.001220230.000000100.442688269.3590000.0836232.636187417.679138-0.837243396.060272-16.390797...239.3646240.0020003.964616318.547300935.914368-3.4776270.24807563.2846381609.201542025-12-18 19:35:199993055111CI-W3V30F16.8461082025-01-21 11:35:08+00002026-01-21 00:18:50+0000
790.001220230.000000100.442688268.6519780.0836232.483914404.910522-0.906091396.222473-16.390797...239.3646240.0020004.766942314.721130903.813232-3.7976250.24807564.6019741645.863892025-12-18 19:39:049993056111CI-W3K14P138.0967702025-01-21 11:35:08+00002026-01-21 00:18:50+0000
890.001220230.000000100.442688270.8364260.0836232.748308403.442100-0.939606396.222473-16.390797...239.3646240.0013624.637836325.616100909.961060-3.6057070.24807562.8696441646.514282025-12-18 19:40:589993057111CI-W3A50T1364.2500002025-01-21 11:35:08+00002026-01-21 00:18:50+0000
990.001220227.700012100.442688270.7156370.0897542.644230425.803000-1.007370396.384521-16.725641...239.0402220.0013615.083293349.286682949.923500-3.7029830.20841365.8502961647.255372025-12-18 20:12:239993058111CI-W3A55T1871.6823002025-01-21 11:35:08+00002026-01-21 00:18:50+0000
1090.001220227.700012100.442688270.6614380.0788592.628080435.462860-0.962243396.384521-17.031452...239.3644410.1086905.151842353.761353949.961060-3.7337880.20741364.7887401644.680422025-12-19 03:26:44
1190.001220227.700012100.442688270.7251590.0788592.628080424.227722-0.827545396.384521-17.031452...239.3644410.1086904.830516353.761353893.074000-3.7337880.20741364.7887401644.680422025-12-19 03:32:37
1290.001220227.700012100.442688269.9770200.0788593.799854350.998047-0.901527395.614441-18.777557...226.5562740.1086905.567447312.856700887.664734-3.4452010.18896765.3761441644.680422025-12-19 03:59:20
1390.001220227.700012100.442688269.0980000.0788593.325234401.663940-0.960138392.380127-16.966602...235.8625180.0251952.443661305.058400963.424100-3.6543450.18896764.8246601392.283942025-12-19 04:07:07
1489.000120133.000000100.442688280.5959000.0788592.347593453.182526-0.409587389.169952-15.958172...238.0998540.0251955.168004417.250244852.529200-3.7029830.26809562.1951981645.463262025-12-19 04:27:35
1591.002320130.000000100.442688283.0378420.1095972.547736364.069400-0.737562394.961900-18.296043...226.2309000.0730104.334429338.391663793.501953-4.7276260.00395972.8045961718.755862025-12-21 14:12:479993059111CI-W3W01P1-2.3568292025-01-21 11:35:08+00002026-01-21 00:18:50+0000
\n", - "

16 rows × 33 columns

\n", + "

9993060 rows × 5 columns

\n", "" ], "text/plain": [ - "variable CI-J3J01S1 CI-J3P01T1A CI-J3P03S1 CI-W3A05F1 CI-W3A50A1 \\\n", - "0 87.999020 238.000000 100.442688 272.869100 0.083338 \n", - "1 87.999020 238.000000 100.442688 272.649200 0.083338 \n", - "2 86.997925 228.200012 100.442688 264.225372 0.082358 \n", - "3 86.997925 228.200012 100.442688 266.714172 0.082361 \n", - "4 91.991210 139.299988 100.442688 280.392334 0.082361 \n", - "5 90.001220 230.000000 100.442688 268.893158 0.083623 \n", - "6 90.001220 230.000000 100.442688 269.359000 0.083623 \n", - "7 90.001220 230.000000 100.442688 268.651978 0.083623 \n", - "8 90.001220 230.000000 100.442688 270.836426 0.083623 \n", - "9 90.001220 227.700012 100.442688 270.715637 0.089754 \n", - "10 90.001220 227.700012 100.442688 270.661438 0.078859 \n", - "11 90.001220 227.700012 100.442688 270.725159 0.078859 \n", - "12 90.001220 227.700012 100.442688 269.977020 0.078859 \n", - "13 90.001220 227.700012 100.442688 269.098000 0.078859 \n", - "14 89.000120 133.000000 100.442688 280.595900 0.078859 \n", - "15 91.002320 130.000000 100.442688 283.037842 0.109597 \n", + " model_id variable value timestamp \\\n", + "0 111 CI-W3V30F1 8.514851 2026-01-20 19:41:23+0000 \n", + "1 111 CI-W3A71P3 -2.308690 2026-01-20 19:41:23+0000 \n", + "2 111 CI-J3P01T1A 228.900000 2026-01-20 19:41:23+0000 \n", + "3 111 CI-W3W03I1 75.367424 2026-01-20 19:41:23+0000 \n", + "4 111 CI-J3J01S1 87.999020 2026-01-20 19:41:23+0000 \n", + "... ... ... ... ... \n", + "9993055 111 CI-W3V30F1 6.846108 2025-01-21 11:35:08+0000 \n", + "9993056 111 CI-W3K14P1 38.096770 2025-01-21 11:35:08+0000 \n", + "9993057 111 CI-W3A50T1 364.250000 2025-01-21 11:35:08+0000 \n", + "9993058 111 CI-W3A55T1 871.682300 2025-01-21 11:35:08+0000 \n", + "9993059 111 CI-W3W01P1 -2.356829 2025-01-21 11:35:08+0000 \n", "\n", - "variable CI-W3A50A2 CI-W3A50A3 CI-W3A50P1 CI-W3A50T1 CI-W3A55P1 ... \\\n", - "0 2.636171 589.246400 -0.923784 383.722400 -18.167961 ... \n", - "1 2.267340 618.462100 -0.771080 381.468872 -17.495136 ... \n", - "2 3.254068 444.800200 -0.789923 389.007800 -16.590466 ... \n", - "3 2.444070 481.586060 -0.551620 390.612854 -16.791473 ... \n", - "4 1.835279 436.144100 -0.486019 387.727000 -17.495136 ... \n", - "5 2.572147 410.169100 -0.929219 395.898200 -16.390797 ... \n", - "6 2.636187 417.679138 -0.837243 396.060272 -16.390797 ... \n", - "7 2.483914 404.910522 -0.906091 396.222473 -16.390797 ... \n", - "8 2.748308 403.442100 -0.939606 396.222473 -16.390797 ... \n", - "9 2.644230 425.803000 -1.007370 396.384521 -16.725641 ... \n", - "10 2.628080 435.462860 -0.962243 396.384521 -17.031452 ... \n", - "11 2.628080 424.227722 -0.827545 396.384521 -17.031452 ... \n", - "12 3.799854 350.998047 -0.901527 395.614441 -18.777557 ... \n", - "13 3.325234 401.663940 -0.960138 392.380127 -16.966602 ... \n", - "14 2.347593 453.182526 -0.409587 389.169952 -15.958172 ... \n", - "15 2.547736 364.069400 -0.737562 394.961900 -18.296043 ... \n", + " created_at \n", + "0 2026-01-20 23:28:18+0000 \n", + "1 2026-01-20 23:28:18+0000 \n", + "2 2026-01-20 23:28:18+0000 \n", + "3 2026-01-20 23:28:18+0000 \n", + "4 2026-01-20 23:28:18+0000 \n", + "... ... \n", + "9993055 2026-01-21 00:18:50+0000 \n", + "9993056 2026-01-21 00:18:50+0000 \n", + "9993057 2026-01-21 00:18:50+0000 \n", + "9993058 2026-01-21 00:18:50+0000 \n", + "9993059 2026-01-21 00:18:50+0000 \n", "\n", - "variable CI-W3V33P1 CI-W3W01A1 CI-W3W01A2 CI-W3W01A3 CI-W3W01G1 \\\n", - "0 259.565500 0.008220 4.190174 376.848267 600.466900 \n", - "1 263.099854 0.008220 2.611867 435.797668 600.466900 \n", - "2 223.346313 0.085927 9.447197 343.596000 600.000000 \n", - "3 240.661469 0.085927 9.324739 462.062256 600.000000 \n", - "4 249.935165 0.059602 5.359922 367.250300 600.000000 \n", - "5 241.309967 0.002000 4.262301 340.296700 880.778200 \n", - "6 239.364624 0.002000 3.964616 318.547300 935.914368 \n", - "7 239.364624 0.002000 4.766942 314.721130 903.813232 \n", - "8 239.364624 0.001362 4.637836 325.616100 909.961060 \n", - "9 239.040222 0.001361 5.083293 349.286682 949.923500 \n", - "10 239.364441 0.108690 5.151842 353.761353 949.961060 \n", - "11 239.364441 0.108690 4.830516 353.761353 893.074000 \n", - "12 226.556274 0.108690 5.567447 312.856700 887.664734 \n", - "13 235.862518 0.025195 2.443661 305.058400 963.424100 \n", - "14 238.099854 0.025195 5.168004 417.250244 852.529200 \n", - "15 226.230900 0.073010 4.334429 338.391663 793.501953 \n", - "\n", - "variable CI-W3W01P1 CI-W3W01P2 CI-W3W03I1 CI-W3W03S1 timestamp \n", - "0 -3.109598 0.221798 63.605830 1545.50293 2025-12-17 22:12:42 \n", - "1 -3.030156 0.153527 65.785706 1545.50293 2025-12-17 22:41:54 \n", - "2 -2.436767 0.350289 69.886520 1616.15491 2025-12-18 19:08:08 \n", - "3 -3.030156 0.208616 64.659730 1628.22876 2025-12-18 19:23:08 \n", - "4 -3.286316 0.276372 59.949337 1691.23462 2025-12-18 19:31:54 \n", - "5 -3.671064 0.248075 65.195390 1609.20154 2025-12-18 19:34:28 \n", - "6 -3.477627 0.248075 63.284638 1609.20154 2025-12-18 19:35:19 \n", - "7 -3.797625 0.248075 64.601974 1645.86389 2025-12-18 19:39:04 \n", - "8 -3.605707 0.248075 62.869644 1646.51428 2025-12-18 19:40:58 \n", - "9 -3.702983 0.208413 65.850296 1647.25537 2025-12-18 20:12:23 \n", - "10 -3.733788 0.207413 64.788740 1644.68042 2025-12-19 03:26:44 \n", - "11 -3.733788 0.207413 64.788740 1644.68042 2025-12-19 03:32:37 \n", - "12 -3.445201 0.188967 65.376144 1644.68042 2025-12-19 03:59:20 \n", - "13 -3.654345 0.188967 64.824660 1392.28394 2025-12-19 04:07:07 \n", - "14 -3.702983 0.268095 62.195198 1645.46326 2025-12-19 04:27:35 \n", - "15 -4.727626 0.003959 72.804596 1718.75586 2025-12-21 14:12:47 \n", - "\n", - "[16 rows x 33 columns]" + "[9993060 rows x 5 columns]" ] }, "metadata": {}, @@ -956,22 +2400,12 @@ } ], "source": [ - "from pandas import read_csv, to_datetime\n", + "from pandas import read_parquet\n", "\n", - "df = read_csv('/home/grezewave/Downloads/laborious_data_202512220821.csv')\n", - "data = df.pivot(index='timestamp', columns='variable', values='value')\n", - "data['timestamp'] = data.index\n", + "df = read_parquet('~/Downloads/data_2026-01-21_11-32-10.parquet')\n", "\n", - "#Remove tz from timestamp\n", - "data['timestamp'] = to_datetime(data['timestamp'])\n", - "data['timestamp'] = data['timestamp'].dt.tz_localize(None)\n", - "\n", - "# Back to string and add \"\"\n", - "data['timestamp'] = data['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')\n", - "data.reset_index(drop=True, inplace=True)\n", - "data.to_csv('VC-model-data.csv', index=False)\n", - "\n", - "display(data)" + "display(df)\n", + "\n" ] } ], diff --git a/values.yaml b/values.yaml index 47bfd0b..c07158e 100644 --- a/values.yaml +++ b/values.yaml @@ -242,18 +242,18 @@ env: - name: SCOUTER_MAX_CACHED_WORKFLOWS value: "200" - - name: SCOUTER_WORKFLOW_POLLER_BEHAVIUR_MINIMUM + - name: SCOUTER_WORKFLOW_POLLER_BEHAVIOUR_MINIMUM value: "10" - - name: SCOUTER_WORKFLOW_POLLER_BEHAVIUR_INITIAL + - name: SCOUTER_WORKFLOW_POLLER_BEHAVIOUR_INITIAL value: "100" - - name: SCOUTER_WORKFLOW_POLLER_BEHAVIUR_MAXIMUM + - name: SCOUTER_WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM value: "200" - - name: SCOUTER_ACTIVITY_POLLER_BEHAVIUR_MINIMUM + - name: SCOUTER_ACTIVITY_POLLER_BEHAVIOUR_MINIMUM value: "10" - - name: SCOUTER_ACTIVITY_POLLER_BEHAVIUR_INITIAL + - name: SCOUTER_ACTIVITY_POLLER_BEHAVIOUR_INITIAL value: "100" - - name: SCOUTER_ACTIVITY_POLLER_BEHAVIUR_MAXIMUM + - name: SCOUTER_ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM value: "200" From fb4bef317d4c3a3bd500a4da6f702f0330d6bd5b Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 23 Jan 2026 10:10:52 -0300 Subject: [PATCH 16/21] SIENTIAPDE-1478 Remove unused import for DATETIME_FORMAT_WITH_TZ in API class to streamline code and improve readability. --- scouter/activities/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scouter/activities/api.py b/scouter/activities/api.py index 530ff44..60a261e 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -10,7 +10,6 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.metrics_controller import MetricsController from sientia_do.observability.sientia_monitoring import SientiaMonitoring from sientia_do.repository.pi_web_api_client import PIWebAPIClient - from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ class API(SientiaMonitoring): From cf367f2319b6cf8c44bf185e040d124e2637ab97 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 23 Jan 2026 10:40:30 -0300 Subject: [PATCH 17/21] SIENTIAPDE-1478 Enhance API class to format timestamps correctly and improve logging. Update test cases to include 'end_time' parameter for better data retrieval flexibility. --- scouter/activities/api.py | 11 ++++++----- tests/activities/test_api.py | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scouter/activities/api.py b/scouter/activities/api.py index 60a261e..21efcd7 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -10,6 +10,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.metrics_controller import MetricsController from sientia_do.observability.sientia_monitoring import SientiaMonitoring from sientia_do.repository.pi_web_api_client import PIWebAPIClient + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ class API(SientiaMonitoring): @@ -137,13 +138,13 @@ class API(SientiaMonitoring): ) raise e - # latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ) + latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ) - # self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata) + self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata) - # # Normalize the package timestamp - # valid_timestamp_values = latest_values['timestamp'].dropna() - # latest_values['timestamp'] = valid_timestamp_values.max() + # Normalize the package timestamp + valid_timestamp_values = latest_values['timestamp'].dropna() + latest_values['timestamp'] = valid_timestamp_values.max() self.info(f'Gathered {len(latest_values)} tag values', metadata=metadata) diff --git a/tests/activities/test_api.py b/tests/activities/test_api.py index 977cd0d..dbfbd2e 100644 --- a/tests/activities/test_api.py +++ b/tests/activities/test_api.py @@ -139,6 +139,7 @@ async def test_get_tag_values_success(api_activity): 'tag3': {'webid': 'webid3', 'aggr_function': 'avg', 'data_range': [0, 100]}, }, start_time='*-1d', + end_time='*', max_count=10, metadata=metadata['metadata'], request_timeout=30, @@ -193,6 +194,7 @@ async def test_get_tag_values_with_default_max_count(api_activity): endpoint='/streamsets/recorded', web_ids={'tag1': {'webid': 'webid1', 'aggr_function': 'avg', 'data_range': [0, 100]}}, start_time='*-1h', + end_time='*', max_count=1, metadata=metadata['metadata'], request_timeout=15, From dc8ff4db1a4a4cf4eb1fa2bfa3caf2154b0b6e48 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 23 Jan 2026 13:09:49 -0300 Subject: [PATCH 18/21] SIENTIAPDE-1478 Update sientia-dataops-library dependency version in requirements.txt from 1.8.0 to 1.8.2 for improved functionality. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d18dd00..a02d39c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ psycopg2-binary sqlalchemy redis pymongo -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.8.0 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.8.2 prometheus-client pycurl \ No newline at end of file From 59e8f9c8c7f74bc0c71b8b397845b6cdb7ecb432 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 23 Jan 2026 13:15:28 -0300 Subject: [PATCH 19/21] Update scouter/workflow/pi_web_api_scouter.py Co-authored-by: codeant-ai[bot] <151821869+codeant-ai[bot]@users.noreply.github.com> --- scouter/workflow/pi_web_api_scouter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scouter/workflow/pi_web_api_scouter.py b/scouter/workflow/pi_web_api_scouter.py index acaa1dc..d9d6d96 100644 --- a/scouter/workflow/pi_web_api_scouter.py +++ b/scouter/workflow/pi_web_api_scouter.py @@ -59,7 +59,7 @@ class PIWebAPIScouter: to WebIds and processing rules, including: - webid (str): PI Web API WebId for the tag - data_range: [min, max] values for data validation - - aggr_function: Aggregation method (avg, mdn, max, min, lts) + - aggr_func: Aggregation method (avg, mdn, max, min, lts) - frequency: Data collection frequency in milliseconds - topics: List of Kafka topics for data routing From 1a91bee5482bdbd555d9d4304afe0ca03bfc5941 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 23 Jan 2026 13:37:28 -0300 Subject: [PATCH 20/21] Update tests.ipynb Co-authored-by: codeant-ai[bot] <151821869+codeant-ai[bot]@users.noreply.github.com> --- tests.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.ipynb b/tests.ipynb index 7831532..e2d55ed 100644 --- a/tests.ipynb +++ b/tests.ipynb @@ -222,7 +222,7 @@ " 'Content-Type': 'application/json',\n", " 'Accept': 'application/json',\n", " 'X-Requested-With': 'piwebapistreams', # Header recomendado pelo PI Web API\n", - " 'Authorization': \"Basic dmlkX3ZjbmV0XHN2Yy5waW9zaS5wcmQud2ViYXBpOlN2Y1ByRFdlQkBQaQ==\"\n", + 'Authorization': "Basic " + __import__('os').environ.get('PI_WEB_API_BASIC_AUTH', '') "}\n", "\n", "web_ids = {}\n", From c904b85be26fda41e85ec65bd42365c71905dfec Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 23 Jan 2026 13:37:42 -0300 Subject: [PATCH 21/21] Update tests.ipynb Co-authored-by: codeant-ai[bot] <151821869+codeant-ai[bot]@users.noreply.github.com> --- tests.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.ipynb b/tests.ipynb index e2d55ed..8dd70f6 100644 --- a/tests.ipynb +++ b/tests.ipynb @@ -267,7 +267,7 @@ "api = API(\n", " base_url='https://pivision.votorantimcimentos.com/piwebapi',\n", " auth_type='basic',\n", - " auth_token='dmlkX3ZjbmV0XHN2Yy5waW9zaS5wcmQud2ViYXBpOlN2Y1ByRFdlQkBQaQ==',\n", + auth_token=__import__('os').environ.get('PI_WEB_API_BASIC_AUTH', ''), " logger=MagicMock(),\n", " notification_handler=AsyncMock(),\n", " metrics_controller=AsyncMock(),\n",