diff --git a/scouter/activities/api.py b/scouter/activities/api.py index eb00b2e..f059d35 100644 --- a/scouter/activities/api.py +++ b/scouter/activities/api.py @@ -9,9 +9,10 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.logger import Logger from sientia_do.observability.metrics_controller import MetricsController from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ from scouter.utils.clients.pi_web_api_client import PIWebAPIClient - from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ + class API(SientiaMonitoring): """ diff --git a/scouter/utils/clients/pi_web_api_client.py b/scouter/utils/clients/pi_web_api_client.py index d4bbf12..d18528a 100644 --- a/scouter/utils/clients/pi_web_api_client.py +++ b/scouter/utils/clients/pi_web_api_client.py @@ -213,7 +213,6 @@ class PIWebAPIClient(SientiaMonitoring): tags=core_labels, ) - status_code = c.getinfo(pycurl.RESPONSE_CODE) body = buffer.getvalue().decode('utf-8', errors='replace') diff --git a/scouter/worker/prepare_worker.py b/scouter/worker/prepare_worker.py index fe16fd3..a3057d5 100644 --- a/scouter/worker/prepare_worker.py +++ b/scouter/worker/prepare_worker.py @@ -1,14 +1,12 @@ import os +import re from collections.abc import Sequence from typing import Any +from sientia_do.observability.logger import Logger from temporalio.client import Client from temporalio.worker import PollerBehaviorAutoscaling, Worker -from sientia_do.observability.logger import Logger - -import re - parameters = [ ('MAX_CONCURRENT_WORKFLOW_TASKS', '200'), ('MAX_CONCURRENT_ACTIVITIES', '200'), @@ -22,7 +20,6 @@ parameters = [ ('ACTIVITY_POLLER_BEHAVIUR_MAXIMUM', '200'), ] -import re def camel_to_kebab(text: str) -> str: """Convert camelCase or PascalCase to kebab-case.""" diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 40744c9..4de701a 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -20,9 +20,9 @@ with workflow.unsafe.imports_passed_through(): build_postgres_config, build_redis_config, ) + from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter from scouter.workflow.scouter import Scouter from scouter.workflow.sub_workflows.core_scouter import CoreScouter - from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter # Environment configuration POD_ID = os.getenv('HOSTNAME', 'localhost') diff --git a/tests/activities/test_api.py b/tests/activities/test_api.py index 8b49219..91bd374 100644 --- a/tests/activities/test_api.py +++ b/tests/activities/test_api.py @@ -112,13 +112,19 @@ async def test_get_tag_values_success(api_activity): # Mock DataFrame response mock_df = pd.DataFrame( { - 'timestamp': ['2023-01-01 12:00:00', '2023-01-01 12:01:00', '2023-01-01 12:02:00'], + 'timestamp': [ + '2023-01-01 12:00:00+0000', + '2023-01-01 12:01:00+0000', + '2023-01-01 12:02:00+0000', + ], 'name': ['tag1', 'tag2', 'tag3'], 'value': [10.5, 20.3, 30.7], 'tag': ['webid1', 'webid2', 'webid3'], } ) + mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp']) + api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df) # Execute @@ -143,6 +149,9 @@ async def test_get_tag_values_success(api_activity): assert result[0]['value'] == 10.5 assert result[1]['name'] == 'tag2' assert result[2]['name'] == 'tag3' + assert result[0]['timestamp'] == '2023-01-01 12:00:00+0000' + assert result[1]['timestamp'] == '2023-01-01 12:01:00+0000' + assert result[2]['timestamp'] == '2023-01-01 12:02:00+0000' @pytest.mark.asyncio @@ -165,13 +174,15 @@ async def test_get_tag_values_with_default_max_count(api_activity): mock_df = pd.DataFrame( { - 'timestamp': ['2023-01-01 12:00:00'], + 'timestamp': ['2023-01-01 12:00:00+0000'], 'name': ['tag1'], 'value': [42.0], 'tag': ['webid1'], } ) + mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp']) + api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df) # Execute @@ -217,13 +228,15 @@ async def test_get_tag_values_with_none_webids(api_activity): mock_df = pd.DataFrame( { - 'timestamp': ['2023-01-01 12:00:00', '2023-01-01 12:01:00'], + 'timestamp': ['2023-01-01 12:00:00+0000', '2023-01-01 12:01:00+0000'], 'name': ['tag1', 'tag3'], 'value': [10.5, 30.7], 'tag': ['webid1', 'webid3'], } ) + mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp']) + api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df) # Execute @@ -303,13 +316,15 @@ async def test_get_tag_values_with_nan_values(api_activity): # Mock DataFrame with NaN values mock_df = pd.DataFrame( { - 'timestamp': ['2023-01-01 12:00:00', '2023-01-01 12:00:00'], + 'timestamp': ['2023-01-01 12:00:00+0000', '2023-01-01 12:00:00+0000'], 'name': ['tag1', 'tag2'], 'value': [10.0, float('nan')], 'tag': ['webid1', 'webid2'], } ) + mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp']) + api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df) # Execute diff --git a/tests/utils/test_connectors_config.py b/tests/utils/test_connectors_config.py index f71a0fc..be7ab70 100644 --- a/tests/utils/test_connectors_config.py +++ b/tests/utils/test_connectors_config.py @@ -173,9 +173,9 @@ def test_build_api_config_with_env_vars(): with patch.dict( os.environ, { - 'API_BASE_URL': 'https://api.production.com', - 'API_AUTH_TYPE': 'bearer', - 'API_AUTH_TOKEN': 'secret_token_123', + 'PI_WEB_API_BASE_URL': 'https://api.production.com', + 'PI_WEB_API_AUTH_TYPE': 'bearer', + 'PI_WEB_API_AUTH_TOKEN': 'secret_token_123', }, ): config = build_api_config() diff --git a/tests/workflow/test_pi_web_api_scouter.py b/tests/workflow/test_pi_web_api_scouter.py index 5348411..4f7f381 100644 --- a/tests/workflow/test_pi_web_api_scouter.py +++ b/tests/workflow/test_pi_web_api_scouter.py @@ -20,16 +20,18 @@ async def test_pi_web_api_scouter_workflow(mock_workflow, pi_web_api_scouter): 'model_name': 'test_model', 'model_id': 'test_model_id', 'schedule_name': 'test_schedule', - 'endpoint': '/streamsets/recorded', 'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'}, - 'period': {'start_time': '2025-01-01T00:00:00Z'}, - 'api_timeout': 30, - 'max_count': 10, 'trigger_laborious': True, 'filters': {'quality': 'good'}, 'schema': 'test_schema', 'table_name': 'test_table', 'retention_time': 3600, + 'pi_web_api_query': { + 'endpoint': '/streamsets/recorded', + 'period': '*-1d', + 'max_count': 10, + 'api_timeout': 30, + }, } ) @@ -48,7 +50,7 @@ async def test_pi_web_api_scouter_workflow(mock_workflow, pi_web_api_scouter): **expected_metadata, 'endpoint': '/streamsets/recorded', 'web_ids': {'tag1': 'webid1', 'tag2': 'webid2'}, - 'period': {'start_time': '2025-01-01T00:00:00Z'}, + 'period': '*-1d', 'api_timeout': 30, 'max_count': 10, }, @@ -62,11 +64,7 @@ async def test_pi_web_api_scouter_workflow(mock_workflow, pi_web_api_scouter): 'model_name': 'test_model', 'model_id': 'test_model_id', 'schedule_name': 'test_schedule', - 'endpoint': '/streamsets/recorded', 'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'}, - 'period': {'start_time': '2025-01-01T00:00:00Z'}, - 'api_timeout': 30, - 'max_count': 10, 'trigger_laborious': True, 'filters': {'quality': 'good'}, 'schema': 'test_schema', @@ -75,6 +73,12 @@ async def test_pi_web_api_scouter_workflow(mock_workflow, pi_web_api_scouter): 'workflow_name': 'scouter', 'data': 'test_data', 'metadata': expected_metadata, + 'pi_web_api_query': { + 'endpoint': '/streamsets/recorded', + 'period': '*-1d', + 'max_count': 10, + 'api_timeout': 30, + }, }, ) @@ -88,15 +92,18 @@ async def test_pi_web_api_scouter_workflow_empty(mock_workflow, pi_web_api_scout 'model_name': 'test_model', 'model_id': 'test_model_id', 'schedule_name': 'test_schedule', - 'endpoint': '/streamsets/recorded', 'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'}, - 'period': {'start_time': '2025-01-01T00:00:00Z'}, - 'api_timeout': 30, 'trigger_laborious': True, 'filters': {'quality': 'good'}, 'schema': 'test_schema', 'table_name': 'test_table', 'retention_time': 3600, + 'pi_web_api_query': { + 'endpoint': '/streamsets/recorded', + 'period': '*-1d', + 'max_count': 1, + 'api_timeout': 30, + }, } ) @@ -113,11 +120,11 @@ async def test_pi_web_api_scouter_workflow_empty(mock_workflow, pi_web_api_scout Activities.get_tag_values, { **expected_metadata, - 'endpoint': '/streamsets/recorded', 'web_ids': {'tag1': 'webid1', 'tag2': 'webid2'}, - 'period': {'start_time': '2025-01-01T00:00:00Z'}, + 'period': '*-1d', 'api_timeout': 30, 'max_count': 1, + 'endpoint': '/streamsets/recorded', }, start_to_close_timeout=ANY, retry_policy=ANY,