SIENTIAPDE-1445

Refactor imports and enhance timestamp handling in API and test files

- Removed redundant import statements in api.py and prepare_worker.py for cleaner code.
- Updated timestamp format in test cases to include timezone information for consistency.
- Adjusted test configurations to align with new API structure for improved accuracy in data handling.

scouter/activities/api.py

@@ -9,9 +9,10 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.observability.logger import Logger
     from sientia_do.observability.metrics_controller import MetricsController
     from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
 
     from scouter.utils.clients.pi_web_api_client import PIWebAPIClient
-    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
+
 
 class API(SientiaMonitoring):
     """

scouter/utils/clients/pi_web_api_client.py

@@ -213,7 +213,6 @@ class PIWebAPIClient(SientiaMonitoring):
                 tags=core_labels,
             )
 
-
             status_code = c.getinfo(pycurl.RESPONSE_CODE)
             body = buffer.getvalue().decode('utf-8', errors='replace')
 

scouter/worker/prepare_worker.py

@@ -1,14 +1,12 @@
 import os
+import re
 from collections.abc import Sequence
 from typing import Any
 
+from sientia_do.observability.logger import Logger
 from temporalio.client import Client
 from temporalio.worker import PollerBehaviorAutoscaling, Worker
 
-from sientia_do.observability.logger import Logger
-
-import re
-
 parameters = [
     ('MAX_CONCURRENT_WORKFLOW_TASKS', '200'),
     ('MAX_CONCURRENT_ACTIVITIES', '200'),
@@ -22,7 +20,6 @@ parameters = [
     ('ACTIVITY_POLLER_BEHAVIUR_MAXIMUM', '200'),
 ]
 
-import re
 
 def camel_to_kebab(text: str) -> str:
     """Convert camelCase or PascalCase to kebab-case."""

scouter/worker/worker.py

@@ -20,9 +20,9 @@ with workflow.unsafe.imports_passed_through():
         build_postgres_config,
         build_redis_config,
     )
+    from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter
     from scouter.workflow.scouter import Scouter
     from scouter.workflow.sub_workflows.core_scouter import CoreScouter
-    from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter
 
 # Environment configuration
 POD_ID = os.getenv('HOSTNAME', 'localhost')

tests/activities/test_api.py

@@ -112,13 +112,19 @@ async def test_get_tag_values_success(api_activity):
     # Mock DataFrame response
     mock_df = pd.DataFrame(
         {
-            'timestamp': ['2023-01-01 12:00:00', '2023-01-01 12:01:00', '2023-01-01 12:02:00'],
+            'timestamp': [
+                '2023-01-01 12:00:00+0000',
+                '2023-01-01 12:01:00+0000',
+                '2023-01-01 12:02:00+0000',
+            ],
             'name': ['tag1', 'tag2', 'tag3'],
             'value': [10.5, 20.3, 30.7],
             'tag': ['webid1', 'webid2', 'webid3'],
         }
     )
 
+    mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp'])
+
     api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df)
 
     # Execute
@@ -143,6 +149,9 @@ async def test_get_tag_values_success(api_activity):
     assert result[0]['value'] == 10.5
     assert result[1]['name'] == 'tag2'
     assert result[2]['name'] == 'tag3'
+    assert result[0]['timestamp'] == '2023-01-01 12:00:00+0000'
+    assert result[1]['timestamp'] == '2023-01-01 12:01:00+0000'
+    assert result[2]['timestamp'] == '2023-01-01 12:02:00+0000'
 
 
 @pytest.mark.asyncio
@@ -165,13 +174,15 @@ async def test_get_tag_values_with_default_max_count(api_activity):
 
     mock_df = pd.DataFrame(
         {
-            'timestamp': ['2023-01-01 12:00:00'],
+            'timestamp': ['2023-01-01 12:00:00+0000'],
             'name': ['tag1'],
             'value': [42.0],
             'tag': ['webid1'],
         }
     )
 
+    mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp'])
+
     api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df)
 
     # Execute
@@ -217,13 +228,15 @@ async def test_get_tag_values_with_none_webids(api_activity):
 
     mock_df = pd.DataFrame(
         {
-            'timestamp': ['2023-01-01 12:00:00', '2023-01-01 12:01:00'],
+            'timestamp': ['2023-01-01 12:00:00+0000', '2023-01-01 12:01:00+0000'],
             'name': ['tag1', 'tag3'],
             'value': [10.5, 30.7],
             'tag': ['webid1', 'webid3'],
         }
     )
 
+    mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp'])
+
     api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df)
 
     # Execute
@@ -303,13 +316,15 @@ async def test_get_tag_values_with_nan_values(api_activity):
     # Mock DataFrame with NaN values
     mock_df = pd.DataFrame(
         {
-            'timestamp': ['2023-01-01 12:00:00', '2023-01-01 12:00:00'],
+            'timestamp': ['2023-01-01 12:00:00+0000', '2023-01-01 12:00:00+0000'],
             'name': ['tag1', 'tag2'],
             'value': [10.0, float('nan')],
             'tag': ['webid1', 'webid2'],
         }
     )
 
+    mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp'])
+
     api_activity.pi_web_api_client.get_latest_values_df = AsyncMock(return_value=mock_df)
 
     # Execute

tests/utils/test_connectors_config.py

@@ -173,9 +173,9 @@ def test_build_api_config_with_env_vars():
     with patch.dict(
         os.environ,
         {
-            'API_BASE_URL': 'https://api.production.com',
+            'PI_WEB_API_BASE_URL': 'https://api.production.com',
-            'API_AUTH_TYPE': 'bearer',
+            'PI_WEB_API_AUTH_TYPE': 'bearer',
-            'API_AUTH_TOKEN': 'secret_token_123',
+            'PI_WEB_API_AUTH_TOKEN': 'secret_token_123',
         },
     ):
         config = build_api_config()

tests/workflow/test_pi_web_api_scouter.py

@@ -20,16 +20,18 @@ async def test_pi_web_api_scouter_workflow(mock_workflow, pi_web_api_scouter):
             'model_name': 'test_model',
             'model_id': 'test_model_id',
             'schedule_name': 'test_schedule',
-            'endpoint': '/streamsets/recorded',
             'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'},
-            'period': {'start_time': '2025-01-01T00:00:00Z'},
-            'api_timeout': 30,
-            'max_count': 10,
             'trigger_laborious': True,
             'filters': {'quality': 'good'},
             'schema': 'test_schema',
             'table_name': 'test_table',
             'retention_time': 3600,
+            'pi_web_api_query': {
+                'endpoint': '/streamsets/recorded',
+                'period': '*-1d',
+                'max_count': 10,
+                'api_timeout': 30,
+            },
         }
     )
 
@@ -48,7 +50,7 @@ async def test_pi_web_api_scouter_workflow(mock_workflow, pi_web_api_scouter):
             **expected_metadata,
             'endpoint': '/streamsets/recorded',
             'web_ids': {'tag1': 'webid1', 'tag2': 'webid2'},
-            'period': {'start_time': '2025-01-01T00:00:00Z'},
+            'period': '*-1d',
             'api_timeout': 30,
             'max_count': 10,
         },
@@ -62,11 +64,7 @@ async def test_pi_web_api_scouter_workflow(mock_workflow, pi_web_api_scouter):
             'model_name': 'test_model',
             'model_id': 'test_model_id',
             'schedule_name': 'test_schedule',
-            'endpoint': '/streamsets/recorded',
             'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'},
-            'period': {'start_time': '2025-01-01T00:00:00Z'},
-            'api_timeout': 30,
-            'max_count': 10,
             'trigger_laborious': True,
             'filters': {'quality': 'good'},
             'schema': 'test_schema',
@@ -75,6 +73,12 @@ async def test_pi_web_api_scouter_workflow(mock_workflow, pi_web_api_scouter):
             'workflow_name': 'scouter',
             'data': 'test_data',
             'metadata': expected_metadata,
+            'pi_web_api_query': {
+                'endpoint': '/streamsets/recorded',
+                'period': '*-1d',
+                'max_count': 10,
+                'api_timeout': 30,
+            },
         },
     )
 
@@ -88,15 +92,18 @@ async def test_pi_web_api_scouter_workflow_empty(mock_workflow, pi_web_api_scout
             'model_name': 'test_model',
             'model_id': 'test_model_id',
             'schedule_name': 'test_schedule',
-            'endpoint': '/streamsets/recorded',
             'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'},
-            'period': {'start_time': '2025-01-01T00:00:00Z'},
-            'api_timeout': 30,
             'trigger_laborious': True,
             'filters': {'quality': 'good'},
             'schema': 'test_schema',
             'table_name': 'test_table',
             'retention_time': 3600,
+            'pi_web_api_query': {
+                'endpoint': '/streamsets/recorded',
+                'period': '*-1d',
+                'max_count': 1,
+                'api_timeout': 30,
+            },
         }
     )
 
@@ -113,11 +120,11 @@ async def test_pi_web_api_scouter_workflow_empty(mock_workflow, pi_web_api_scout
         Activities.get_tag_values,
         {
             **expected_metadata,
-            'endpoint': '/streamsets/recorded',
             'web_ids': {'tag1': 'webid1', 'tag2': 'webid2'},
-            'period': {'start_time': '2025-01-01T00:00:00Z'},
+            'period': '*-1d',
             'api_timeout': 30,
             'max_count': 1,
+            'endpoint': '/streamsets/recorded',
         },
         start_to_close_timeout=ANY,
         retry_policy=ANY,