SIENTIAPDE-1445

Enhance Activities and API Integration

- Updated Activities class to include API operations for external data ingestion.
- Added API configuration builder to connectors_config.py for environment variable management.
- Integrated API configuration into worker setup.
- Expanded unit tests to cover new API functionality and configuration handling.
- Updated requirements.txt to include pycurl and prometheus-client for enhanced metrics support.

tests/workflow/test_pi_web_api_scouter.py

@@ -0,0 +1,126 @@
+from unittest.mock import ANY, AsyncMock, patch
+
+from pytest import fixture, mark
+
+from scouter.activities.activities import Activities
+from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter
+
+
+@fixture
+def pi_web_api_scouter():
+    return PIWebAPIScouter()
+
+
+@mark.asyncio
+@patch('scouter.workflow.pi_web_api_scouter.workflow', new_callable=AsyncMock)
+async def test_pi_web_api_scouter_workflow(mock_workflow, pi_web_api_scouter):
+    mock_workflow.execute_local_activity_method.return_value = 'test_data'
+    await pi_web_api_scouter.run(
+        input_data={
+            'model_name': 'test_model',
+            'model_id': 'test_model_id',
+            'schedule_name': 'test_schedule',
+            'endpoint': '/streamsets/recorded',
+            'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'},
+            'period': {'start_time': '2025-01-01T00:00:00Z'},
+            'api_timeout': 30,
+            'max_count': 10,
+            'trigger_laborious': True,
+            'filters': {'quality': 'good'},
+            'schema': 'test_schema',
+            'table_name': 'test_table',
+            'retention_time': 3600,
+        }
+    )
+
+    expected_metadata = {
+        'metadata': {
+            'model_id': 'test_model_id',
+            'model_name': 'test_model',
+            'schedule_name': 'test_schedule',
+            'workflow_name': 'scouter',
+        }
+    }
+
+    mock_workflow.execute_local_activity_method.assert_called_once_with(
+        Activities.get_tag_values,
+        {
+            **expected_metadata,
+            'endpoint': '/streamsets/recorded',
+            'web_ids': {'tag1': 'webid1', 'tag2': 'webid2'},
+            'period': {'start_time': '2025-01-01T00:00:00Z'},
+            'api_timeout': 30,
+            'max_count': 10,
+        },
+        start_to_close_timeout=ANY,
+        retry_policy=ANY,
+    )
+
+    mock_workflow.execute_child_workflow.assert_called_once_with(
+        'subworkflow.core_scouter',
+        {
+            'model_name': 'test_model',
+            'model_id': 'test_model_id',
+            'schedule_name': 'test_schedule',
+            'endpoint': '/streamsets/recorded',
+            'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'},
+            'period': {'start_time': '2025-01-01T00:00:00Z'},
+            'api_timeout': 30,
+            'max_count': 10,
+            'trigger_laborious': True,
+            'filters': {'quality': 'good'},
+            'schema': 'test_schema',
+            'table_name': 'test_table',
+            'retention_time': 3600,
+            'workflow_name': 'scouter',
+            'data': 'test_data',
+            'metadata': expected_metadata,
+        },
+    )
+
+
+@mark.asyncio
+@patch('scouter.workflow.pi_web_api_scouter.workflow', new_callable=AsyncMock)
+async def test_pi_web_api_scouter_workflow_empty(mock_workflow, pi_web_api_scouter):
+    mock_workflow.execute_local_activity_method.return_value = []
+    await pi_web_api_scouter.run(
+        input_data={
+            'model_name': 'test_model',
+            'model_id': 'test_model_id',
+            'schedule_name': 'test_schedule',
+            'endpoint': '/streamsets/recorded',
+            'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'},
+            'period': {'start_time': '2025-01-01T00:00:00Z'},
+            'api_timeout': 30,
+            'trigger_laborious': True,
+            'filters': {'quality': 'good'},
+            'schema': 'test_schema',
+            'table_name': 'test_table',
+            'retention_time': 3600,
+        }
+    )
+
+    expected_metadata = {
+        'metadata': {
+            'model_id': 'test_model_id',
+            'model_name': 'test_model',
+            'schedule_name': 'test_schedule',
+            'workflow_name': 'scouter',
+        }
+    }
+
+    mock_workflow.execute_local_activity_method.assert_called_once_with(
+        Activities.get_tag_values,
+        {
+            **expected_metadata,
+            'endpoint': '/streamsets/recorded',
+            'web_ids': {'tag1': 'webid1', 'tag2': 'webid2'},
+            'period': {'start_time': '2025-01-01T00:00:00Z'},
+            'api_timeout': 30,
+            'max_count': 1,
+        },
+        start_to_close_timeout=ANY,
+        retry_policy=ANY,
+    )
+
+    mock_workflow.execute_child_workflow.assert_not_called()