SIENTIAPDE-1445

Enhance Activities and API Integration

- Updated Activities class to include API operations for external data ingestion.
- Added API configuration builder to connectors_config.py for environment variable management.
- Integrated API configuration into worker setup.
- Expanded unit tests to cover new API functionality and configuration handling.
- Updated requirements.txt to include pycurl and prometheus-client for enhanced metrics support.

scouter/workflow/pi_web_api_scouter.py

@@ -0,0 +1,101 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from datetime import timedelta
+    from typing import Any
+
+    from sientia_do.temporal.policies import retry_policy
+
+    from scouter.activities.activities import Activities
+
+
+@workflow.defn(name='pi_web_api_scouter')
+class PIWebAPIScouter:
+    """
+    PI Web API Scouter workflow that orchestrates data ingestion from PI systems.
+
+    This workflow serves as the entry point for PI Web API data processing pipelines.
+    Unlike the standard Scouter that loads from MongoDB, this workflow directly queries
+    PI Web API endpoints to retrieve tag values and processes them for downstream use.
+
+    The workflow implements a direct API ingestion pattern with:
+    - Real-time data retrieval from PI Web API
+    - Configurable time periods and data point limits
+    - Error handling and retry policies
+    - Child workflow orchestration for data processing
+    - Integration with CoreScouter for standardized processing
+    """
+
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]) -> None:
+        """
+        Execute the PI Web API Scouter workflow.
+
+        This method orchestrates the complete data ingestion process from PI Web API:
+        1. Retrieves tag values from PI Web API using configured WebIds
+        2. Validates and normalizes the retrieved data
+        3. Delegates data processing to the CoreScouter workflow
+
+        Args:
+            input_data (dict[str, Any]): Configuration and parameters for the workflow execution.
+                Required fields:
+                - model_name (str): Name of the data model being processed
+                - model_id (str): Unique identifier for the data model
+                - schedule_name (str): Unique identifier for the data collection schedule
+                - endpoint (str): PI Web API endpoint path (e.g., '/streamsets/recorded')
+                - web_ids (dict[str, str | None]): Mapping of tag names to WebIds
+                - period (dict[str, str]): Time period configuration with 'start_time'
+                - api_timeout (int): Request timeout in seconds for PI Web API calls
+                - max_count (int, optional): Maximum data points per tag. Defaults to 1
+                - trigger_laborious (bool): Flag to enable intensive data processing
+                - filters (dict[str, str]): Data quality filters configuration
+                - schema (str): Target database schema for data export
+                - table_name (str): Target table name for data export
+                - retention_time (int): Data retention period in Redis (seconds)
+                - model_tags (dict[str, Any]): Tag-specific configuration including:
+                    - data_range: [min, max] values for data validation
+                    - aggr_function: Aggregation method (avg, mdn, max, min, lts)
+                    - frequency: Data collection frequency in milliseconds
+                    - topics: List of Kafka topics for data routing
+
+        Returns:
+            None: This workflow doesn't return data, it orchestrates data processing
+
+        Raises:
+            WorkflowExecutionError: If workflow execution fails
+            ActivityExecutionError: If any activity fails after retry attempts
+            PIMSRequestError: If PI Web API request fails
+        """
+
+        input_data['workflow_name'] = 'scouter'
+
+        metadata = {
+            'metadata': {
+                'model_id': input_data['model_id'],
+                'model_name': input_data['model_name'],
+                'schedule_name': input_data['schedule_name'],
+                'workflow_name': input_data['workflow_name'],
+            }
+        }
+
+        data = await workflow.execute_local_activity_method(
+            Activities.get_tag_values,
+            {
+                **metadata,
+                'endpoint': input_data['endpoint'],
+                'web_ids': input_data['model_tags'],
+                'period': input_data['period'],
+                'api_timeout': input_data['api_timeout'],
+                'max_count': input_data.get('max_count', 1),
+            },
+            start_to_close_timeout=timedelta(seconds=60),
+            retry_policy=retry_policy,
+        )
+
+        if not data:
+            return
+
+        input_data['data'] = data
+        input_data['metadata'] = metadata
+
+        await workflow.execute_child_workflow('subworkflow.core_scouter', input_data)