Code import - branch feature/SIENTIAPDE-1646

scouter/workflow/pi_web_api_scouter.py

@@ -0,0 +1,108 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from datetime import timedelta
+    from typing import Any
+
+    from sientia_do.temporal.policies import retry_policy
+
+    from scouter.activities.activities import Activities
+
+
+@workflow.defn(name='pi_web_api_scouter')
+class PIWebAPIScouter:
+    """
+    PI Web API Scouter workflow that orchestrates data ingestion from PI systems.
+
+    This workflow serves as the entry point for PI Web API data processing pipelines.
+    Unlike the standard Scouter that loads from MongoDB, this workflow directly queries
+    PI Web API endpoints to retrieve tag values and processes them for downstream use.
+
+    The workflow implements a direct API ingestion pattern with:
+    - Real-time data retrieval from PI Web API
+    - Configurable time periods and data point limits
+    - Error handling and retry policies
+    - Child workflow orchestration for data processing
+    - Integration with CoreScouter for standardized processing
+    """
+
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]) -> None:
+        """
+        Execute the PI Web API Scouter workflow.
+
+        This method orchestrates the complete data ingestion process from PI Web API:
+        1. Retrieves tag values from PI Web API using configured WebIds
+        2. Validates and normalizes the retrieved data (timestamps are normalized)
+        3. Delegates data processing to the CoreScouter workflow
+
+        If no data is retrieved from the PI Web API, the workflow exits early without
+        invoking the CoreScouter workflow.
+
+        Args:
+            input_data (dict[str, Any]): Configuration and parameters for the workflow execution.
+                Required fields:
+                - schedule_name (str): Unique identifier for the data collection schedule
+                - model_name (str): Name of the data model being processed
+                - model_id (str): Unique identifier for the data model
+                - pi_web_api_query (dict[str, Any]): PI Web API query configuration containing:
+                    - endpoint (str): PI Web API endpoint path (e.g., '/streamsets/recorded')
+                    - period (str): Time period configuration (e.g., '*-1d', '*-1h')
+                    - api_timeout (int): Request timeout in seconds for PI Web API calls
+                    - max_count (int, optional): Maximum data points per tag. Defaults to 1
+                - trigger_laborious (bool): Flag to enable intensive data processing
+                - filters (dict[str, str]): Data quality filters configuration
+                - schema (str): Target database schema for data export
+                - table_name (str): Target table name for data export
+                - retention_time (int): Data retention period in Redis (seconds)
+                - model_tags (dict[str, Any]): Tag-specific configuration mapping tag names
+                  to WebIds and processing rules, including:
+                    - webid (str): PI Web API WebId for the tag
+                    - data_range: [min, max] values for data validation
+                    - aggr_func: Aggregation method (avg, mdn, max, min, lts)
+                    - frequency: Data collection frequency in milliseconds
+                    - topics: List of Kafka topics for data routing
+
+        Returns:
+            None: This workflow doesn't return data, it orchestrates data processing
+
+        Raises:
+            WorkflowExecutionError: If workflow execution fails
+            ActivityExecutionError: If any activity fails after retry attempts
+            PIMSRequestError: If PI Web API request fails
+        """
+
+        input_data['workflow_name'] = 'pi_web_api_scouter'
+
+        metadata = {
+            'metadata': {
+                'model_id': input_data['model_id'],
+                'model_name': input_data['model_name'],
+                'schedule_name': input_data['schedule_name'],
+                'workflow_name': input_data['workflow_name'],
+            }
+        }
+
+        pi_web_api_query = input_data['pi_web_api_query']
+
+        data = await workflow.execute_local_activity_method(
+            Activities.get_tag_values,
+            {
+                **metadata,
+                'endpoint': pi_web_api_query['endpoint'],
+                'web_ids': input_data['model_tags'],
+                'period': pi_web_api_query['period'],
+                'max_count': pi_web_api_query.get('max_count', 1),
+                'api_timeout': pi_web_api_query['api_timeout'],
+            },
+            start_to_close_timeout=timedelta(seconds=60),
+            retry_policy=retry_policy,
+        )
+
+        if not data:
+            return
+
+        input_data['data'] = data
+        input_data['metadata'] = metadata
+
+        await workflow.execute_child_workflow('subworkflow.core_scouter', input_data)

scouter/workflow/scouter.py

@@ -0,0 +1,120 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from datetime import timedelta
+    from typing import Any
+
+    from sientia_do.temporal.policies import retry_policy
+
+    from scouter.activities.activities import Activities
+
+
+@workflow.defn(name='scouter')
+class Scouter:
+    """
+    Main Scouter workflow that orchestrates data ingestion and processing.
+
+    This workflow serves as the entry point for data processing pipelines. It loads
+    data from MongoDB collections, manages data timestamps for incremental processing,
+    and delegates the actual data processing to the CoreScouter workflow.
+
+    The workflow implements a robust data ingestion pattern with:
+    - Incremental data loading based on last processed timestamp
+    - Automatic timestamp management for data continuity
+    - Error handling and retry policies
+    - Child workflow orchestration for data processing
+    """
+
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]) -> None:
+        """
+        Execute the main Scouter workflow.
+
+        This method orchestrates the complete data ingestion process:
+        1. Retrieves the last processed timestamp from Redis
+        2. Loads new data from MongoDB since the last timestamp using collection name
+           format: `raw_{schedule_name}`
+        3. Updates the last processed timestamp with the most recent data point
+        4. Delegates data processing to the CoreScouter workflow
+
+        If no new data is found in MongoDB, the workflow exits early without updating
+        the timestamp or invoking the CoreScouter workflow.
+
+        Args:
+            input_data (dict[str, Any]): Configuration and parameters for the workflow execution.
+                Required fields:
+                - topic (str): The Kafka topic name for data source identification
+                - schedule_name (str): Unique identifier for the data collection schedule
+                - model_name (str): Name of the data model being processed
+                - model_id (str): Unique identifier for the data model
+                - trigger_laborious (bool): Flag to enable intensive data processing
+                - filters (dict[str, str]): Data quality filters configuration
+                - schema (str): Target database schema for data export
+                - table_name (str): Target table name for data export
+                - retention_time (int): Data retention period in Redis (seconds)
+                - model_tags (dict[str, Any]): Tag-specific configuration including:
+                    - data_range: [min, max] values for data validation
+                    - aggr_function: Aggregation method (avg, mdn, max, min, lts)
+                    - frequency: Data collection frequency in milliseconds
+                    - topics: List of Kafka topics for data routing
+
+        Returns:
+            None: This workflow doesn't return data, it orchestrates data processing
+
+        Raises:
+            WorkflowExecutionError: If workflow execution fails
+            ActivityExecutionError: If any activity fails after retry attempts
+        """
+
+        input_data['workflow_name'] = 'scouter'
+
+        metadata = {
+            'metadata': {
+                'model_id': input_data['model_id'],
+                'model_name': input_data['model_name'],
+                'schedule_name': input_data['schedule_name'],
+                'workflow_name': input_data['workflow_name'],
+            }
+        }
+
+        last_data_timestamp = await workflow.execute_local_activity_method(
+            Activities.get_last_data_timestamp,
+            {
+                **metadata,
+                'workflow_name': input_data['workflow_name'],
+                'schedule_name': input_data['schedule_name'],
+            },
+            start_to_close_timeout=timedelta(seconds=60),
+            retry_policy=retry_policy,
+        )
+
+        data = await workflow.execute_local_activity_method(
+            Activities.load_latest_data,
+            {
+                **metadata,
+                'collection_name': f'raw_{input_data["schedule_name"]}',
+                'last_data_timestamp': last_data_timestamp,
+            },
+            start_to_close_timeout=timedelta(seconds=60),
+            retry_policy=retry_policy,
+        )
+
+        if not data:
+            return
+
+        await workflow.execute_activity_method(
+            Activities.put_last_data_timestamp,
+            {
+                **metadata,
+                'data': data,
+                'workflow_name': input_data['workflow_name'],
+                'schedule_name': input_data['schedule_name'],
+            },
+            start_to_close_timeout=timedelta(seconds=60),
+            retry_policy=retry_policy,
+        )
+
+        input_data['data'] = data
+        input_data['metadata'] = metadata
+
+        await workflow.execute_child_workflow('subworkflow.core_scouter', input_data)

scouter/workflow/sub_workflows/core_scouter.py

@@ -0,0 +1,152 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from datetime import timedelta
+    from typing import Any
+
+    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
+    from sientia_do.temporal.policies import retry_policy
+
+    from scouter.activities.activities import Activities
+
+
+@workflow.defn(name='subworkflow.core_scouter')
+class CoreScouter:
+    """
+    Core data processing workflow that handles data quality, aggregation, and export.
+
+    This workflow implements the core data processing pipeline for industrial data:
+    - Data quality validation and filtering
+    - Time-series data aggregation using configurable functions
+    - Data grouping and temporary storage in Redis
+    - Asynchronous export to PostgreSQL for persistent storage
+    - Metrics collection and monitoring
+
+    The workflow is designed for high-throughput data processing with configurable
+    quality gates and aggregation strategies. It is typically invoked as a child
+    workflow by parent workflows such as Scouter or PIWebAPIScouter.
+    """
+
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]) -> None:
+        """
+        Execute the core data processing workflow.
+
+        This method processes industrial time-series data through a series of stages:
+        1. Data Quality Gate: Applies configurable filters for data validation
+        2. Data Aggregation: Groups and aggregates data using specified functions
+        3. Data Grouping: Organizes data by tags and applies retention policies
+        4. Data Export: Persists processed data to PostgreSQL with timestamp conversion
+        5. Metrics Collection: Records processing metrics for monitoring
+
+        The workflow implements early exit conditions:
+        - If held_data is empty after grouping, the workflow exits without exporting
+        - If data export results in zero or negative affected_rows, the workflow exits
+          without writing metrics or storing debug packages
+
+        Args:
+            input_data (dict[str, Any]): Complete workflow configuration and data.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - workflow_name (str): Name of the parent workflow
+                - schedule_name (str): Data collection schedule identifier
+                - model_name (str): Data model name
+                - model_id (str): Unique model identifier
+                - data (dict[str, Any]): Raw time-series data to process
+                - trigger_laborious (bool): Enable intensive processing mode
+                - filters (dict[str, str]): Data quality filter configurations
+                - schema (str): Target database schema
+                - table_name (str): Target database table
+                - retention_time (int): Redis data retention period (seconds)
+                - model_tags (dict[str, Any]): Tag-specific processing rules
+                - fill_missing_tags (bool): Enable filling of missing tag values
+                - debug_data_package (bool, optional): Store data packages for debugging.
+                  When True, stores both raw and processed data in MongoDB for debugging
+
+        Returns:
+            None: This workflow processes data but doesn't return results
+
+        Raises:
+            WorkflowExecutionError: If workflow execution fails
+            ActivityExecutionError: If any activity fails after retry attempts
+        """
+
+        metadata = input_data['metadata']
+
+        filtered_data = await workflow.execute_local_activity_method(
+            Activities.data_quality_gate,
+            {
+                **metadata,
+                'filters': input_data['filters'],
+                'data': input_data['data'],
+                'model_tags': input_data['model_tags'],
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=60),
+        )
+
+        grouped_data = await workflow.execute_local_activity_method(
+            Activities.aggregate_data,
+            {**metadata, 'data': filtered_data, 'model_tags': input_data['model_tags']},
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=60),
+        )
+
+        held_data = await workflow.execute_local_activity_method(
+            Activities.group_and_hold_data,
+            {
+                **metadata,
+                'schedule_name': input_data['schedule_name'],
+                'workflow_name': input_data['workflow_name'],
+                'data': grouped_data,
+                'model_id': input_data['model_id'],
+                'model_tags': input_data['model_tags'],
+                'retention_time': input_data['retention_time'],
+                'fill_missing_tags': input_data['fill_missing_tags'],
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=60),
+        )
+
+        if held_data == {}:
+            return
+
+        data_exported = await workflow.execute_activity_method(
+            Activities.export_data_to_postgres,
+            {
+                **metadata,
+                'schema': input_data['schema'],
+                'table_name': input_data['table_name'],
+                'data': held_data,
+                'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ},
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=60),
+        )
+
+        if data_exported.get('affected_rows', 0) <= 0:
+            return
+
+        await workflow.execute_activity_method(
+            Activities.write_metrics,
+            {
+                **metadata,
+                'tag_values': held_data,
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=60),
+        )
+
+        if input_data.get('debug_data_package', False):
+            await workflow.execute_activity_method(
+                Activities.store_data_package,
+                {
+                    **metadata,
+                    'data': input_data['data'],
+                    'held_data': held_data,
+                    'workflow_name': input_data['workflow_name'],
+                    'schedule_name': input_data['schedule_name'],
+                },
+                retry_policy=retry_policy,
+                start_to_close_timeout=timedelta(seconds=60),
+            )