Code import - branch feature/SIENTIAPDE-1646
This commit is contained in:
152
scouter/workflow/sub_workflows/core_scouter.py
Normal file
152
scouter/workflow/sub_workflows/core_scouter.py
Normal file
@@ -0,0 +1,152 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from scouter.activities.activities import Activities
|
||||
|
||||
|
||||
@workflow.defn(name='subworkflow.core_scouter')
|
||||
class CoreScouter:
|
||||
"""
|
||||
Core data processing workflow that handles data quality, aggregation, and export.
|
||||
|
||||
This workflow implements the core data processing pipeline for industrial data:
|
||||
- Data quality validation and filtering
|
||||
- Time-series data aggregation using configurable functions
|
||||
- Data grouping and temporary storage in Redis
|
||||
- Asynchronous export to PostgreSQL for persistent storage
|
||||
- Metrics collection and monitoring
|
||||
|
||||
The workflow is designed for high-throughput data processing with configurable
|
||||
quality gates and aggregation strategies. It is typically invoked as a child
|
||||
workflow by parent workflows such as Scouter or PIWebAPIScouter.
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]) -> None:
|
||||
"""
|
||||
Execute the core data processing workflow.
|
||||
|
||||
This method processes industrial time-series data through a series of stages:
|
||||
1. Data Quality Gate: Applies configurable filters for data validation
|
||||
2. Data Aggregation: Groups and aggregates data using specified functions
|
||||
3. Data Grouping: Organizes data by tags and applies retention policies
|
||||
4. Data Export: Persists processed data to PostgreSQL with timestamp conversion
|
||||
5. Metrics Collection: Records processing metrics for monitoring
|
||||
|
||||
The workflow implements early exit conditions:
|
||||
- If held_data is empty after grouping, the workflow exits without exporting
|
||||
- If data export results in zero or negative affected_rows, the workflow exits
|
||||
without writing metrics or storing debug packages
|
||||
|
||||
Args:
|
||||
input_data (dict[str, Any]): Complete workflow configuration and data.
|
||||
Required fields:
|
||||
- metadata (dict[str, Any]): Workflow execution metadata
|
||||
- workflow_name (str): Name of the parent workflow
|
||||
- schedule_name (str): Data collection schedule identifier
|
||||
- model_name (str): Data model name
|
||||
- model_id (str): Unique model identifier
|
||||
- data (dict[str, Any]): Raw time-series data to process
|
||||
- trigger_laborious (bool): Enable intensive processing mode
|
||||
- filters (dict[str, str]): Data quality filter configurations
|
||||
- schema (str): Target database schema
|
||||
- table_name (str): Target database table
|
||||
- retention_time (int): Redis data retention period (seconds)
|
||||
- model_tags (dict[str, Any]): Tag-specific processing rules
|
||||
- fill_missing_tags (bool): Enable filling of missing tag values
|
||||
- debug_data_package (bool, optional): Store data packages for debugging.
|
||||
When True, stores both raw and processed data in MongoDB for debugging
|
||||
|
||||
Returns:
|
||||
None: This workflow processes data but doesn't return results
|
||||
|
||||
Raises:
|
||||
WorkflowExecutionError: If workflow execution fails
|
||||
ActivityExecutionError: If any activity fails after retry attempts
|
||||
"""
|
||||
|
||||
metadata = input_data['metadata']
|
||||
|
||||
filtered_data = await workflow.execute_local_activity_method(
|
||||
Activities.data_quality_gate,
|
||||
{
|
||||
**metadata,
|
||||
'filters': input_data['filters'],
|
||||
'data': input_data['data'],
|
||||
'model_tags': input_data['model_tags'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
grouped_data = await workflow.execute_local_activity_method(
|
||||
Activities.aggregate_data,
|
||||
{**metadata, 'data': filtered_data, 'model_tags': input_data['model_tags']},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
held_data = await workflow.execute_local_activity_method(
|
||||
Activities.group_and_hold_data,
|
||||
{
|
||||
**metadata,
|
||||
'schedule_name': input_data['schedule_name'],
|
||||
'workflow_name': input_data['workflow_name'],
|
||||
'data': grouped_data,
|
||||
'model_id': input_data['model_id'],
|
||||
'model_tags': input_data['model_tags'],
|
||||
'retention_time': input_data['retention_time'],
|
||||
'fill_missing_tags': input_data['fill_missing_tags'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
if held_data == {}:
|
||||
return
|
||||
|
||||
data_exported = await workflow.execute_activity_method(
|
||||
Activities.export_data_to_postgres,
|
||||
{
|
||||
**metadata,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name'],
|
||||
'data': held_data,
|
||||
'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ},
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
if data_exported.get('affected_rows', 0) <= 0:
|
||||
return
|
||||
|
||||
await workflow.execute_activity_method(
|
||||
Activities.write_metrics,
|
||||
{
|
||||
**metadata,
|
||||
'tag_values': held_data,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
if input_data.get('debug_data_package', False):
|
||||
await workflow.execute_activity_method(
|
||||
Activities.store_data_package,
|
||||
{
|
||||
**metadata,
|
||||
'data': input_data['data'],
|
||||
'held_data': held_data,
|
||||
'workflow_name': input_data['workflow_name'],
|
||||
'schedule_name': input_data['schedule_name'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
Reference in New Issue
Block a user