gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1316

Browse Source 
Update .gitignore and refactor metrics.py, activities.py, and gates.py for improved clarity and consistency. Added coverage.xml and cache directories to .gitignore. Standardized string formatting and parameter handling in metrics and activities classes, enhancing code readability. Removed the deprecated faker.py file and adjusted related tests accordingly.
This commit is contained in:
vitor-aignosi
2025-10-16 13:31:12 -03:00
parent 8bdbf049b8
commit 97eb5bc904
27 changed files with 1101 additions and 1336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
scouter/workflow/fake_data.py
View File

@@ -1,57 +0,0 @@
from temporalio import workflow
with workflow.unsafe.imports_passed_through():
from scouter.activities.faker import Faker
from datetime import timedelta
from typing import Dict, Any
from sientia_do.temporal.policies import retry_policy
@workflow.defn(name="fake_data")
class FakeData:
"""
Test data generation workflow for development and testing purposes.
This workflow generates synthetic industrial sensor data and publishes it to
Kafka topics. It's designed for:
- Development and testing of data processing pipelines
- Load testing of downstream systems
- Demonstration of data flow patterns
- Validation of data quality filters and aggregation functions
The generated data simulates realistic industrial sensor readings with
configurable message counts and topic routing.
"""
@workflow.run
async def run(self, workflow_input: Dict[str, Any]) -> str:
"""
Execute the fake data generation workflow.
This method generates synthetic sensor data and publishes it to the specified
Kafka topic. The data includes realistic industrial sensor readings with
configurable parameters for testing and development purposes.
Args:
workflow_input (dict[str, Any]): Workflow configuration parameters.
Required fields:
- topic (str): Kafka topic name for data publication
- metadata (dict[str, Any], optional): Workflow execution metadata
- num_messages (int, optional): Number of messages to generate.
Defaults to random count between 1 and available sensor tags.
Returns:
str: Success confirmation message
Raises:
WorkflowExecutionError: If workflow execution fails
ActivityExecutionError: If data generation or Kafka publishing fails
"""
await workflow.execute_activity_method(
Faker.generate_and_send_data,
{
'topic': workflow_input['topic']
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
)

29
scouter/workflow/scouter.py
View File

@@ -1,13 +1,15 @@
from temporalio import workflow
with workflow.unsafe.imports_passed_through():
from scouter.activities.activities import Activities
from typing import Any
from datetime import timedelta
from typing import Any
from sientia_do.temporal.policies import retry_policy
from scouter.activities.activities import Activities
@workflow.defn(name="scouter")
@workflow.defn(name='scouter')
class Scouter:
"""
Main Scouter workflow that orchestrates data ingestion and processing.
@@ -67,7 +69,7 @@ class Scouter:
'model_id': input_data['model_id'],
'model_name': input_data['model_name'],
'schedule_name': input_data['schedule_name'],
'workflow_name': input_data['workflow_name']
'workflow_name': input_data['workflow_name'],
}
}
@@ -76,21 +78,21 @@ class Scouter:
{
**metadata,
'workflow_name': input_data['workflow_name'],
'schedule_name': input_data['schedule_name']
'schedule_name': input_data['schedule_name'],
},
start_to_close_timeout=timedelta(seconds=60),
retry_policy=retry_policy
retry_policy=retry_policy,
)
data = await workflow.execute_local_activity_method(
Activities.load_latest_data,
{
**metadata,
'collection_name': f"raw_{input_data['schedule_name']}",
'last_data_timestamp': last_data_timestamp
'collection_name': f'raw_{input_data["schedule_name"]}',
'last_data_timestamp': last_data_timestamp,
},
start_to_close_timeout=timedelta(seconds=60),
retry_policy=retry_policy
retry_policy=retry_policy,
)
if data == {}:
@@ -102,16 +104,13 @@ class Scouter:
**metadata,
'data': data,
'workflow_name': input_data['workflow_name'],
'schedule_name': input_data['schedule_name']
'schedule_name': input_data['schedule_name'],
},
start_to_close_timeout=timedelta(seconds=60),
retry_policy=retry_policy
retry_policy=retry_policy,
)
input_data['data'] = data
input_data['metadata'] = metadata
await workflow.execute_child_workflow(
'core_scouter',
input_data
)
await workflow.execute_child_workflow('core_scouter', input_data)

37
scouter/workflow/sub_workflows/core_scouter.py
View File

@@ -1,14 +1,16 @@
from temporalio import workflow
with workflow.unsafe.imports_passed_through():
from scouter.activities.activities import Activities
from typing import Any
from datetime import timedelta
from sientia_do.temporal.policies import retry_policy
from typing import Any
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from sientia_do.temporal.policies import retry_policy
from scouter.activities.activities import Activities
@workflow.defn(name="core_scouter")
@workflow.defn(name='core_scouter')
class CoreScouter:
"""
Core data processing workflow that handles data quality, aggregation, and export.
@@ -68,21 +70,17 @@ class CoreScouter:
**metadata,
'filters': input_data['filters'],
'data': input_data['data'],
'model_tags': input_data['model_tags']
'model_tags': input_data['model_tags'],
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)
grouped_data = await workflow.execute_local_activity_method(
Activities.aggregate_data,
{
**metadata,
'data': filtered_data,
'model_tags': input_data['model_tags']
},
{**metadata, 'data': filtered_data, 'model_tags': input_data['model_tags']},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)
held_data = await workflow.execute_local_activity_method(
@@ -94,10 +92,10 @@ class CoreScouter:
'data': grouped_data,
'model_id': input_data['model_id'],
'model_tags': input_data['model_tags'],
'retention_time': input_data['retention_time']
'retention_time': input_data['retention_time'],
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)
if held_data == {}:
@@ -110,13 +108,10 @@ class CoreScouter:
'schema': input_data['schema'],
'table_name': input_data['table_name'],
'data': held_data,
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_WITH_TZ
}
'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ},
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)
await workflow.execute_activity_method(
@@ -125,7 +120,7 @@ class CoreScouter:
**metadata,
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)
if input_data.get('debug_data_package', False):
@@ -139,5 +134,5 @@ class CoreScouter:
'schedule_name': input_data['schedule_name'],
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)