SIENTIAPDE-1005
Implement workflows for fake data generation, scouter processing, and core scouter operations - Added `FakeData` workflow to generate random data and send it to a Kafka topic. - Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow. - Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL. - Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling. - Enhanced Redis and Postgres activities with robust testing for data handling and error notifications. - Introduced quality filters for data validation and implemented tests to verify their functionality.
This commit is contained in:
45
input_sample.json
Normal file
45
input_sample.json
Normal file
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"topic": "fake_data",
|
||||
"workflow_name": "scouter-fake-pipeline",
|
||||
"schedule_name": "scouter-fake-pipeline",
|
||||
"model_name": "fake_model",
|
||||
"model_id": 1,
|
||||
"trigger_laborious": false,
|
||||
"filters": {
|
||||
"NULL_VALUES_FILTER": {
|
||||
"policy": "KEEP"
|
||||
},
|
||||
"OUT_OF_BOUNDS_FILTER": {
|
||||
"policy": "DISCARD"
|
||||
}
|
||||
},
|
||||
"schema": "fake_schema",
|
||||
"table_name": "fake_table",
|
||||
"retention_time": 3600,
|
||||
"model_tags": {
|
||||
"Temperature Sensor": {
|
||||
"data_range": [0, 50],
|
||||
"aggr_function": "lts"
|
||||
},
|
||||
"Vibration Meter": {
|
||||
"data_range": [0, 50],
|
||||
"aggr_function": "mdn"
|
||||
},
|
||||
"Pressure Gauge": {
|
||||
"data_range": [0, 100],
|
||||
"aggr_function": "avg"
|
||||
},
|
||||
"Flow Meter": {
|
||||
"data_range": [0, 100],
|
||||
"aggr_function": "max"
|
||||
},
|
||||
"Voltage Sensor": {
|
||||
"data_range": [0, 100],
|
||||
"aggr_function": "min"
|
||||
},
|
||||
"Current Sensor": {
|
||||
"data_range": [0, 100],
|
||||
"aggr_function": "avg"
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user