b203b7d22c
Implement workflows for fake data generation, scouter processing, and core scouter operations - Added `FakeData` workflow to generate random data and send it to a Kafka topic. - Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow. - Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL. - Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling. - Enhanced Redis and Postgres activities with robust testing for data handling and error notifications. - Introduced quality filters for data validation and implemented tests to verify their functionality.
45 lines
1.1 KiB
JSON
45 lines
1.1 KiB
JSON
{
|
|
"topic": "fake_data",
|
|
"workflow_name": "scouter-fake-pipeline",
|
|
"schedule_name": "scouter-fake-pipeline",
|
|
"model_name": "fake_model",
|
|
"model_id": 1,
|
|
"trigger_laborious": false,
|
|
"filters": {
|
|
"NULL_VALUES_FILTER": {
|
|
"policy": "KEEP"
|
|
},
|
|
"OUT_OF_BOUNDS_FILTER": {
|
|
"policy": "DISCARD"
|
|
}
|
|
},
|
|
"schema": "fake_schema",
|
|
"table_name": "fake_table",
|
|
"retention_time": 3600,
|
|
"model_tags": {
|
|
"Temperature Sensor": {
|
|
"data_range": [0, 50],
|
|
"aggr_function": "lts"
|
|
},
|
|
"Vibration Meter": {
|
|
"data_range": [0, 50],
|
|
"aggr_function": "mdn"
|
|
},
|
|
"Pressure Gauge": {
|
|
"data_range": [0, 100],
|
|
"aggr_function": "avg"
|
|
},
|
|
"Flow Meter": {
|
|
"data_range": [0, 100],
|
|
"aggr_function": "max"
|
|
},
|
|
"Voltage Sensor": {
|
|
"data_range": [0, 100],
|
|
"aggr_function": "min"
|
|
},
|
|
"Current Sensor": {
|
|
"data_range": [0, 100],
|
|
"aggr_function": "avg"
|
|
}
|
|
}
|
|
} |