diff --git a/input_samples_30.json b/input_samples_30.json new file mode 100644 index 0000000..3455816 --- /dev/null +++ b/input_samples_30.json @@ -0,0 +1,1568 @@ +[ + { + "id": 2, + "schedule_name": "scouter-opcua-pipeline-2", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 3, + "schedule_name": "scouter-opcua-pipeline-3", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 4, + "schedule_name": "scouter-opcua-pipeline-4", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 5, + "schedule_name": "scouter-opcua-pipeline-5", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 6, + "schedule_name": "scouter-opcua-pipeline-6", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 7, + "schedule_name": "scouter-opcua-pipeline-7", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 8, + "schedule_name": "scouter-opcua-pipeline-8", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 9, + "schedule_name": "scouter-opcua-pipeline-9", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 10, + "schedule_name": "scouter-opcua-pipeline-10", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 11, + "schedule_name": "scouter-opcua-pipeline-11", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 12, + "schedule_name": "scouter-opcua-pipeline-12", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 13, + "schedule_name": "scouter-opcua-pipeline-13", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 14, + "schedule_name": "scouter-opcua-pipeline-14", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 15, + "schedule_name": "scouter-opcua-pipeline-15", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 16, + "schedule_name": "scouter-opcua-pipeline-16", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 17, + "schedule_name": "scouter-opcua-pipeline-17", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 18, + "schedule_name": "scouter-opcua-pipeline-18", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 19, + "schedule_name": "scouter-opcua-pipeline-19", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 20, + "schedule_name": "scouter-opcua-pipeline-20", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 21, + "schedule_name": "scouter-opcua-pipeline-21", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 22, + "schedule_name": "scouter-opcua-pipeline-22", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 23, + "schedule_name": "scouter-opcua-pipeline-23", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 24, + "schedule_name": "scouter-opcua-pipeline-24", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 25, + "schedule_name": "scouter-opcua-pipeline-25", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 26, + "schedule_name": "scouter-opcua-pipeline-26", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 27, + "schedule_name": "scouter-opcua-pipeline-27", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 28, + "schedule_name": "scouter-opcua-pipeline-28", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 29, + "schedule_name": "scouter-opcua-pipeline-29", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 30, + "schedule_name": "scouter-opcua-pipeline-30", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + } +] \ No newline at end of file diff --git a/specs_30.json b/specs_30.json new file mode 100644 index 0000000..60be595 --- /dev/null +++ b/specs_30.json @@ -0,0 +1,3743 @@ +[ + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-2", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-2", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-2", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-3", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-3", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-3", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-4", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-4", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-4", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-5", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-5", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-5", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-6", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-6", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-6", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-7", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-7", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-7", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-8", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-8", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-8", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-9", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-9", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-9", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-10", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-10", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-10", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-11", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-11", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-11", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-12", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-12", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-12", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-13", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-13", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-13", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-14", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-14", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-14", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-15", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-15", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-15", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-16", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-16", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-16", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-17", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-17", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-17", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-18", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-18", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-18", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-19", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-19", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-19", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-20", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-20", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-20", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-21", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-21", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-21", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-22", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-22", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-22", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-23", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-23", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-23", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-24", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-24", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-24", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-25", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-25", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-25", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-26", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-26", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-26", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-27", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-27", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-27", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-28", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-28", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-28", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-29", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-29", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-29", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-30", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-30", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-30", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + } +] \ No newline at end of file diff --git a/test.ipynb b/test.ipynb index fb78908..7e5c2a0 100644 --- a/test.ipynb +++ b/test.ipynb @@ -241,6 +241,463 @@ "\n", "display(result)" ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "data = {\n", + " \"id\": \"1\",\n", + " \"schedule_name\": \"scouter-opcua-orchestrated-pipeline\",\n", + " \"model_id\": \"1\",\n", + " \"workflow_type\": \"scouter\",\n", + " \"frequency\": \"30s\",\n", + " \"max_retry_policy\": 1,\n", + " \"read_tags\": [\n", + " {\n", + " \"tag_name\": \"Counter\",\n", + " \"server_id\": \"1\",\n", + " \"aggr_func\": \"avg\",\n", + " \"tag_address\": \"ns=2;i=2\",\n", + " \"frequency\": \"15000\",\n", + " \"data_range\": [\n", + " -100,\n", + " 100\n", + " ]\n", + " },\n", + " {\n", + " \"tag_name\": \"Rollout\",\n", + " \"server_id\": \"1\",\n", + " \"aggr_func\": \"mdn\",\n", + " \"tag_address\": \"ns=2;i=3\",\n", + " \"frequency\": \"15000\",\n", + " \"data_range\": [\n", + " -100,\n", + " 100\n", + " ]\n", + " },\n", + " {\n", + " \"tag_name\": \"Square\",\n", + " \"server_id\": \"1\",\n", + " \"aggr_func\": \"lts\",\n", + " \"tag_address\": \"ns=2;i=4\",\n", + " \"frequency\": \"15000\",\n", + " \"data_range\": [\n", + " -100,\n", + " 100\n", + " ]\n", + " }\n", + " ],\n", + " \"filters\": [\n", + " {\n", + " \"filter_name\": \"OUT_OF_BOUNDS_FILTER\",\n", + " \"policy\": \"DISCARD\"\n", + " },\n", + " {\n", + " \"filter_name\": \"NULL_VALUES_FILTER\",\n", + " \"policy\": \"DISCARD\"\n", + " }\n", + " ],\n", + " \"tag_retention_minutes\": 60\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "\n", + "# Generate 29 more, changing only id and schedule_name\n", + "json_list = []\n", + "for i in range(30):\n", + " obj = data.copy()\n", + " obj['id'] = i + 1 # or any other unique id logic\n", + " obj['schedule_name'] = f\"scouter-opcua-pipeline-{i+1}\"\n", + " json_list.append(obj)\n", + "\n", + "# Save to a new file\n", + "with open('input_samples_30.json', 'w') as f:\n", + " json.dump(json_list, f, indent=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "spec = {\n", + " \"type\": \"kafka\",\n", + " \"spec\": {\n", + " \"dataSchema\": {\n", + " \"dataSource\": \"raw_scouter-opcua-orchestrated-pipeline\",\n", + " \"timestampSpec\": {\n", + " \"column\": \"kafka.timestamp\",\n", + " \"format\": \"millis\",\n", + " \"missingValue\": None\n", + " },\n", + " \"dimensionsSpec\": {\n", + " \"dimensions\": [],\n", + " \"dimensionExclusions\": [\n", + " \"__time\",\n", + " \"kafka.timestamp\"\n", + " ],\n", + " \"includeAllDimensions\": False,\n", + " \"useSchemaDiscovery\": True\n", + " },\n", + " \"metricsSpec\": [],\n", + " \"granularitySpec\": {\n", + " \"type\": \"uniform\",\n", + " \"segmentGranularity\": \"DAY\",\n", + " \"queryGranularity\": {\n", + " \"type\": \"none\"\n", + " },\n", + " \"rollup\": False,\n", + " \"intervals\": []\n", + " },\n", + " \"transformSpec\": {\n", + " \"filter\": None,\n", + " \"transforms\": []\n", + " }\n", + " },\n", + " \"ioConfig\": {\n", + " \"topic\": \"raw_scouter-opcua-orchestrated-pipeline\",\n", + " \"topicPattern\": None,\n", + " \"inputFormat\": {\n", + " \"type\": \"kafka\",\n", + " \"headerFormat\": None,\n", + " \"keyFormat\": None,\n", + " \"valueFormat\": {\n", + " \"type\": \"json\",\n", + " \"keepNoneColumns\": False,\n", + " \"assumeNewlineDelimited\": False,\n", + " \"useJsonNodeReader\": False\n", + " },\n", + " \"headerColumnPrefix\": \"kafka.header.\",\n", + " \"keyColumnName\": \"kafka.key\",\n", + " \"timestampColumnName\": \"kafka.timestamp\",\n", + " \"topicColumnName\": \"kafka.topic\"\n", + " },\n", + " \"replicas\": 1,\n", + " \"taskCount\": 1,\n", + " \"taskDuration\": \"PT3600S\",\n", + " \"consumerProperties\": {\n", + " \"bootstrap.servers\": \"kafka.kafka.svc.cluster.local:9092\"\n", + " },\n", + " \"autoScalerConfig\": None,\n", + " \"pollTimeout\": 100,\n", + " \"startDelay\": \"PT5S\",\n", + " \"period\": \"PT30S\",\n", + " \"useEarliestOffset\": True,\n", + " \"completionTimeout\": \"PT1800S\",\n", + " \"lateMessageRejectionPeriod\": None,\n", + " \"earlyMessageRejectionPeriod\": None,\n", + " \"lateMessageRejectionStartDateTime\": None,\n", + " \"configOverrides\": None,\n", + " \"idleConfig\": None,\n", + " \"stopTaskCount\": None,\n", + " \"stream\": \"raw_scouter-opcua-orchestrated-pipeline\",\n", + " \"useEarliestSequenceNumber\": True\n", + " },\n", + " \"tuningConfig\": {\n", + " \"type\": \"kafka\",\n", + " \"appendableIndexSpec\": {\n", + " \"type\": \"onheap\",\n", + " \"preserveExistingMetrics\": False\n", + " },\n", + " \"maxRowsInMemory\": 150000,\n", + " \"maxBytesInMemory\": 0,\n", + " \"skipBytesInMemoryOverheadCheck\": False,\n", + " \"maxRowsPerSegment\": 5000000,\n", + " \"maxTotalRows\": None,\n", + " \"intermediatePersistPeriod\": \"PT10M\",\n", + " \"maxPendingPersists\": 0,\n", + " \"indexSpec\": {\n", + " \"bitmap\": {\n", + " \"type\": \"roaring\"\n", + " },\n", + " \"dimensionCompression\": \"lz4\",\n", + " \"stringDictionaryEncoding\": {\n", + " \"type\": \"utf8\"\n", + " },\n", + " \"metricCompression\": \"lz4\",\n", + " \"longEncoding\": \"longs\"\n", + " },\n", + " \"indexSpecForIntermediatePersists\": {\n", + " \"bitmap\": {\n", + " \"type\": \"roaring\"\n", + " },\n", + " \"dimensionCompression\": \"lz4\",\n", + " \"stringDictionaryEncoding\": {\n", + " \"type\": \"utf8\"\n", + " },\n", + " \"metricCompression\": \"lz4\",\n", + " \"longEncoding\": \"longs\"\n", + " },\n", + " \"reportParseExceptions\": False,\n", + " \"handoffConditionTimeout\": 900000,\n", + " \"resetOffsetAutomatically\": False,\n", + " \"segmentWriteOutMediumFactory\": None,\n", + " \"workerThreads\": None,\n", + " \"chatRetries\": 8,\n", + " \"httpTimeout\": \"PT10S\",\n", + " \"shutdownTimeout\": \"PT80S\",\n", + " \"offsetFetchPeriod\": \"PT30S\",\n", + " \"intermediateHandoffPeriod\": \"P2147483647D\",\n", + " \"logParseExceptions\": False,\n", + " \"maxParseExceptions\": 2147483647,\n", + " \"maxSavedParseExceptions\": 0,\n", + " \"numPersistThreads\": 1,\n", + " \"skipSequenceNumberAvailabilityCheck\": False,\n", + " \"repartitionTransitionDuration\": \"PT120S\"\n", + " }\n", + " },\n", + " \"context\": None,\n", + " \"suspended\": False\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "raw_scouter-opcua-pipeline-2\n", + "raw_scouter-opcua-pipeline-3\n", + "raw_scouter-opcua-pipeline-4\n", + "raw_scouter-opcua-pipeline-5\n", + "raw_scouter-opcua-pipeline-6\n", + "raw_scouter-opcua-pipeline-7\n", + "raw_scouter-opcua-pipeline-8\n", + "raw_scouter-opcua-pipeline-9\n", + "raw_scouter-opcua-pipeline-10\n", + "raw_scouter-opcua-pipeline-11\n", + "raw_scouter-opcua-pipeline-12\n", + "raw_scouter-opcua-pipeline-13\n", + "raw_scouter-opcua-pipeline-14\n", + "raw_scouter-opcua-pipeline-15\n", + "raw_scouter-opcua-pipeline-16\n", + "raw_scouter-opcua-pipeline-17\n", + "raw_scouter-opcua-pipeline-18\n", + "raw_scouter-opcua-pipeline-19\n", + "raw_scouter-opcua-pipeline-20\n", + "raw_scouter-opcua-pipeline-21\n", + "raw_scouter-opcua-pipeline-22\n", + "raw_scouter-opcua-pipeline-23\n", + "raw_scouter-opcua-pipeline-24\n", + "raw_scouter-opcua-pipeline-25\n", + "raw_scouter-opcua-pipeline-26\n", + "raw_scouter-opcua-pipeline-27\n", + "raw_scouter-opcua-pipeline-28\n", + "raw_scouter-opcua-pipeline-29\n", + "raw_scouter-opcua-pipeline-30\n" + ] + } + ], + "source": [ + "import json\n", + "from copy import deepcopy\n", + "\n", + "json_list = []\n", + "i = 0\n", + "for i in range(1, 30):\n", + " topic = f\"raw_scouter-opcua-pipeline-{i+1}\"\n", + " print(topic)\n", + " obj = deepcopy(spec)\n", + " obj['spec']['dataSchema']['dataSource'] = topic\n", + " obj['spec']['ioConfig']['topic'] = topic\n", + " obj['spec']['ioConfig']['stream'] = topic\n", + "\n", + " json_list.append(obj)\n", + "\n", + "# Save to a new file\n", + "with open('specs_30.json', 'w') as f:\n", + " json.dump(json_list, f, indent=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[OK] raw_scouter-opcua-pipeline-2 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-3 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-4 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-5 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-6 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-7 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-8 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-9 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-10 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-11 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-12 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-13 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-14 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-15 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-16 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-17 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-18 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-19 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-20 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-21 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-22 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-23 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-24 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-25 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-26 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-27 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-28 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-29 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-30 enviado.\n" + ] + } + ], + "source": [ + "import os\n", + "import requests\n", + "\n", + "DRUID_OVERLORD = os.getenv(\"DRUID_OVERLORD\", \"http://localhost:8082\")\n", + "SUPERVISOR_ENDPOINT = f\"{DRUID_OVERLORD}/druid/indexer/v1/supervisor\"\n", + "\n", + "def enviar_supervisores(specs):\n", + " for spec in specs:\n", + " resp = requests.post(\n", + " SUPERVISOR_ENDPOINT,\n", + " headers={\"Content-Type\": \"application/json\"},\n", + " json=spec\n", + " )\n", + " if resp.status_code == 200:\n", + " print(f\"[OK] {spec['spec']['dataSchema']['dataSource']} enviado.\")\n", + " else:\n", + " print(f\"[ERRO] {spec['spec']['dataSchema']['dataSource']}: {resp.status_code} → {resp.text}\")\n", + "\n", + "with open(\"./specs_30.json\", \"r\") as f:\n", + " specs = json.load(f)\n", + "enviar_supervisores(specs)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "ename": "ConnectionError", + "evalue": "HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /druid/indexer/v1/supervisor/raw_scouter-opcua-orchestrated-pipeline-2/terminate (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mConnectionRefusedError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:198\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 197\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m198\u001b[39m sock = \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 199\u001b[39m \u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 200\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 201\u001b[39m \u001b[43m \u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 202\u001b[39m \u001b[43m \u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 203\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 204\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.gaierror \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 84\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m85\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[32m 86\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 87\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/util/connection.py:73\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 72\u001b[39m sock.bind(source_address)\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[43msock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 74\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", + "\u001b[31mConnectionRefusedError\u001b[39m: [Errno 111] Connection refused", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[31mNewConnectionError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connectionpool.py:787\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 786\u001b[39m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m787\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 788\u001b[39m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 789\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 790\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 791\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 792\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 793\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 794\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 795\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 796\u001b[39m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 797\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 798\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 799\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 800\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 802\u001b[39m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connectionpool.py:493\u001b[39m, in \u001b[36mHTTPConnectionPool._make_request\u001b[39m\u001b[34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 492\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m493\u001b[39m \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 494\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 495\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 497\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 498\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 499\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 500\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 501\u001b[39m \u001b[43m \u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m=\u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 502\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 504\u001b[39m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[32m 505\u001b[39m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[32m 506\u001b[39m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:494\u001b[39m, in \u001b[36mHTTPConnection.request\u001b[39m\u001b[34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 493\u001b[39m \u001b[38;5;28mself\u001b[39m.putheader(header, value)\n\u001b[32m--> \u001b[39m\u001b[32m494\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[38;5;66;03m# If we're given a body we start sending that in chunks.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.11/http/client.py:1298\u001b[39m, in \u001b[36mHTTPConnection.endheaders\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1297\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[32m-> \u001b[39m\u001b[32m1298\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.11/http/client.py:1058\u001b[39m, in \u001b[36mHTTPConnection._send_output\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1057\u001b[39m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m._buffer[:]\n\u001b[32m-> \u001b[39m\u001b[32m1058\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1060\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1061\u001b[39m \n\u001b[32m 1062\u001b[39m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.11/http/client.py:996\u001b[39m, in \u001b[36mHTTPConnection.send\u001b[39m\u001b[34m(self, data)\u001b[39m\n\u001b[32m 995\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.auto_open:\n\u001b[32m--> \u001b[39m\u001b[32m996\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 997\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:325\u001b[39m, in \u001b[36mHTTPConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 324\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m325\u001b[39m \u001b[38;5;28mself\u001b[39m.sock = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 326\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._tunnel_host:\n\u001b[32m 327\u001b[39m \u001b[38;5;66;03m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:213\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 212\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m--> \u001b[39m\u001b[32m213\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[32m 214\u001b[39m \u001b[38;5;28mself\u001b[39m, \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 215\u001b[39m ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m 217\u001b[39m sys.audit(\u001b[33m\"\u001b[39m\u001b[33mhttp.client.connect\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mself\u001b[39m.host, \u001b[38;5;28mself\u001b[39m.port)\n", + "\u001b[31mNewConnectionError\u001b[39m: : Failed to establish a new connection: [Errno 111] Connection refused", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[31mMaxRetryError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/adapters.py:667\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 666\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m667\u001b[39m resp = \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 668\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 669\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 670\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 671\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 672\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 673\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 674\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 675\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 676\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 677\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 678\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 679\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 681\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connectionpool.py:841\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 839\u001b[39m new_e = ProtocolError(\u001b[33m\"\u001b[39m\u001b[33mConnection aborted.\u001b[39m\u001b[33m\"\u001b[39m, new_e)\n\u001b[32m--> \u001b[39m\u001b[32m841\u001b[39m retries = \u001b[43mretries\u001b[49m\u001b[43m.\u001b[49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 842\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m=\u001b[49m\u001b[43msys\u001b[49m\u001b[43m.\u001b[49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[32m 843\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 844\u001b[39m retries.sleep()\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/util/retry.py:519\u001b[39m, in \u001b[36mRetry.increment\u001b[39m\u001b[34m(self, method, url, response, error, _pool, _stacktrace)\u001b[39m\n\u001b[32m 518\u001b[39m reason = error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[32m--> \u001b[39m\u001b[32m519\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mreason\u001b[39;00m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[32m 521\u001b[39m log.debug(\u001b[33m\"\u001b[39m\u001b[33mIncremented Retry for (url=\u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m\"\u001b[39m, url, new_retry)\n", + "\u001b[31mMaxRetryError\u001b[39m: HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /druid/indexer/v1/supervisor/raw_scouter-opcua-orchestrated-pipeline-2/terminate (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[31mConnectionError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[20]\u001b[39m\u001b[32m, line 16\u001b[39m\n\u001b[32m 13\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m❌ Erro \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresp.status_code\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresp.text\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m 15\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[32m1\u001b[39m, \u001b[32m30\u001b[39m):\n\u001b[32m---> \u001b[39m\u001b[32m16\u001b[39m \u001b[43mterminate_supervisor\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43mf\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mraw_scouter-opcua-orchestrated-pipeline-\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mi\u001b[49m\u001b[43m+\u001b[49m\u001b[32;43m1\u001b[39;49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[20]\u001b[39m\u001b[32m, line 7\u001b[39m, in \u001b[36mterminate_supervisor\u001b[39m\u001b[34m(supervisor_id)\u001b[39m\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mterminate_supervisor\u001b[39m(supervisor_id):\n\u001b[32m 6\u001b[39m url = \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mDRUID\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/druid/indexer/v1/supervisor/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msupervisor_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/terminate\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m7\u001b[39m resp = \u001b[43mrequests\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 8\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m resp.status_code == \u001b[32m200\u001b[39m:\n\u001b[32m 9\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m✅ Supervisor \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msupervisor_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m encerrado.\u001b[39m\u001b[33m\"\u001b[39m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/api.py:115\u001b[39m, in \u001b[36mpost\u001b[39m\u001b[34m(url, data, json, **kwargs)\u001b[39m\n\u001b[32m 103\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpost\u001b[39m(url, data=\u001b[38;5;28;01mNone\u001b[39;00m, json=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m 104\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33mr\u001b[39m\u001b[33;03m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[32m 105\u001b[39m \n\u001b[32m 106\u001b[39m \u001b[33;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 112\u001b[39m \u001b[33;03m :rtype: requests.Response\u001b[39;00m\n\u001b[32m 113\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m115\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpost\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[43m=\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/api.py:59\u001b[39m, in \u001b[36mrequest\u001b[39m\u001b[34m(method, url, **kwargs)\u001b[39m\n\u001b[32m 55\u001b[39m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[32m 56\u001b[39m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[32m 57\u001b[39m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[32m 58\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m sessions.Session() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[32m---> \u001b[39m\u001b[32m59\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/sessions.py:589\u001b[39m, in \u001b[36mSession.request\u001b[39m\u001b[34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[39m\n\u001b[32m 584\u001b[39m send_kwargs = {\n\u001b[32m 585\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mtimeout\u001b[39m\u001b[33m\"\u001b[39m: timeout,\n\u001b[32m 586\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mallow_redirects\u001b[39m\u001b[33m\"\u001b[39m: allow_redirects,\n\u001b[32m 587\u001b[39m }\n\u001b[32m 588\u001b[39m send_kwargs.update(settings)\n\u001b[32m--> \u001b[39m\u001b[32m589\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 591\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/sessions.py:703\u001b[39m, in \u001b[36mSession.send\u001b[39m\u001b[34m(self, request, **kwargs)\u001b[39m\n\u001b[32m 700\u001b[39m start = preferred_clock()\n\u001b[32m 702\u001b[39m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m703\u001b[39m r = \u001b[43madapter\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 705\u001b[39m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[32m 706\u001b[39m elapsed = preferred_clock() - start\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/adapters.py:700\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 696\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e.reason, _SSLError):\n\u001b[32m 697\u001b[39m \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[32m 698\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request=request)\n\u001b[32m--> \u001b[39m\u001b[32m700\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n\u001b[32m 702\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 703\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n", + "\u001b[31mConnectionError\u001b[39m: HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /druid/indexer/v1/supervisor/raw_scouter-opcua-orchestrated-pipeline-2/terminate (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))" + ] + } + ], + "source": [ + "import os\n", + "import requests\n", + "\n", + "DRUID = os.getenv(\"DRUID_URL\", \"http://localhost:8082\")\n", + "def terminate_supervisor(supervisor_id):\n", + " url = f\"{DRUID}/druid/indexer/v1/supervisor/{supervisor_id}/terminate\"\n", + " resp = requests.post(url)\n", + " if resp.status_code == 200:\n", + " print(f\"✅ Supervisor '{supervisor_id}' encerrado.\")\n", + " elif resp.status_code == 404:\n", + " print(f\"⚠️ Supervisor '{supervisor_id}' não encontrado.\")\n", + " else:\n", + " print(f\"❌ Erro {resp.status_code}: {resp.text}\")\n", + "\n", + "for i in range(1, 30):\n", + " terminate_supervisor(f\"raw_scouter-opcua-orchestrated-pipeline-{i+1}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Document deleted successfully\n" + ] + } + ], + "source": [ + "from pymongo import MongoClient\n", + "import os\n", + "# Get MongoDB connection details from environment variables\n", + "MONGODB_USERNAME = os.getenv(\"MONGODB_USERNAME\", \"root\")\n", + "MONGODB_PASSWORD = os.getenv(\"MONGODB_PASSWORD\", \"wKZDbMNU1c\") \n", + "MONGODB_URL = os.getenv(\"MONGODB_URL\", \"localhost:27018\")\n", + "MONGODB_DATABASE = os.getenv(\"MONGODB_DATABASE\", \"sientia\")\n", + "\n", + "# Create MongoDB client\n", + "client = MongoClient(\n", + " f\"mongodb://{MONGODB_USERNAME}:{MONGODB_PASSWORD}@{MONGODB_URL}\"\n", + ")\n", + "\n", + "# Get database and collection\n", + "db = client[MONGODB_DATABASE]\n", + "collection = db[\"pipelines\"] # Replace with actual collection name\n", + "\n", + "for i in range(2, 31):\n", + " # Delete a document matching specific criteria\n", + " result = collection.delete_one({\"schedule_name\": f\"scouter-opcua-pipeline-{i}\"}) # Replace with actual query\n", + "\n", + "if result.deleted_count > 0:\n", + " print(\"✅ Document deleted successfully\")\n", + "else:\n", + " print(\"⚠️ No matching document found\")\n", + "\n", + "# Close the connection\n", + "client.close()\n" + ] } ], "metadata": { diff --git a/values.yaml b/values.yaml index 6686b55..3bf6fa4 100644 --- a/values.yaml +++ b/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 1 +replicaCount: 3 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: @@ -141,7 +141,7 @@ env: - name: POSTGRES_MIN_CONNECTIONS value: "10" - name: POSTGRES_MAX_CONNECTIONS - value: "20" + value: "40" - name: KAFKA_BOOTSTRAP_SERVERS value: "kafka.kafka.svc.cluster.local:9092"