3ee7e3687a
Update test.ipynb to generate and save multiple pipeline specifications and input samples in JSON format. Modify values.yaml to increase replica count from 1 to 3 and adjust PostgreSQL max connections from 20 to 40 for improved scalability.
3743 lines
111 KiB
JSON
3743 lines
111 KiB
JSON
[
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-2",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-2",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-2",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-3",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-3",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-3",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-4",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-4",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-4",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-5",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-5",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-5",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-6",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-6",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-6",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-7",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-7",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-7",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-8",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-8",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-8",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-9",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-9",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-9",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-10",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-10",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-10",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-11",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-11",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-11",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-12",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-12",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-12",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-13",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-13",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-13",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-14",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-14",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-14",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-15",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-15",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-15",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-16",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-16",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-16",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-17",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-17",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-17",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-18",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-18",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-18",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-19",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-19",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-19",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-20",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-20",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-20",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-21",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-21",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-21",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-22",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-22",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-22",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-23",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-23",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-23",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-24",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-24",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-24",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-25",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-25",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-25",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-26",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-26",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-26",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-27",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-27",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-27",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-28",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-28",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-28",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-29",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-29",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-29",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
},
|
|
{
|
|
"type": "kafka",
|
|
"spec": {
|
|
"dataSchema": {
|
|
"dataSource": "raw_scouter-opcua-pipeline-30",
|
|
"timestampSpec": {
|
|
"column": "kafka.timestamp",
|
|
"format": "millis",
|
|
"missingValue": null
|
|
},
|
|
"dimensionsSpec": {
|
|
"dimensions": [],
|
|
"dimensionExclusions": [
|
|
"__time",
|
|
"kafka.timestamp"
|
|
],
|
|
"includeAllDimensions": false,
|
|
"useSchemaDiscovery": true
|
|
},
|
|
"metricsSpec": [],
|
|
"granularitySpec": {
|
|
"type": "uniform",
|
|
"segmentGranularity": "DAY",
|
|
"queryGranularity": {
|
|
"type": "none"
|
|
},
|
|
"rollup": false,
|
|
"intervals": []
|
|
},
|
|
"transformSpec": {
|
|
"filter": null,
|
|
"transforms": []
|
|
}
|
|
},
|
|
"ioConfig": {
|
|
"topic": "raw_scouter-opcua-pipeline-30",
|
|
"topicPattern": null,
|
|
"inputFormat": {
|
|
"type": "kafka",
|
|
"headerFormat": null,
|
|
"keyFormat": null,
|
|
"valueFormat": {
|
|
"type": "json",
|
|
"keepNoneColumns": false,
|
|
"assumeNewlineDelimited": false,
|
|
"useJsonNodeReader": false
|
|
},
|
|
"headerColumnPrefix": "kafka.header.",
|
|
"keyColumnName": "kafka.key",
|
|
"timestampColumnName": "kafka.timestamp",
|
|
"topicColumnName": "kafka.topic"
|
|
},
|
|
"replicas": 1,
|
|
"taskCount": 1,
|
|
"taskDuration": "PT3600S",
|
|
"consumerProperties": {
|
|
"bootstrap.servers": "kafka.kafka.svc.cluster.local:9092"
|
|
},
|
|
"autoScalerConfig": null,
|
|
"pollTimeout": 100,
|
|
"startDelay": "PT5S",
|
|
"period": "PT30S",
|
|
"useEarliestOffset": true,
|
|
"completionTimeout": "PT1800S",
|
|
"lateMessageRejectionPeriod": null,
|
|
"earlyMessageRejectionPeriod": null,
|
|
"lateMessageRejectionStartDateTime": null,
|
|
"configOverrides": null,
|
|
"idleConfig": null,
|
|
"stopTaskCount": null,
|
|
"stream": "raw_scouter-opcua-pipeline-30",
|
|
"useEarliestSequenceNumber": true
|
|
},
|
|
"tuningConfig": {
|
|
"type": "kafka",
|
|
"appendableIndexSpec": {
|
|
"type": "onheap",
|
|
"preserveExistingMetrics": false
|
|
},
|
|
"maxRowsInMemory": 150000,
|
|
"maxBytesInMemory": 0,
|
|
"skipBytesInMemoryOverheadCheck": false,
|
|
"maxRowsPerSegment": 5000000,
|
|
"maxTotalRows": null,
|
|
"intermediatePersistPeriod": "PT10M",
|
|
"maxPendingPersists": 0,
|
|
"indexSpec": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"indexSpecForIntermediatePersists": {
|
|
"bitmap": {
|
|
"type": "roaring"
|
|
},
|
|
"dimensionCompression": "lz4",
|
|
"stringDictionaryEncoding": {
|
|
"type": "utf8"
|
|
},
|
|
"metricCompression": "lz4",
|
|
"longEncoding": "longs"
|
|
},
|
|
"reportParseExceptions": false,
|
|
"handoffConditionTimeout": 900000,
|
|
"resetOffsetAutomatically": false,
|
|
"segmentWriteOutMediumFactory": null,
|
|
"workerThreads": null,
|
|
"chatRetries": 8,
|
|
"httpTimeout": "PT10S",
|
|
"shutdownTimeout": "PT80S",
|
|
"offsetFetchPeriod": "PT30S",
|
|
"intermediateHandoffPeriod": "P2147483647D",
|
|
"logParseExceptions": false,
|
|
"maxParseExceptions": 2147483647,
|
|
"maxSavedParseExceptions": 0,
|
|
"numPersistThreads": 1,
|
|
"skipSequenceNumberAvailabilityCheck": false,
|
|
"repartitionTransitionDuration": "PT120S"
|
|
}
|
|
},
|
|
"context": null,
|
|
"suspended": false
|
|
}
|
|
] |