From 74a2d9aa3e5b5c3aa99111ba4d9eda3bf874e3e1 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 27 Jun 2025 09:34:58 -0300 Subject: [PATCH 01/36] SIENTIAPDE-1110 Update GITHUB_BRANCH in values.yaml to SIENTIAPDE-1110-criar-testes-e-2-e and remove prepare_activity from worker.py --- scouter/worker/worker.py | 1 - values.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 321620a..8ccfa91 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -73,7 +73,6 @@ async def main(): activities.aggregate_data, activities.group_and_hold_data, activities.export_data_to_postgres, - activities.prepare_activity, ] ), Worker( diff --git a/values.yaml b/values.yaml index 67923f8..48db1f1 100644 --- a/values.yaml +++ b/values.yaml @@ -123,7 +123,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git" - name: GITHUB_BRANCH - value: "main" + value: "SIENTIAPDE-1110-criar-testes-e-2-e" - name: PYTHON_APP value: "scouter.worker.worker" From 9afb74be21fc568b3cac40c8bcfd6d239505ae6f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 27 Jun 2025 10:07:43 -0300 Subject: [PATCH 02/36] SIENTIAPDE-1110 Add metadata to input_data in Scouter and update CoreScouter to use it --- scouter/workflow/scouter.py | 1 + scouter/workflow/sub_workflows/core_scouter.py | 9 +-------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/scouter/workflow/scouter.py b/scouter/workflow/scouter.py index 292834b..ecf0716 100644 --- a/scouter/workflow/scouter.py +++ b/scouter/workflow/scouter.py @@ -55,6 +55,7 @@ class Scouter: return input_data['data'] = data + input_data['metadata'] = metadata await workflow.execute_child_workflow( 'core_scouter', diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py index 9628aea..de0e79a 100644 --- a/scouter/workflow/sub_workflows/core_scouter.py +++ b/scouter/workflow/sub_workflows/core_scouter.py @@ -32,14 +32,7 @@ class CoreScouter: retention_time (int): The retention time for data in redis in seconds. """ - metadata = { - 'metadata': { - 'model_id': input_data['model_id'], - 'model_name': input_data['model_name'], - 'schedule_name': input_data['schedule_name'], - 'workflow_name': input_data['workflow_name'] - } - } + metadata = input_data['metadata'] filtered_data = await workflow.execute_local_activity_method( Activities.data_quality_gate, From 6144bcf8ffd3ea8e4e273b3a8e98bc4ee3edc843 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 30 Jun 2025 14:11:30 -0300 Subject: [PATCH 03/36] SIENTIAPDE-1110 Update Kafka polling time in values.yaml and refactor Kafka subscription in kafka.py to use KafkaConsumer for improved message handling. --- scouter/activities/kafka.py | 9 ++++++++- values.yaml | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scouter/activities/kafka.py b/scouter/activities/kafka.py index 089b28b..a05b292 100644 --- a/scouter/activities/kafka.py +++ b/scouter/activities/kafka.py @@ -56,7 +56,14 @@ class Kafka(BaseActivity): topic = input_data["topic"] # Subscribe to the specified topic - self.kafka_connector.subscribe([topic]) + self.kafka_connector = KafkaConsumer( + topic, + bootstrap_servers=self.bootstrap_servers, + auto_offset_reset="earliest", + enable_auto_commit=True, + group_id=self.group_id, + value_deserializer=lambda x: json.loads(x.decode("utf-8")) + ) # List to store message values message_values = [] diff --git a/values.yaml b/values.yaml index 48db1f1..8006b67 100644 --- a/values.yaml +++ b/values.yaml @@ -146,7 +146,7 @@ env: - name: KAFKA_BOOTSTRAP_SERVERS value: "kafka.kafka.svc.cluster.local:9092" - name: KAFKA_POLLING_TIME - value: "1000" + value: "10000" - name: REDIS_HOST value: "redis-master.redis.svc.cluster.local" From d33a5cbb72d320ce11fa4f5d0cbedf74a2be3ad3 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 30 Jun 2025 14:12:02 -0300 Subject: [PATCH 04/36] SIENTIAPDE-1110 Refactor Kafka activity to use local kafka_connector variable for improved readability and ensure proper closure of the Kafka consumer after polling. --- scouter/activities/kafka.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scouter/activities/kafka.py b/scouter/activities/kafka.py index a05b292..8fe1454 100644 --- a/scouter/activities/kafka.py +++ b/scouter/activities/kafka.py @@ -56,7 +56,7 @@ class Kafka(BaseActivity): topic = input_data["topic"] # Subscribe to the specified topic - self.kafka_connector = KafkaConsumer( + kafka_connector = KafkaConsumer( topic, bootstrap_servers=self.bootstrap_servers, auto_offset_reset="earliest", @@ -69,7 +69,7 @@ class Kafka(BaseActivity): message_values = [] # Poll for messages - records = self.kafka_connector.poll(timeout_ms=self.polling_time) + records = kafka_connector.poll(timeout_ms=self.polling_time) self.debug( f"Polled {len(records)} records from topic: {topic}", @@ -95,4 +95,6 @@ class Kafka(BaseActivity): metadata=metadata ) + kafka_connector.close() + return DataFrame(message_values).to_dict() From a53804331b6b514666d806b32a432ed43a856c9c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 30 Jun 2025 14:36:15 -0300 Subject: [PATCH 05/36] SIENTIAPDE-1110 Enhance Kafka activity initialization by adding bootstrap_servers and group_id attributes for improved configuration management. --- scouter/activities/kafka.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scouter/activities/kafka.py b/scouter/activities/kafka.py index 8fe1454..64ad62e 100644 --- a/scouter/activities/kafka.py +++ b/scouter/activities/kafka.py @@ -15,6 +15,8 @@ class Kafka(BaseActivity): def __init__(self, bootstrap_servers: str, polling_time: int, group_id: str, logger: Logger, notification_handler: NotificationHandler): self.polling_time = polling_time + self.bootstrap_servers = bootstrap_servers + self.group_id = group_id self.kafka_connector = KafkaConsumer( bootstrap_servers=bootstrap_servers, From 66b568838e3f112e464f41ff3de4695547eb724d Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 08:21:00 -0300 Subject: [PATCH 06/36] SIENTIAPDE-1110 Refactor Kafka activity to use instance-level kafka_connector for improved message polling and management, replacing local variable usage with class attribute methods. --- scouter/activities/kafka.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/scouter/activities/kafka.py b/scouter/activities/kafka.py index 64ad62e..ccd434f 100644 --- a/scouter/activities/kafka.py +++ b/scouter/activities/kafka.py @@ -58,20 +58,13 @@ class Kafka(BaseActivity): topic = input_data["topic"] # Subscribe to the specified topic - kafka_connector = KafkaConsumer( - topic, - bootstrap_servers=self.bootstrap_servers, - auto_offset_reset="earliest", - enable_auto_commit=True, - group_id=self.group_id, - value_deserializer=lambda x: json.loads(x.decode("utf-8")) - ) + self.kafka_connector.subscribe([topic]) # List to store message values message_values = [] # Poll for messages - records = kafka_connector.poll(timeout_ms=self.polling_time) + records = self.kafka_connector.poll(timeout_ms=self.polling_time) self.debug( f"Polled {len(records)} records from topic: {topic}", @@ -97,6 +90,6 @@ class Kafka(BaseActivity): metadata=metadata ) - kafka_connector.close() + self.kafka_connector.unsubscribe() return DataFrame(message_values).to_dict() From 704099c38756a8f12466dbd9f2fb019958eccf24 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 10:44:37 -0300 Subject: [PATCH 07/36] SIENTIAPDE-1110 Update Kafka activity to use AIOKafkaConsumer for asynchronous message handling, enhancing consumer management and adding support for multiple topics. --- requirements.txt | 1 + scouter/activities/kafka.py | 79 +++++++++++++++++++++---------------- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/requirements.txt b/requirements.txt index c8054ba..c29f059 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,6 @@ psycopg2-binary sqlalchemy asyncua redis +aiokafka git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.2.0 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.1 diff --git a/scouter/activities/kafka.py b/scouter/activities/kafka.py index ccd434f..ec0aea4 100644 --- a/scouter/activities/kafka.py +++ b/scouter/activities/kafka.py @@ -6,9 +6,10 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.temporal.activities.base import BaseActivity from sientia_do.temporal.utils.logger import Logger from typing import Any - from kafka import KafkaConsumer + from aiokafka import AIOKafkaConsumer from pandas import DataFrame import json + import asyncio class Kafka(BaseActivity): @@ -17,24 +18,32 @@ class Kafka(BaseActivity): self.polling_time = polling_time self.bootstrap_servers = bootstrap_servers self.group_id = group_id - - self.kafka_connector = KafkaConsumer( - bootstrap_servers=bootstrap_servers, - auto_offset_reset="earliest", - enable_auto_commit=True, - group_id=group_id, - value_deserializer=lambda x: json.loads(x.decode("utf-8")) - ) - + self.consumers = {} + self._consumer_tasks = {} BaseActivity.__init__(self, logger, notification_handler) - def close(self): - """Closes the connector connection.""" - self.info("Closing Kafka connector...") - self.kafka_connector.close() + async def close(self): + """Closes all consumer connections.""" + self.info("Closing Kafka connectors...") + for _topic, consumer in self.consumers.items(): + await consumer.stop() - def __del__(self): - self.close() + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.close() + + async def create_consumer(self, topic: str): + consumer = AIOKafkaConsumer( + bootstrap_servers=self.bootstrap_servers, + auto_offset_reset="earliest", + enable_auto_commit=True, + group_id=f"{self.group_id}-{topic}", + value_deserializer=lambda x: json.loads(x.decode("utf-8")) + ) + await consumer.start() + self.consumers[topic] = consumer @activity.defn(name="load_from_kafka") async def load_from_kafka(self, input_data: dict[str, Any]) -> dict[str, Any]: @@ -57,28 +66,25 @@ class Kafka(BaseActivity): topic = input_data["topic"] - # Subscribe to the specified topic - self.kafka_connector.subscribe([topic]) + if topic not in self.consumers: + await self.create_consumer(topic) + + consumer = self.consumers[topic] + + await consumer.subscribe([topic]) - # List to store message values message_values = [] - # Poll for messages - records = self.kafka_connector.poll(timeout_ms=self.polling_time) - - self.debug( - f"Polled {len(records)} records from topic: {topic}", - metadata=metadata - ) - - # Process the polled records - for _topic_partition, msgs in records.items(): - for msg in msgs: + end_time = asyncio.get_event_loop().time() + (self.polling_time / 1000) + while asyncio.get_event_loop().time() < end_time: + try: + msg = await asyncio.wait_for(consumer.getone(), timeout=(end_time - asyncio.get_event_loop().time())) message_values.append(msg.value) - - # Return empty dict if no messages were received - if not message_values: - return {} + except asyncio.TimeoutError: + break + except Exception as e: + self.error(f"Error while consuming: {e}", metadata=metadata) + break self.debug( f"Loaded {len(message_values)} messages from topic: {topic}", @@ -90,6 +96,9 @@ class Kafka(BaseActivity): metadata=metadata ) - self.kafka_connector.unsubscribe() + await consumer.unsubscribe() + + if not message_values: + return {} return DataFrame(message_values).to_dict() From 946d953598a65b41a4e871e1238c908559f61bc7 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 10:57:12 -0300 Subject: [PATCH 08/36] SIENTIAPDE-1110 Refactor Kafka activity to improve message consumption by using getmany for batch retrieval, enhancing performance and simplifying message handling logic. --- scouter/activities/kafka.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/scouter/activities/kafka.py b/scouter/activities/kafka.py index ec0aea4..6276455 100644 --- a/scouter/activities/kafka.py +++ b/scouter/activities/kafka.py @@ -71,22 +71,19 @@ class Kafka(BaseActivity): consumer = self.consumers[topic] - await consumer.subscribe([topic]) + consumer.subscribe(topics=[topic]) message_values = [] - end_time = asyncio.get_event_loop().time() + (self.polling_time / 1000) - while asyncio.get_event_loop().time() < end_time: - try: - msg = await asyncio.wait_for(consumer.getone(), timeout=(end_time - asyncio.get_event_loop().time())) - message_values.append(msg.value) - except asyncio.TimeoutError: - break - except Exception as e: - self.error(f"Error while consuming: {e}", metadata=metadata) - break + messages = await consumer.getmany(timeout_ms=self.polling_time) - self.debug( + for tp, msgs in messages.items(): + msg_topic = tp.topic + if msg_topic == topic: + for msg in msgs: + message_values.append(msg.value) + + self.info( f"Loaded {len(message_values)} messages from topic: {topic}", metadata=metadata ) @@ -96,7 +93,7 @@ class Kafka(BaseActivity): metadata=metadata ) - await consumer.unsubscribe() + consumer.unsubscribe() if not message_values: return {} From e7b05ebf3aa78600b5f23c80b2d40e996c9b5bba Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 11:05:03 -0300 Subject: [PATCH 09/36] SIENTIAPDE-1110 Add logging for consumer creation in Kafka activity to enhance traceability of topic management. --- scouter/activities/kafka.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scouter/activities/kafka.py b/scouter/activities/kafka.py index 6276455..ea36655 100644 --- a/scouter/activities/kafka.py +++ b/scouter/activities/kafka.py @@ -67,6 +67,10 @@ class Kafka(BaseActivity): topic = input_data["topic"] if topic not in self.consumers: + self.info( + f"Creating consumer for topic: {topic}", + metadata=metadata + ) await self.create_consumer(topic) consumer = self.consumers[topic] From 53337de74a995b31cdd180520b43b1721df1f535 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 15:10:32 -0300 Subject: [PATCH 10/36] SIENTIAPDE-1110 SIENTIAPDE-1110 Add MongoDB integration and enhance Redis activity with timestamp management functions. --- requirements.txt | 1 + scouter/activities/activities.py | 14 +++- scouter/activities/mongodb.py | 121 +++++++++++++++++++++++++++++ scouter/activities/redis.py | 41 ++++++++++ scouter/utils/connectors_config.py | 12 +++ scouter/workflow/scouter.py | 44 +++++++++-- 6 files changed, 227 insertions(+), 6 deletions(-) create mode 100644 scouter/activities/mongodb.py diff --git a/requirements.txt b/requirements.txt index c29f059..3e098f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,5 +4,6 @@ sqlalchemy asyncua redis aiokafka +pymongo git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.2.0 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.1 diff --git a/scouter/activities/activities.py b/scouter/activities/activities.py index 8fc7efb..ddeae2e 100644 --- a/scouter/activities/activities.py +++ b/scouter/activities/activities.py @@ -7,16 +7,18 @@ with workflow.unsafe.imports_passed_through(): from scouter.activities.redis import Redis from scouter.activities.kafka import Kafka from scouter.activities.gates import Gates + from scouter.activities.mongodb import MongoDB from typing import Any -class Activities(Postgres, Redis, Kafka, Gates): +class Activities(Postgres, Redis, Kafka, Gates, MongoDB): """Activities class that combines multiple services with proper initialization.""" def __init__(self, postgres_config: dict[str, Any], redis_config: dict[str, Any], kafka_config: dict[str, Any], + mongodb_config: dict[str, Any], logger: Logger, notification_handler: NotificationHandler): @@ -62,6 +64,16 @@ class Activities(Postgres, Redis, Kafka, Gates): notification_handler=notification_handler ) + # Initialize MongoDB + MongoDB.__init__( + self, + connection_string=mongodb_config['connection_string'], + database_name=mongodb_config['database_name'], + logger=logger, + notification_handler=notification_handler + ) + def shutdown(self): Postgres.close(self) Kafka.close(self) + MongoDB.close(self) diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py new file mode 100644 index 0000000..81370db --- /dev/null +++ b/scouter/activities/mongodb.py @@ -0,0 +1,121 @@ +from temporalio import workflow, activity + +with workflow.unsafe.imports_passed_through(): + from typing import Any + import traceback + from logging import Logger + import datetime + from pymongo import MongoClient + from sientia_do.notifications.handlers import NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.temporal.activities.base import BaseActivity + + +def clear_mongo_id(docs: list) -> list: + """ + Remove the MongoDB internal `_id` field from the document. + + Args: + docs (list): The document to clear. + + Returns: + list: The documents without the `_id` field. + """ + for doc in docs: + if isinstance(doc, list): + clear_mongo_id(doc) + + elif isinstance(doc, dict): + if "_id" in doc: + del doc["_id"] + + for key, value in doc.items(): + if isinstance(value, list): + clear_mongo_id(value) + elif isinstance(value, dict): + clear_mongo_id([value]) + + return docs + + +class MongoDB(BaseActivity): + def __init__(self, connection_string: str, database_name: str, + logger: Logger, + notification_handler: NotificationHandler): + self.connection_string = connection_string + self.database_name = database_name + + self.client = MongoClient( + self.connection_string, serverSelectionTimeoutMS=5000) + self.client.server_info() # Trigger an exception if connection fails + + self.database = self.client[self.database_name] + + # Initialize MongoDB client here (omitted for brevity) + logger.info("MongoDB connection initialized") + + BaseActivity.__init__(self, + logger=logger, + notification_handler=notification_handler) + + def shutdown(self): + """ + Close the MongoDB client connection. + """ + try: + if self.client: + self.logger.info("Closing MongoDB connection...") + self.client.close() + self.logger.info("MongoDB connection closed successfully") + except Exception as e: + self.logger.error(f"Failed to close MongoDB connection: {e}") + + def __del__(self): + """ + Destructor to ensure MongoDB client is closed when the object is deleted. + """ + self.shutdown() + + @activity.defn(name="load_latest_data") + async def load_latest_data(self, input_data: dict[str, Any]) -> dict[str, Any]: + """ + Loads the latest data from MongoDB. + """ + metadata = input_data['metadata'] + collection_name = input_data['collection_name'] + last_data_timestamp = input_data['last_data_timestamp'] + + try: + + data_filter = { + "inserted_at": { + "$gt": datetime.strptime(last_data_timestamp, "%Y-%m-%d %H:%M:%S.%f") + } + } + + data = self.database[collection_name].find(data_filter, {"_id": 0}) + + data = clear_mongo_id(data) + + self.info( + f"Loaded {len(data)} documents from MongoDB", + metadata=metadata + ) + + self.debug( + f"Loaded data: {data}", + metadata=metadata + ) + + return data + except Exception as e: + trace = traceback.format_exc() + self.send_notification( + metadata=metadata, + notification_id="MONGO_LOAD_ERROR", + message=f"Error loading data from MongoDB: {e}", + block="load_latest_data", + level=NotificationLevel.ERROR, + attachment_content=trace + ) + raise e diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 09c3f40..1083d27 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -18,6 +18,47 @@ class Redis(RedisBase): RedisBase.__init__(self, host, port, username, password, logger, notification_handler) + @activity.defn(name="get_last_data_timestamp") + async def get_last_data_timestamp(self, input_data: dict[str, Any]) -> dict[str, Any]: + """ + Gets the last data timestamp from redis. + """ + metadata = input_data['metadata'] + key = f"{input_data['workflow_name']}_{input_data['schedule_name']}" + + data_hold = self.get(key) + + self.debug( + f"Last collected timestamp: {data_hold}", + metadata=metadata + ) + + if not data_hold: + return None + + return data_hold + + @activity.defn(name="put_last_data_timestamp") + async def put_last_data_timestamp(self, input_data: dict[str, Any]): + """ + Puts the last data timestamp into redis. + """ + metadata = input_data['metadata'] + key = f"{input_data['workflow_name']}_{input_data['schedule_name']}" + + data = DataFrame(input_data['data']) + + last_data_timestamp = data['inserted_at'].max() + + self.debug( + f"Last collected timestamp to insert: {last_data_timestamp}", + metadata=metadata + ) + + self.set(key, last_data_timestamp, ttl=input_data['retention_time']) + + return last_data_timestamp + @activity.defn(name="group_and_hold_data") async def group_and_hold_data(self, input_data: dict[str, Any]): """ diff --git a/scouter/utils/connectors_config.py b/scouter/utils/connectors_config.py index 6acc4ae..64a7feb 100644 --- a/scouter/utils/connectors_config.py +++ b/scouter/utils/connectors_config.py @@ -28,3 +28,15 @@ def build_redis_config(): 'username': getenv('REDIS_USERNAME', None), 'password': getenv('REDIS_PASSWORD', None) } + + +def build_mongodb_config(): + username = getenv('MONGODB_USERNAME', 'sientia') + password = getenv('MONGODB_PASSWORD', 'sientia') + uri = getenv('MONGODB_URL', 'localhost:27017') + + connection_string = f'mongodb://{username}:{password}@{uri}' + return { + 'connection_string': connection_string, + 'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia') + } diff --git a/scouter/workflow/scouter.py b/scouter/workflow/scouter.py index ecf0716..cf36c49 100644 --- a/scouter/workflow/scouter.py +++ b/scouter/workflow/scouter.py @@ -41,14 +41,48 @@ class Scouter: } } - data = await workflow.execute_activity_method( - Activities.load_from_kafka, + # data = await workflow.execute_activity_method( + # Activities.load_from_kafka, + # { + # **metadata, + # 'topic': input_data['topic'] + # }, + # retry_policy=retry_policy, + # start_to_close_timeout=timedelta(seconds=60) + # ) + + last_data_timestamp = await workflow.execute_local_activity_method( + Activities.get_last_data_timestamp, { **metadata, - 'topic': input_data['topic'] + 'workflow_name': input_data['workflow_name'], + 'schedule_name': input_data['schedule_name'] }, - retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=60), + retry_policy=retry_policy + ) + + data = await workflow.execute_local_activity_method( + Activities.load_latest_data, + { + **metadata, + 'collection_name': f"raw_{input_data['schedule_name']}", + 'last_data_timestamp': last_data_timestamp + }, + start_to_close_timeout=timedelta(seconds=60), + retry_policy=retry_policy + ) + + await workflow.execute_activity_method( + Activities.put_last_data_timestamp, + { + **metadata, + 'data': data, + 'workflow_name': input_data['workflow_name'], + 'schedule_name': input_data['schedule_name'] + }, + start_to_close_timeout=timedelta(seconds=60), + retry_policy=retry_policy ) if data == {}: From 97d66ed6cd80cfe086a2b9d1448ee797b7794933 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 15:15:18 -0300 Subject: [PATCH 11/36] SIENTIAPDE-1110 Add MongoDB configuration to values.yaml and integrate MongoDB support in worker.py --- scouter/worker/worker.py | 6 ++++-- values.yaml | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 8ccfa91..818d129 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -15,7 +15,8 @@ with workflow.unsafe.imports_passed_through(): from scouter.utils.connectors_config import ( build_postgres_config, build_kafka_config, - build_redis_config + build_redis_config, + build_mongodb_config ) @@ -41,7 +42,8 @@ async def main(): notification_handler=notification_handler, postgres_config=build_postgres_config(), kafka_config=build_kafka_config(), - redis_config=build_redis_config() + redis_config=build_redis_config(), + mongodb_config=build_mongodb_config() ) logger.info('Starting Faker Activities...') diff --git a/values.yaml b/values.yaml index 8006b67..037bc0c 100644 --- a/values.yaml +++ b/values.yaml @@ -173,6 +173,15 @@ env: - name: TEMPORAL_NAMESPACE value: "default" + - name: MONGODB_USERNAME + value: "root" + - name: MONGODB_PASSWORD + value: "wKZDbMNU1c" + - name: MONGODB_URL + value: "my-release-mongodb.mongodb.svc.cluster.local:27017" + - name: MONGODB_DATABASE + value: "sientia" + ssh: enabled: true secretName: git-ssh-key-sientia-scouter-worker From 51f4eb1435dadebe68f2f592e3f23c5be44b7ca3 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 15:16:55 -0300 Subject: [PATCH 12/36] SIENTIAPDE-1110 Add new activities to worker.py for data management: load_latest_data, get_last_data_timestamp, and put_last_data_timestamp. --- scouter/worker/worker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 818d129..74ad479 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -70,6 +70,9 @@ async def main(): task_queue='scouter-queue', workflows=[Scouter, CoreScouter], activities=[ + activities.load_latest_data, + activities.get_last_data_timestamp, + activities.put_last_data_timestamp, activities.load_from_kafka, activities.data_quality_gate, activities.aggregate_data, From 9b524ea4281c7625867a7c1b589fa0b52e73b628 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 15:20:50 -0300 Subject: [PATCH 13/36] SIENTIAPDE-1110 Refactor MongoDB activity to import datetime directly and format 'inserted_at' timestamps to ISO 8601 format for improved data consistency. --- scouter/activities/mongodb.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index 81370db..a85ea71 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through(): from typing import Any import traceback from logging import Logger - import datetime + from datetime import datetime from pymongo import MongoClient from sientia_do.notifications.handlers import NotificationHandler from sientia_do.notifications.models import NotificationLevel @@ -97,6 +97,9 @@ class MongoDB(BaseActivity): data = clear_mongo_id(data) + for item in data: + item['inserted_at'] = item['inserted_at'].isoformat() + self.info( f"Loaded {len(data)} documents from MongoDB", metadata=metadata From 1908741febb99d091f52573a10d339c058a81f25 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 15:26:57 -0300 Subject: [PATCH 14/36] SIENTIAPDE-1110 Add debug logging for MongoDB data loading to enhance traceability of input data. --- scouter/activities/mongodb.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index a85ea71..485a492 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -85,6 +85,11 @@ class MongoDB(BaseActivity): collection_name = input_data['collection_name'] last_data_timestamp = input_data['last_data_timestamp'] + self.debug( + f"Loading data from MongoDB: {input_data}", + metadata=metadata + ) + try: data_filter = { From d7d44556c844532fbb44e0d6a048887ce1b0fc55 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 15:31:27 -0300 Subject: [PATCH 15/36] SIENTIAPDE-1110 Update MongoDB activity to handle None last_data_timestamp by using an empty filter, improving data retrieval logic. --- scouter/activities/mongodb.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index 485a492..87fbc51 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -92,11 +92,14 @@ class MongoDB(BaseActivity): try: - data_filter = { - "inserted_at": { - "$gt": datetime.strptime(last_data_timestamp, "%Y-%m-%d %H:%M:%S.%f") + if last_data_timestamp is None: + data_filter = {} + else: + data_filter = { + "inserted_at": { + "$gt": datetime.strptime(last_data_timestamp, "%Y-%m-%d %H:%M:%S.%f") + } } - } data = self.database[collection_name].find(data_filter, {"_id": 0}) From f441873969e61170733af1505c40d72af5cbecca Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 15:33:51 -0300 Subject: [PATCH 16/36] SIENTIAPDE-1110 Refactor MongoDB data retrieval to convert cursor to list, improving data handling and compatibility. --- scouter/activities/mongodb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index 87fbc51..ce70bf8 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -101,7 +101,8 @@ class MongoDB(BaseActivity): } } - data = self.database[collection_name].find(data_filter, {"_id": 0}) + data = list(self.database[collection_name].find( + data_filter, {"_id": 0})) data = clear_mongo_id(data) From f183bac90d3f4b7f1e999737fad795b9d06395a3 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 15:37:40 -0300 Subject: [PATCH 17/36] SIENTIAPDE-1110 Enhance MongoDB data return format by converting to DataFrame dictionary, improving data structure for downstream processing. --- scouter/activities/mongodb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index ce70bf8..67ac799 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -1,3 +1,4 @@ +from pandas import DataFrame from temporalio import workflow, activity with workflow.unsafe.imports_passed_through(): @@ -119,7 +120,7 @@ class MongoDB(BaseActivity): metadata=metadata ) - return data + return DataFrame(data).to_dict() except Exception as e: trace = traceback.format_exc() self.send_notification( From c9b9974f9d9a74e8a6f225c7b9282600f924737c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 15:41:49 -0300 Subject: [PATCH 18/36] SIENTIAPDE-1110 Refactor Redis activity to remove unused 'ttl' parameter in set method, streamlining data handling. --- scouter/activities/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 1083d27..5aea18e 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -55,7 +55,7 @@ class Redis(RedisBase): metadata=metadata ) - self.set(key, last_data_timestamp, ttl=input_data['retention_time']) + self.set(key, last_data_timestamp) return last_data_timestamp From 0e915f5491dd13776e595c765797cc6921644308 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 15:45:03 -0300 Subject: [PATCH 19/36] SIENTIAPDE-1110 Update Redis activity to include specific prefixes for keys in last data timestamp and held data, enhancing clarity in data management. --- scouter/activities/redis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 5aea18e..803ada8 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -44,7 +44,7 @@ class Redis(RedisBase): Puts the last data timestamp into redis. """ metadata = input_data['metadata'] - key = f"{input_data['workflow_name']}_{input_data['schedule_name']}" + key = f"last_data_timestamp_{input_data['workflow_name']}_{input_data['schedule_name']}" data = DataFrame(input_data['data']) @@ -81,7 +81,7 @@ class Redis(RedisBase): data = DataFrame(input_data['data']) retention_time = input_data['retention_time'] - key = f"{input_data['workflow_name']}_{input_data['schedule_name']}" + key = f"held_data_{input_data['workflow_name']}_{input_data['schedule_name']}" data_hold = self.get(key) From 337a54217c718bb7d95c281c2cce91f8c1d2dbe4 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 15:48:18 -0300 Subject: [PATCH 20/36] SIENTIAPDE-1110 Fix aggregation function key in Gates activity from 'aggr_function' to 'aggr_func' for consistency in data processing. --- scouter/activities/gates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py index 3db376b..fbdb5e0 100644 --- a/scouter/activities/gates.py +++ b/scouter/activities/gates.py @@ -93,7 +93,7 @@ class Gates(BaseActivity): for (tag, name), group in grouped: # Get the aggregation function from model_tags aggr_function = input_data['model_tags'].get( - name, {}).get('aggr_function', 'lts') + name, {}).get('aggr_func', 'lts') group.sort_values(by='timestamp', inplace=True) From b71880129095cbbb4e4fcd9ee4b31e0f47fd9fcc Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 15:53:47 -0300 Subject: [PATCH 21/36] SIENTIAPDE-1110 Update get_last_data_timestamp method in Redis activity to return a string or None, improving type clarity for timestamp retrieval. --- scouter/activities/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 803ada8..5689472 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -19,7 +19,7 @@ class Redis(RedisBase): password, logger, notification_handler) @activity.defn(name="get_last_data_timestamp") - async def get_last_data_timestamp(self, input_data: dict[str, Any]) -> dict[str, Any]: + async def get_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None: """ Gets the last data timestamp from redis. """ From 82ce1852e3f8a0b4bbe3d898b48b2589c86e74e5 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 16:03:40 -0300 Subject: [PATCH 22/36] SIENTIAPDE-1110 Add debug logging for data filter in MongoDB activity to enhance traceability of filtering criteria. --- scouter/activities/mongodb.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index 67ac799..2bf2813 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -102,6 +102,11 @@ class MongoDB(BaseActivity): } } + self.debug( + f"Data filter: {data_filter}", + metadata=metadata + ) + data = list(self.database[collection_name].find( data_filter, {"_id": 0})) From defac7b1f0f7e7e1c689e63b1062198c074a2cc6 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 16:06:52 -0300 Subject: [PATCH 23/36] SIENTIAPDE-1110 Update Redis activity to include a specific prefix for the last data timestamp key, enhancing clarity in key management. --- scouter/activities/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 5689472..b4ad730 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -24,7 +24,7 @@ class Redis(RedisBase): Gets the last data timestamp from redis. """ metadata = input_data['metadata'] - key = f"{input_data['workflow_name']}_{input_data['schedule_name']}" + key = f"last_data_timestamp_{input_data['workflow_name']}_{input_data['schedule_name']}" data_hold = self.get(key) From 91f2d8610bc62f1d437eee150409fa132c28bd0e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Jul 2025 16:09:46 -0300 Subject: [PATCH 24/36] SIENTIAPDE-1110 Update MongoDB activity to format 'inserted_at' timestamps as strings in the format 'YYYY-MM-DD HH:MM:SS.ssssss', improving readability of timestamp data. --- scouter/activities/mongodb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index 2bf2813..a02db81 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -113,7 +113,8 @@ class MongoDB(BaseActivity): data = clear_mongo_id(data) for item in data: - item['inserted_at'] = item['inserted_at'].isoformat() + item['inserted_at'] = item['inserted_at'].strftime( + "%Y-%m-%d %H:%M:%S.%f") self.info( f"Loaded {len(data)} documents from MongoDB", From 332612ac8481b19e89a23033a6ee513b78ec6c08 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 2 Jul 2025 09:02:59 -0300 Subject: [PATCH 25/36] SIENTIAPDE-1110 Integrate Druid activity into the Activities class, adding support for Druid configuration and initialization. Update worker and connectors configuration to accommodate Druid, enhancing data processing capabilities. --- requirements.txt | 1 + scouter/activities/activities.py | 13 +++++- scouter/activities/mongodb.py | 4 +- scouter/activities/pydruid.py | 74 ++++++++++++++++++++++++++++++ scouter/utils/connectors_config.py | 7 +++ scouter/worker/worker.py | 7 ++- scouter/workflow/scouter.py | 15 +++++- 7 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 scouter/activities/pydruid.py diff --git a/requirements.txt b/requirements.txt index 3e098f3..c249857 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ aiokafka pymongo git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.2.0 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.1 +pydruid[pandas] \ No newline at end of file diff --git a/scouter/activities/activities.py b/scouter/activities/activities.py index ddeae2e..53f7f0f 100644 --- a/scouter/activities/activities.py +++ b/scouter/activities/activities.py @@ -8,10 +8,11 @@ with workflow.unsafe.imports_passed_through(): from scouter.activities.kafka import Kafka from scouter.activities.gates import Gates from scouter.activities.mongodb import MongoDB + from scouter.activities.pydruid import Druid from typing import Any -class Activities(Postgres, Redis, Kafka, Gates, MongoDB): +class Activities(Postgres, Redis, Kafka, Gates, MongoDB, Druid): """Activities class that combines multiple services with proper initialization.""" def __init__(self, @@ -19,6 +20,7 @@ class Activities(Postgres, Redis, Kafka, Gates, MongoDB): redis_config: dict[str, Any], kafka_config: dict[str, Any], mongodb_config: dict[str, Any], + druid_config: dict[str, Any], logger: Logger, notification_handler: NotificationHandler): @@ -73,6 +75,15 @@ class Activities(Postgres, Redis, Kafka, Gates, MongoDB): notification_handler=notification_handler ) + # Initialize Druid + Druid.__init__( + self, + host=druid_config['host'], + port=druid_config['port'], + logger=logger, + notification_handler=notification_handler + ) + def shutdown(self): Postgres.close(self) Kafka.close(self) diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index a02db81..2191d55 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -1,15 +1,15 @@ -from pandas import DataFrame from temporalio import workflow, activity with workflow.unsafe.imports_passed_through(): from typing import Any import traceback - from logging import Logger from datetime import datetime from pymongo import MongoClient + from pandas import DataFrame from sientia_do.notifications.handlers import NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.base import BaseActivity + from sientia_do.temporal.utils.logger import Logger def clear_mongo_id(docs: list) -> list: diff --git a/scouter/activities/pydruid.py b/scouter/activities/pydruid.py new file mode 100644 index 0000000..2dc2478 --- /dev/null +++ b/scouter/activities/pydruid.py @@ -0,0 +1,74 @@ +from temporalio import workflow, activity + +with workflow.unsafe.imports_passed_through(): + import pandas as pd + from typing import List, Optional, Any + from datetime import datetime, timedelta + from pydruid.client import PyDruid + from pydruid.query import QueryBuilder + from sientia_do.temporal.activities.base import BaseActivity + from sientia_do.notifications.handlers import NotificationHandler + from sientia_do.temporal.utils.logger import Logger + + +class Druid(BaseActivity): + def __init__(self, host: str, port: int, + logger: Logger, notification_handler: NotificationHandler, + endpoint: str = "druid/v2"): + + self.host = host + self.port = port + self.endpoint = endpoint + self.client = PyDruid( + f"http://{self.host}:{self.port}", {self.endpoint} + ) + logger.info( + f"Druid client initialized with host: {self.host}, port: {self.port}") + + BaseActivity.__init__(self, logger=logger, + notification_handler=notification_handler) + + def shutdown(self): + self.client.close() + + def __del__(self): + self.shutdown() + + @activity.defn(name="load_latest_druid_data") + async def load_latest_druid_data(self, input_data: dict[str, Any]) -> dict[str, Any]: + """ + Loads the latest data from Druid. + """ + metadata = input_data['metadata'] + datasource = f"raw_{input_data['schedule_name']}" + last_data_timestamp = datetime.strptime( + input_data['last_data_timestamp'], "%Y-%m-%d %H:%M:%S.%f") + + self.debug( + f"Loading data from Druid: {input_data}", metadata=metadata) + + end_time = datetime(9999, 12, 31, 23, 59, 59) + + interval = f"{last_data_timestamp.isoformat()}Z/{end_time.isoformat()}Z" + + builder = QueryBuilder() + + query = builder.scan( + { + "datasource": datasource, + "intervals": interval, + "columns": ["timestamp", "value", "tag"], + "limit": 10000, + } + ) + + result = query.export_pandas() + + self.info( + f"Loaded {len(result)} rows from Druid" + ) + + self.debug( + f"Druid query result: {result}", metadata=metadata) + + return result.to_dict(orient="records") diff --git a/scouter/utils/connectors_config.py b/scouter/utils/connectors_config.py index 64a7feb..2179fac 100644 --- a/scouter/utils/connectors_config.py +++ b/scouter/utils/connectors_config.py @@ -40,3 +40,10 @@ def build_mongodb_config(): 'connection_string': connection_string, 'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia') } + + +def build_druid_config(): + return { + 'host': getenv('DRUID_HOST', 'localhost'), + 'port': int(getenv('DRUID_PORT', '8082')), + } diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 74ad479..b99c05c 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -16,7 +16,8 @@ with workflow.unsafe.imports_passed_through(): build_postgres_config, build_kafka_config, build_redis_config, - build_mongodb_config + build_mongodb_config, + build_druid_config ) @@ -43,7 +44,8 @@ async def main(): postgres_config=build_postgres_config(), kafka_config=build_kafka_config(), redis_config=build_redis_config(), - mongodb_config=build_mongodb_config() + mongodb_config=build_mongodb_config(), + druid_config=build_druid_config() ) logger.info('Starting Faker Activities...') @@ -71,6 +73,7 @@ async def main(): workflows=[Scouter, CoreScouter], activities=[ activities.load_latest_data, + activities.load_latest_druid_data, activities.get_last_data_timestamp, activities.put_last_data_timestamp, activities.load_from_kafka, diff --git a/scouter/workflow/scouter.py b/scouter/workflow/scouter.py index cf36c49..e9ca1f2 100644 --- a/scouter/workflow/scouter.py +++ b/scouter/workflow/scouter.py @@ -62,11 +62,22 @@ class Scouter: retry_policy=retry_policy ) + # data = await workflow.execute_local_activity_method( + # Activities.load_latest_data, + # { + # **metadata, + # 'collection_name': f"raw_{input_data['schedule_name']}", + # 'last_data_timestamp': last_data_timestamp + # }, + # start_to_close_timeout=timedelta(seconds=60), + # retry_policy=retry_policy + # ) + data = await workflow.execute_local_activity_method( - Activities.load_latest_data, + Activities.load_latest_druid_data, { **metadata, - 'collection_name': f"raw_{input_data['schedule_name']}", + 'schedule_name': input_data['schedule_name'], 'last_data_timestamp': last_data_timestamp }, start_to_close_timeout=timedelta(seconds=60), From c3c8e1b7666df6c13dac0ec9f908fd46855d6200 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 2 Jul 2025 09:25:46 -0300 Subject: [PATCH 26/36] SIENTIAPDE-1110 Update Druid activity to handle None values for last data timestamp, ensuring robust date parsing. Bump image tag in values.yaml to 0.2.3 for version consistency. --- scouter/activities/pydruid.py | 8 ++++++-- values.yaml | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scouter/activities/pydruid.py b/scouter/activities/pydruid.py index 2dc2478..cb9998c 100644 --- a/scouter/activities/pydruid.py +++ b/scouter/activities/pydruid.py @@ -41,8 +41,12 @@ class Druid(BaseActivity): """ metadata = input_data['metadata'] datasource = f"raw_{input_data['schedule_name']}" - last_data_timestamp = datetime.strptime( - input_data['last_data_timestamp'], "%Y-%m-%d %H:%M:%S.%f") + last_data_timestamp_str = input_data['last_data_timestamp'] + if last_data_timestamp_str is None: + last_data_timestamp = datetime(1970, 1, 1, 0, 0, 0) + else: + last_data_timestamp = datetime.strptime( + last_data_timestamp_str, "%Y-%m-%d %H:%M:%S.%f") self.debug( f"Loading data from Druid: {input_data}", metadata=metadata) diff --git a/values.yaml b/values.yaml index 037bc0c..589a8c7 100644 --- a/values.yaml +++ b/values.yaml @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.2.2" + tag: "0.2.3" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: From 879c1fdc495939afe04db1d776a21790245fee77 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 2 Jul 2025 09:27:02 -0300 Subject: [PATCH 27/36] SIENTIAPDE-1110 Add logging for data loading in Druid activity, enhancing traceability of data retrieval intervals. --- scouter/activities/pydruid.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scouter/activities/pydruid.py b/scouter/activities/pydruid.py index cb9998c..ec0cc21 100644 --- a/scouter/activities/pydruid.py +++ b/scouter/activities/pydruid.py @@ -55,6 +55,10 @@ class Druid(BaseActivity): interval = f"{last_data_timestamp.isoformat()}Z/{end_time.isoformat()}Z" + self.info( + f"Loading data from Druid: {datasource} with interval: {interval}" + ) + builder = QueryBuilder() query = builder.scan( From 5decb018d271126e71cb05aa9d2b786a11b503d4 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 2 Jul 2025 09:27:14 -0300 Subject: [PATCH 28/36] SIENTIAPDE-1110 --- scouter/activities/pydruid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scouter/activities/pydruid.py b/scouter/activities/pydruid.py index ec0cc21..d1aa0ee 100644 --- a/scouter/activities/pydruid.py +++ b/scouter/activities/pydruid.py @@ -79,4 +79,4 @@ class Druid(BaseActivity): self.debug( f"Druid query result: {result}", metadata=metadata) - return result.to_dict(orient="records") + return result.to_dict() From 430fe04202aad01b2d15de56eae9f6ab82c4c0a8 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 2 Jul 2025 10:19:04 -0300 Subject: [PATCH 29/36] SIENTIAPDE-1110 Enhance Druid activity by adding configuration for Druid host and port in values.yaml. Refactor data loading logic to utilize SQLAlchemy for querying Druid, improving query efficiency and readability. Update timestamp handling to streamline data retrieval process. --- scouter/activities/pydruid.py | 42 ++---- test.ipynb | 267 ++++++++++++++++++++++++++++++++++ values.yaml | 5 + 3 files changed, 287 insertions(+), 27 deletions(-) create mode 100644 test.ipynb diff --git a/scouter/activities/pydruid.py b/scouter/activities/pydruid.py index d1aa0ee..473c173 100644 --- a/scouter/activities/pydruid.py +++ b/scouter/activities/pydruid.py @@ -6,6 +6,8 @@ with workflow.unsafe.imports_passed_through(): from datetime import datetime, timedelta from pydruid.client import PyDruid from pydruid.query import QueryBuilder + from sqlalchemy.engine import create_engine + from sqlalchemy import MetaData, Table, select, text from sientia_do.temporal.activities.base import BaseActivity from sientia_do.notifications.handlers import NotificationHandler from sientia_do.temporal.utils.logger import Logger @@ -13,15 +15,12 @@ with workflow.unsafe.imports_passed_through(): class Druid(BaseActivity): def __init__(self, host: str, port: int, - logger: Logger, notification_handler: NotificationHandler, - endpoint: str = "druid/v2"): + logger: Logger, notification_handler: NotificationHandler): self.host = host self.port = port - self.endpoint = endpoint - self.client = PyDruid( - f"http://{self.host}:{self.port}", {self.endpoint} - ) + self.engine = create_engine( + f'druid://{self.host}:{self.port}/druid/v2/sql/') logger.info( f"Druid client initialized with host: {self.host}, port: {self.port}") @@ -41,36 +40,25 @@ class Druid(BaseActivity): """ metadata = input_data['metadata'] datasource = f"raw_{input_data['schedule_name']}" - last_data_timestamp_str = input_data['last_data_timestamp'] - if last_data_timestamp_str is None: - last_data_timestamp = datetime(1970, 1, 1, 0, 0, 0) - else: - last_data_timestamp = datetime.strptime( - last_data_timestamp_str, "%Y-%m-%d %H:%M:%S.%f") - + last_data_timestamp = input_data['last_data_timestamp'] self.debug( f"Loading data from Druid: {input_data}", metadata=metadata) - end_time = datetime(9999, 12, 31, 23, 59, 59) - - interval = f"{last_data_timestamp.isoformat()}Z/{end_time.isoformat()}Z" + query = f'"__time" > TIMESTAMP \'{last_data_timestamp}\'' self.info( - f"Loading data from Druid: {datasource} with interval: {interval}" + f"Loading data from Druid: {datasource} with query: {query}" ) - builder = QueryBuilder() + places = Table(datasource, MetaData(), autoload_with=self.engine) + stmt = select(places).where(text(query)) - query = builder.scan( - { - "datasource": datasource, - "intervals": interval, - "columns": ["timestamp", "value", "tag"], - "limit": 10000, - } - ) + result = pd.read_sql(stmt, self.engine) - result = query.export_pandas() + result["inserted_at"] = pd.to_datetime(result["__time"]).dt.strftime( + "%Y-%m-%d %H:%M:%S.%f") + + result.drop(columns=["__time"], inplace=True) self.info( f"Loaded {len(result)} rows from Druid" diff --git a/test.ipynb b/test.ipynb new file mode 100644 index 0000000..fb78908 --- /dev/null +++ b/test.ipynb @@ -0,0 +1,267 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy.engine import create_engine\n", + "\n", + "engine = create_engine('druid://localhost:8082/druid/v2/sql/')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy import MetaData, Table\n", + "\n", + "metadata = MetaData()\n", + "places = Table('raw_scouter-opcua-orchestrated-pipeline', metadata, autoload_with=engine)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy import select\n", + "\n", + "stmt = select(places)\n", + "with engine.connect() as conn:\n", + " result = conn.execute(stmt)\n", + " for row in result:\n", + " print(row)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_30057/1086103244.py:5: SADeprecationWarning: The dbapi() classmethod on dialect classes has been renamed to import_dbapi(). Implement an import_dbapi() classmethod directly on class to remove this warning; the old .dbapi() classmethod may be maintained for backwards compatibility.\n", + " engine = create_engine('druid://localhost:8082/druid/v2/sql/')\n", + "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pydruid/db/sqlalchemy.py:188: SAWarning: Dialect druid:rest will not make use of SQL compilation caching as it does not set the 'supports_statement_cache' attribute to ``True``. This can have significant performance implications including some performance degradations in comparison to prior SQLAlchemy versions. Dialect maintainers should seek to set this attribute to True after appropriate development and testing for SQLAlchemy 1.4 caching support. Alternatively, this attribute may be set to False which will disable this warning. (Background on this warning at: https://sqlalche.me/e/20/cprf)\n", + " result = connection.execute(text(query))\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namekafka.topictagvaluetimestampinserted_at
0Counterraw_scouter-opcua-orchestrated-pipelinens=2;i=2-50.1322025-07-02 13:05:192025-07-02 13:05:19.729000
1Rolloutraw_scouter-opcua-orchestrated-pipelinens=2;i=370.7672025-07-02 13:05:192025-07-02 13:05:19.731000
2Squareraw_scouter-opcua-orchestrated-pipelinens=2;i=4-58.4482025-07-02 13:05:192025-07-02 13:05:19.732000
3Counterraw_scouter-opcua-orchestrated-pipelinens=2;i=2-50.1262025-07-02 13:05:242025-07-02 13:05:24.728000
4Rolloutraw_scouter-opcua-orchestrated-pipelinens=2;i=369.1992025-07-02 13:05:242025-07-02 13:05:24.730000
.....................
373Rolloutraw_scouter-opcua-orchestrated-pipelinens=2;i=385.9212025-07-02 13:15:402025-07-02 13:15:40.230000
374Squareraw_scouter-opcua-orchestrated-pipelinens=2;i=4-69.2962025-07-02 13:15:402025-07-02 13:15:40.232000
375Counterraw_scouter-opcua-orchestrated-pipelinens=2;i=2-71.2072025-07-02 13:15:452025-07-02 13:15:45.228000
376Rolloutraw_scouter-opcua-orchestrated-pipelinens=2;i=384.6652025-07-02 13:15:452025-07-02 13:15:45.231000
377Squareraw_scouter-opcua-orchestrated-pipelinens=2;i=4-67.5922025-07-02 13:15:452025-07-02 13:15:45.233000
\n", + "

378 rows × 6 columns

\n", + "
" + ], + "text/plain": [ + " name kafka.topic tag value \\\n", + "0 Counter raw_scouter-opcua-orchestrated-pipeline ns=2;i=2 -50.132 \n", + "1 Rollout raw_scouter-opcua-orchestrated-pipeline ns=2;i=3 70.767 \n", + "2 Square raw_scouter-opcua-orchestrated-pipeline ns=2;i=4 -58.448 \n", + "3 Counter raw_scouter-opcua-orchestrated-pipeline ns=2;i=2 -50.126 \n", + "4 Rollout raw_scouter-opcua-orchestrated-pipeline ns=2;i=3 69.199 \n", + ".. ... ... ... ... \n", + "373 Rollout raw_scouter-opcua-orchestrated-pipeline ns=2;i=3 85.921 \n", + "374 Square raw_scouter-opcua-orchestrated-pipeline ns=2;i=4 -69.296 \n", + "375 Counter raw_scouter-opcua-orchestrated-pipeline ns=2;i=2 -71.207 \n", + "376 Rollout raw_scouter-opcua-orchestrated-pipeline ns=2;i=3 84.665 \n", + "377 Square raw_scouter-opcua-orchestrated-pipeline ns=2;i=4 -67.592 \n", + "\n", + " timestamp inserted_at \n", + "0 2025-07-02 13:05:19 2025-07-02 13:05:19.729000 \n", + "1 2025-07-02 13:05:19 2025-07-02 13:05:19.731000 \n", + "2 2025-07-02 13:05:19 2025-07-02 13:05:19.732000 \n", + "3 2025-07-02 13:05:24 2025-07-02 13:05:24.728000 \n", + "4 2025-07-02 13:05:24 2025-07-02 13:05:24.730000 \n", + ".. ... ... \n", + "373 2025-07-02 13:15:40 2025-07-02 13:15:40.230000 \n", + "374 2025-07-02 13:15:40 2025-07-02 13:15:40.232000 \n", + "375 2025-07-02 13:15:45 2025-07-02 13:15:45.228000 \n", + "376 2025-07-02 13:15:45 2025-07-02 13:15:45.231000 \n", + "377 2025-07-02 13:15:45 2025-07-02 13:15:45.233000 \n", + "\n", + "[378 rows x 6 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from sqlalchemy import create_engine, MetaData, Table, select, func, text\n", + "import pandas as pd\n", + "from datetime import datetime\n", + "\n", + "engine = create_engine('druid://localhost:8082/druid/v2/sql/')\n", + "metadata = MetaData()\n", + "places = Table('raw_scouter-opcua-orchestrated-pipeline', metadata, autoload_with=engine)\n", + "date_str = '2025-01-01'\n", + "stmt = select(places).where(text(f'\"__time\" > TIMESTAMP \\'{date_str}\\''))\n", + "\n", + "result = pd.read_sql(stmt, engine)\n", + "\n", + "result[\"inserted_at\"] = pd.to_datetime(result[\"__time\"]).dt.strftime(\n", + " \"%Y-%m-%d %H:%M:%S.%f\")\n", + "\n", + "result.drop(columns=[\"__time\"], inplace=True)\n", + "\n", + "display(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/values.yaml b/values.yaml index 589a8c7..884ccf7 100644 --- a/values.yaml +++ b/values.yaml @@ -182,6 +182,11 @@ env: - name: MONGODB_DATABASE value: "sientia" + - name: DRUID_HOST + value: "druid-router.druid.svc.cluster.local" + - name: DRUID_PORT + value: "8081" + ssh: enabled: true secretName: git-ssh-key-sientia-scouter-worker From 3bbc6a6a7930ec24f3a862e21645ecd1a2ac4f2a Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 2 Jul 2025 10:21:55 -0300 Subject: [PATCH 30/36] SIENTIAPDE-1110 Update Druid activity to handle None values for last data timestamp, ensuring default timestamp is set to epoch start. Enhance data loading debug logging for improved traceability. --- scouter/activities/pydruid.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scouter/activities/pydruid.py b/scouter/activities/pydruid.py index 473c173..b74b7ae 100644 --- a/scouter/activities/pydruid.py +++ b/scouter/activities/pydruid.py @@ -41,6 +41,8 @@ class Druid(BaseActivity): metadata = input_data['metadata'] datasource = f"raw_{input_data['schedule_name']}" last_data_timestamp = input_data['last_data_timestamp'] + last_data_timestamp = last_data_timestamp if last_data_timestamp is not None else datetime( + 1970, 1, 1, 0, 0, 0) self.debug( f"Loading data from Druid: {input_data}", metadata=metadata) From 6caec639770ef323d7ee9f940454a06259ec167d Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 2 Jul 2025 10:24:35 -0300 Subject: [PATCH 31/36] SIENTIAPDE-1110 Refactor last data timestamp handling in Druid activity to use string format for default value, improving consistency in date representation. --- scouter/activities/pydruid.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scouter/activities/pydruid.py b/scouter/activities/pydruid.py index b74b7ae..e3f1750 100644 --- a/scouter/activities/pydruid.py +++ b/scouter/activities/pydruid.py @@ -41,8 +41,7 @@ class Druid(BaseActivity): metadata = input_data['metadata'] datasource = f"raw_{input_data['schedule_name']}" last_data_timestamp = input_data['last_data_timestamp'] - last_data_timestamp = last_data_timestamp if last_data_timestamp is not None else datetime( - 1970, 1, 1, 0, 0, 0) + last_data_timestamp = last_data_timestamp if last_data_timestamp is not None else '1970-01-01 00:00:00' self.debug( f"Loading data from Druid: {input_data}", metadata=metadata) From 208491498f22506051e698abe747b47cfc084fcd Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 2 Jul 2025 10:35:53 -0300 Subject: [PATCH 32/36] SIENTIAPDE-1110 Update Druid configuration in values.yaml to change port from 8081 to 8888. Refactor pydruid.py to use druid_engine for SQLAlchemy connections, enhancing clarity and consistency in data loading operations. --- scouter/activities/pydruid.py | 6 +++--- values.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scouter/activities/pydruid.py b/scouter/activities/pydruid.py index e3f1750..f19e4bb 100644 --- a/scouter/activities/pydruid.py +++ b/scouter/activities/pydruid.py @@ -19,7 +19,7 @@ class Druid(BaseActivity): self.host = host self.port = port - self.engine = create_engine( + self.druid_engine = create_engine( f'druid://{self.host}:{self.port}/druid/v2/sql/') logger.info( f"Druid client initialized with host: {self.host}, port: {self.port}") @@ -51,10 +51,10 @@ class Druid(BaseActivity): f"Loading data from Druid: {datasource} with query: {query}" ) - places = Table(datasource, MetaData(), autoload_with=self.engine) + places = Table(datasource, MetaData(), autoload_with=self.druid_engine) stmt = select(places).where(text(query)) - result = pd.read_sql(stmt, self.engine) + result = pd.read_sql(stmt, self.druid_engine) result["inserted_at"] = pd.to_datetime(result["__time"]).dt.strftime( "%Y-%m-%d %H:%M:%S.%f") diff --git a/values.yaml b/values.yaml index 884ccf7..6686b55 100644 --- a/values.yaml +++ b/values.yaml @@ -185,7 +185,7 @@ env: - name: DRUID_HOST value: "druid-router.druid.svc.cluster.local" - name: DRUID_PORT - value: "8081" + value: "8888" ssh: enabled: true From 3ee7e3687a240f1abdbdb9eddff0989553aa0a01 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 3 Jul 2025 14:00:09 -0300 Subject: [PATCH 33/36] SIENTIAPDE-1110 Update test.ipynb to generate and save multiple pipeline specifications and input samples in JSON format. Modify values.yaml to increase replica count from 1 to 3 and adjust PostgreSQL max connections from 20 to 40 for improved scalability. --- input_samples_30.json | 1568 +++++++++++++++++ specs_30.json | 3743 +++++++++++++++++++++++++++++++++++++++++ test.ipynb | 457 +++++ values.yaml | 4 +- 4 files changed, 5770 insertions(+), 2 deletions(-) create mode 100644 input_samples_30.json create mode 100644 specs_30.json diff --git a/input_samples_30.json b/input_samples_30.json new file mode 100644 index 0000000..3455816 --- /dev/null +++ b/input_samples_30.json @@ -0,0 +1,1568 @@ +[ + { + "id": 2, + "schedule_name": "scouter-opcua-pipeline-2", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 3, + "schedule_name": "scouter-opcua-pipeline-3", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 4, + "schedule_name": "scouter-opcua-pipeline-4", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 5, + "schedule_name": "scouter-opcua-pipeline-5", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 6, + "schedule_name": "scouter-opcua-pipeline-6", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 7, + "schedule_name": "scouter-opcua-pipeline-7", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 8, + "schedule_name": "scouter-opcua-pipeline-8", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 9, + "schedule_name": "scouter-opcua-pipeline-9", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 10, + "schedule_name": "scouter-opcua-pipeline-10", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 11, + "schedule_name": "scouter-opcua-pipeline-11", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 12, + "schedule_name": "scouter-opcua-pipeline-12", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 13, + "schedule_name": "scouter-opcua-pipeline-13", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 14, + "schedule_name": "scouter-opcua-pipeline-14", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 15, + "schedule_name": "scouter-opcua-pipeline-15", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 16, + "schedule_name": "scouter-opcua-pipeline-16", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 17, + "schedule_name": "scouter-opcua-pipeline-17", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 18, + "schedule_name": "scouter-opcua-pipeline-18", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 19, + "schedule_name": "scouter-opcua-pipeline-19", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 20, + "schedule_name": "scouter-opcua-pipeline-20", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 21, + "schedule_name": "scouter-opcua-pipeline-21", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 22, + "schedule_name": "scouter-opcua-pipeline-22", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 23, + "schedule_name": "scouter-opcua-pipeline-23", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 24, + "schedule_name": "scouter-opcua-pipeline-24", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 25, + "schedule_name": "scouter-opcua-pipeline-25", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 26, + "schedule_name": "scouter-opcua-pipeline-26", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 27, + "schedule_name": "scouter-opcua-pipeline-27", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 28, + "schedule_name": "scouter-opcua-pipeline-28", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 29, + "schedule_name": "scouter-opcua-pipeline-29", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + }, + { + "id": 30, + "schedule_name": "scouter-opcua-pipeline-30", + "model_id": "1", + "workflow_type": "scouter", + "frequency": "30s", + "max_retry_policy": 1, + "read_tags": [ + { + "tag_name": "Counter", + "server_id": "1", + "aggr_func": "avg", + "tag_address": "ns=2;i=2", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Rollout", + "server_id": "1", + "aggr_func": "mdn", + "tag_address": "ns=2;i=3", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + }, + { + "tag_name": "Square", + "server_id": "1", + "aggr_func": "lts", + "tag_address": "ns=2;i=4", + "frequency": "15000", + "data_range": [ + -100, + 100 + ] + } + ], + "filters": [ + { + "filter_name": "OUT_OF_BOUNDS_FILTER", + "policy": "DISCARD" + }, + { + "filter_name": "NULL_VALUES_FILTER", + "policy": "DISCARD" + } + ], + "tag_retention_minutes": 60 + } +] \ No newline at end of file diff --git a/specs_30.json b/specs_30.json new file mode 100644 index 0000000..60be595 --- /dev/null +++ b/specs_30.json @@ -0,0 +1,3743 @@ +[ + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-2", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-2", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-2", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-3", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-3", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-3", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-4", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-4", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-4", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-5", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-5", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-5", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-6", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-6", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-6", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-7", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-7", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-7", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-8", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-8", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-8", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-9", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-9", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-9", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-10", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-10", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-10", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-11", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-11", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-11", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-12", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-12", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-12", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-13", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-13", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-13", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-14", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-14", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-14", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-15", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-15", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-15", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-16", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-16", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-16", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-17", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-17", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-17", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-18", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-18", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-18", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-19", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-19", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-19", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-20", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-20", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-20", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-21", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-21", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-21", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-22", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-22", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-22", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-23", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-23", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-23", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-24", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-24", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-24", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-25", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-25", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-25", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-26", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-26", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-26", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-27", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-27", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-27", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-28", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-28", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-28", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-29", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-29", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-29", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + }, + { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "raw_scouter-opcua-pipeline-30", + "timestampSpec": { + "column": "kafka.timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [], + "dimensionExclusions": [ + "__time", + "kafka.timestamp" + ], + "includeAllDimensions": false, + "useSchemaDiscovery": true + }, + "metricsSpec": [], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "none" + }, + "rollup": false, + "intervals": [] + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "raw_scouter-opcua-pipeline-30", + "topicPattern": null, + "inputFormat": { + "type": "kafka", + "headerFormat": null, + "keyFormat": null, + "valueFormat": { + "type": "json", + "keepNoneColumns": false, + "assumeNewlineDelimited": false, + "useJsonNodeReader": false + }, + "headerColumnPrefix": "kafka.header.", + "keyColumnName": "kafka.key", + "timestampColumnName": "kafka.timestamp", + "topicColumnName": "kafka.topic" + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "kafka.kafka.svc.cluster.local:9092" + }, + "autoScalerConfig": null, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": true, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "configOverrides": null, + "idleConfig": null, + "stopTaskCount": null, + "stream": "raw_scouter-opcua-pipeline-30", + "useEarliestSequenceNumber": true + }, + "tuningConfig": { + "type": "kafka", + "appendableIndexSpec": { + "type": "onheap", + "preserveExistingMetrics": false + }, + "maxRowsInMemory": 150000, + "maxBytesInMemory": 0, + "skipBytesInMemoryOverheadCheck": false, + "maxRowsPerSegment": 5000000, + "maxTotalRows": null, + "intermediatePersistPeriod": "PT10M", + "maxPendingPersists": 0, + "indexSpec": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "indexSpecForIntermediatePersists": { + "bitmap": { + "type": "roaring" + }, + "dimensionCompression": "lz4", + "stringDictionaryEncoding": { + "type": "utf8" + }, + "metricCompression": "lz4", + "longEncoding": "longs" + }, + "reportParseExceptions": false, + "handoffConditionTimeout": 900000, + "resetOffsetAutomatically": false, + "segmentWriteOutMediumFactory": null, + "workerThreads": null, + "chatRetries": 8, + "httpTimeout": "PT10S", + "shutdownTimeout": "PT80S", + "offsetFetchPeriod": "PT30S", + "intermediateHandoffPeriod": "P2147483647D", + "logParseExceptions": false, + "maxParseExceptions": 2147483647, + "maxSavedParseExceptions": 0, + "numPersistThreads": 1, + "skipSequenceNumberAvailabilityCheck": false, + "repartitionTransitionDuration": "PT120S" + } + }, + "context": null, + "suspended": false + } +] \ No newline at end of file diff --git a/test.ipynb b/test.ipynb index fb78908..7e5c2a0 100644 --- a/test.ipynb +++ b/test.ipynb @@ -241,6 +241,463 @@ "\n", "display(result)" ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "data = {\n", + " \"id\": \"1\",\n", + " \"schedule_name\": \"scouter-opcua-orchestrated-pipeline\",\n", + " \"model_id\": \"1\",\n", + " \"workflow_type\": \"scouter\",\n", + " \"frequency\": \"30s\",\n", + " \"max_retry_policy\": 1,\n", + " \"read_tags\": [\n", + " {\n", + " \"tag_name\": \"Counter\",\n", + " \"server_id\": \"1\",\n", + " \"aggr_func\": \"avg\",\n", + " \"tag_address\": \"ns=2;i=2\",\n", + " \"frequency\": \"15000\",\n", + " \"data_range\": [\n", + " -100,\n", + " 100\n", + " ]\n", + " },\n", + " {\n", + " \"tag_name\": \"Rollout\",\n", + " \"server_id\": \"1\",\n", + " \"aggr_func\": \"mdn\",\n", + " \"tag_address\": \"ns=2;i=3\",\n", + " \"frequency\": \"15000\",\n", + " \"data_range\": [\n", + " -100,\n", + " 100\n", + " ]\n", + " },\n", + " {\n", + " \"tag_name\": \"Square\",\n", + " \"server_id\": \"1\",\n", + " \"aggr_func\": \"lts\",\n", + " \"tag_address\": \"ns=2;i=4\",\n", + " \"frequency\": \"15000\",\n", + " \"data_range\": [\n", + " -100,\n", + " 100\n", + " ]\n", + " }\n", + " ],\n", + " \"filters\": [\n", + " {\n", + " \"filter_name\": \"OUT_OF_BOUNDS_FILTER\",\n", + " \"policy\": \"DISCARD\"\n", + " },\n", + " {\n", + " \"filter_name\": \"NULL_VALUES_FILTER\",\n", + " \"policy\": \"DISCARD\"\n", + " }\n", + " ],\n", + " \"tag_retention_minutes\": 60\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "\n", + "# Generate 29 more, changing only id and schedule_name\n", + "json_list = []\n", + "for i in range(30):\n", + " obj = data.copy()\n", + " obj['id'] = i + 1 # or any other unique id logic\n", + " obj['schedule_name'] = f\"scouter-opcua-pipeline-{i+1}\"\n", + " json_list.append(obj)\n", + "\n", + "# Save to a new file\n", + "with open('input_samples_30.json', 'w') as f:\n", + " json.dump(json_list, f, indent=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "spec = {\n", + " \"type\": \"kafka\",\n", + " \"spec\": {\n", + " \"dataSchema\": {\n", + " \"dataSource\": \"raw_scouter-opcua-orchestrated-pipeline\",\n", + " \"timestampSpec\": {\n", + " \"column\": \"kafka.timestamp\",\n", + " \"format\": \"millis\",\n", + " \"missingValue\": None\n", + " },\n", + " \"dimensionsSpec\": {\n", + " \"dimensions\": [],\n", + " \"dimensionExclusions\": [\n", + " \"__time\",\n", + " \"kafka.timestamp\"\n", + " ],\n", + " \"includeAllDimensions\": False,\n", + " \"useSchemaDiscovery\": True\n", + " },\n", + " \"metricsSpec\": [],\n", + " \"granularitySpec\": {\n", + " \"type\": \"uniform\",\n", + " \"segmentGranularity\": \"DAY\",\n", + " \"queryGranularity\": {\n", + " \"type\": \"none\"\n", + " },\n", + " \"rollup\": False,\n", + " \"intervals\": []\n", + " },\n", + " \"transformSpec\": {\n", + " \"filter\": None,\n", + " \"transforms\": []\n", + " }\n", + " },\n", + " \"ioConfig\": {\n", + " \"topic\": \"raw_scouter-opcua-orchestrated-pipeline\",\n", + " \"topicPattern\": None,\n", + " \"inputFormat\": {\n", + " \"type\": \"kafka\",\n", + " \"headerFormat\": None,\n", + " \"keyFormat\": None,\n", + " \"valueFormat\": {\n", + " \"type\": \"json\",\n", + " \"keepNoneColumns\": False,\n", + " \"assumeNewlineDelimited\": False,\n", + " \"useJsonNodeReader\": False\n", + " },\n", + " \"headerColumnPrefix\": \"kafka.header.\",\n", + " \"keyColumnName\": \"kafka.key\",\n", + " \"timestampColumnName\": \"kafka.timestamp\",\n", + " \"topicColumnName\": \"kafka.topic\"\n", + " },\n", + " \"replicas\": 1,\n", + " \"taskCount\": 1,\n", + " \"taskDuration\": \"PT3600S\",\n", + " \"consumerProperties\": {\n", + " \"bootstrap.servers\": \"kafka.kafka.svc.cluster.local:9092\"\n", + " },\n", + " \"autoScalerConfig\": None,\n", + " \"pollTimeout\": 100,\n", + " \"startDelay\": \"PT5S\",\n", + " \"period\": \"PT30S\",\n", + " \"useEarliestOffset\": True,\n", + " \"completionTimeout\": \"PT1800S\",\n", + " \"lateMessageRejectionPeriod\": None,\n", + " \"earlyMessageRejectionPeriod\": None,\n", + " \"lateMessageRejectionStartDateTime\": None,\n", + " \"configOverrides\": None,\n", + " \"idleConfig\": None,\n", + " \"stopTaskCount\": None,\n", + " \"stream\": \"raw_scouter-opcua-orchestrated-pipeline\",\n", + " \"useEarliestSequenceNumber\": True\n", + " },\n", + " \"tuningConfig\": {\n", + " \"type\": \"kafka\",\n", + " \"appendableIndexSpec\": {\n", + " \"type\": \"onheap\",\n", + " \"preserveExistingMetrics\": False\n", + " },\n", + " \"maxRowsInMemory\": 150000,\n", + " \"maxBytesInMemory\": 0,\n", + " \"skipBytesInMemoryOverheadCheck\": False,\n", + " \"maxRowsPerSegment\": 5000000,\n", + " \"maxTotalRows\": None,\n", + " \"intermediatePersistPeriod\": \"PT10M\",\n", + " \"maxPendingPersists\": 0,\n", + " \"indexSpec\": {\n", + " \"bitmap\": {\n", + " \"type\": \"roaring\"\n", + " },\n", + " \"dimensionCompression\": \"lz4\",\n", + " \"stringDictionaryEncoding\": {\n", + " \"type\": \"utf8\"\n", + " },\n", + " \"metricCompression\": \"lz4\",\n", + " \"longEncoding\": \"longs\"\n", + " },\n", + " \"indexSpecForIntermediatePersists\": {\n", + " \"bitmap\": {\n", + " \"type\": \"roaring\"\n", + " },\n", + " \"dimensionCompression\": \"lz4\",\n", + " \"stringDictionaryEncoding\": {\n", + " \"type\": \"utf8\"\n", + " },\n", + " \"metricCompression\": \"lz4\",\n", + " \"longEncoding\": \"longs\"\n", + " },\n", + " \"reportParseExceptions\": False,\n", + " \"handoffConditionTimeout\": 900000,\n", + " \"resetOffsetAutomatically\": False,\n", + " \"segmentWriteOutMediumFactory\": None,\n", + " \"workerThreads\": None,\n", + " \"chatRetries\": 8,\n", + " \"httpTimeout\": \"PT10S\",\n", + " \"shutdownTimeout\": \"PT80S\",\n", + " \"offsetFetchPeriod\": \"PT30S\",\n", + " \"intermediateHandoffPeriod\": \"P2147483647D\",\n", + " \"logParseExceptions\": False,\n", + " \"maxParseExceptions\": 2147483647,\n", + " \"maxSavedParseExceptions\": 0,\n", + " \"numPersistThreads\": 1,\n", + " \"skipSequenceNumberAvailabilityCheck\": False,\n", + " \"repartitionTransitionDuration\": \"PT120S\"\n", + " }\n", + " },\n", + " \"context\": None,\n", + " \"suspended\": False\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "raw_scouter-opcua-pipeline-2\n", + "raw_scouter-opcua-pipeline-3\n", + "raw_scouter-opcua-pipeline-4\n", + "raw_scouter-opcua-pipeline-5\n", + "raw_scouter-opcua-pipeline-6\n", + "raw_scouter-opcua-pipeline-7\n", + "raw_scouter-opcua-pipeline-8\n", + "raw_scouter-opcua-pipeline-9\n", + "raw_scouter-opcua-pipeline-10\n", + "raw_scouter-opcua-pipeline-11\n", + "raw_scouter-opcua-pipeline-12\n", + "raw_scouter-opcua-pipeline-13\n", + "raw_scouter-opcua-pipeline-14\n", + "raw_scouter-opcua-pipeline-15\n", + "raw_scouter-opcua-pipeline-16\n", + "raw_scouter-opcua-pipeline-17\n", + "raw_scouter-opcua-pipeline-18\n", + "raw_scouter-opcua-pipeline-19\n", + "raw_scouter-opcua-pipeline-20\n", + "raw_scouter-opcua-pipeline-21\n", + "raw_scouter-opcua-pipeline-22\n", + "raw_scouter-opcua-pipeline-23\n", + "raw_scouter-opcua-pipeline-24\n", + "raw_scouter-opcua-pipeline-25\n", + "raw_scouter-opcua-pipeline-26\n", + "raw_scouter-opcua-pipeline-27\n", + "raw_scouter-opcua-pipeline-28\n", + "raw_scouter-opcua-pipeline-29\n", + "raw_scouter-opcua-pipeline-30\n" + ] + } + ], + "source": [ + "import json\n", + "from copy import deepcopy\n", + "\n", + "json_list = []\n", + "i = 0\n", + "for i in range(1, 30):\n", + " topic = f\"raw_scouter-opcua-pipeline-{i+1}\"\n", + " print(topic)\n", + " obj = deepcopy(spec)\n", + " obj['spec']['dataSchema']['dataSource'] = topic\n", + " obj['spec']['ioConfig']['topic'] = topic\n", + " obj['spec']['ioConfig']['stream'] = topic\n", + "\n", + " json_list.append(obj)\n", + "\n", + "# Save to a new file\n", + "with open('specs_30.json', 'w') as f:\n", + " json.dump(json_list, f, indent=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[OK] raw_scouter-opcua-pipeline-2 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-3 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-4 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-5 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-6 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-7 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-8 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-9 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-10 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-11 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-12 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-13 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-14 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-15 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-16 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-17 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-18 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-19 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-20 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-21 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-22 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-23 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-24 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-25 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-26 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-27 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-28 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-29 enviado.\n", + "[OK] raw_scouter-opcua-pipeline-30 enviado.\n" + ] + } + ], + "source": [ + "import os\n", + "import requests\n", + "\n", + "DRUID_OVERLORD = os.getenv(\"DRUID_OVERLORD\", \"http://localhost:8082\")\n", + "SUPERVISOR_ENDPOINT = f\"{DRUID_OVERLORD}/druid/indexer/v1/supervisor\"\n", + "\n", + "def enviar_supervisores(specs):\n", + " for spec in specs:\n", + " resp = requests.post(\n", + " SUPERVISOR_ENDPOINT,\n", + " headers={\"Content-Type\": \"application/json\"},\n", + " json=spec\n", + " )\n", + " if resp.status_code == 200:\n", + " print(f\"[OK] {spec['spec']['dataSchema']['dataSource']} enviado.\")\n", + " else:\n", + " print(f\"[ERRO] {spec['spec']['dataSchema']['dataSource']}: {resp.status_code} → {resp.text}\")\n", + "\n", + "with open(\"./specs_30.json\", \"r\") as f:\n", + " specs = json.load(f)\n", + "enviar_supervisores(specs)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "ename": "ConnectionError", + "evalue": "HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /druid/indexer/v1/supervisor/raw_scouter-opcua-orchestrated-pipeline-2/terminate (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mConnectionRefusedError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:198\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 197\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m198\u001b[39m sock = \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 199\u001b[39m \u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 200\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 201\u001b[39m \u001b[43m \u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 202\u001b[39m \u001b[43m \u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 203\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 204\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.gaierror \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 84\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m85\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[32m 86\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 87\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/util/connection.py:73\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 72\u001b[39m sock.bind(source_address)\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[43msock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 74\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", + "\u001b[31mConnectionRefusedError\u001b[39m: [Errno 111] Connection refused", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[31mNewConnectionError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connectionpool.py:787\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 786\u001b[39m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m787\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 788\u001b[39m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 789\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 790\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 791\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 792\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 793\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 794\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 795\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 796\u001b[39m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 797\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 798\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 799\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 800\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 802\u001b[39m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connectionpool.py:493\u001b[39m, in \u001b[36mHTTPConnectionPool._make_request\u001b[39m\u001b[34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 492\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m493\u001b[39m \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 494\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 495\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 497\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 498\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 499\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 500\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 501\u001b[39m \u001b[43m \u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m=\u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 502\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 504\u001b[39m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[32m 505\u001b[39m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[32m 506\u001b[39m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:494\u001b[39m, in \u001b[36mHTTPConnection.request\u001b[39m\u001b[34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 493\u001b[39m \u001b[38;5;28mself\u001b[39m.putheader(header, value)\n\u001b[32m--> \u001b[39m\u001b[32m494\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[38;5;66;03m# If we're given a body we start sending that in chunks.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.11/http/client.py:1298\u001b[39m, in \u001b[36mHTTPConnection.endheaders\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1297\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[32m-> \u001b[39m\u001b[32m1298\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.11/http/client.py:1058\u001b[39m, in \u001b[36mHTTPConnection._send_output\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1057\u001b[39m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m._buffer[:]\n\u001b[32m-> \u001b[39m\u001b[32m1058\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1060\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1061\u001b[39m \n\u001b[32m 1062\u001b[39m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.11/http/client.py:996\u001b[39m, in \u001b[36mHTTPConnection.send\u001b[39m\u001b[34m(self, data)\u001b[39m\n\u001b[32m 995\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.auto_open:\n\u001b[32m--> \u001b[39m\u001b[32m996\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 997\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:325\u001b[39m, in \u001b[36mHTTPConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 324\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m325\u001b[39m \u001b[38;5;28mself\u001b[39m.sock = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 326\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._tunnel_host:\n\u001b[32m 327\u001b[39m \u001b[38;5;66;03m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:213\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 212\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m--> \u001b[39m\u001b[32m213\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[32m 214\u001b[39m \u001b[38;5;28mself\u001b[39m, \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 215\u001b[39m ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m 217\u001b[39m sys.audit(\u001b[33m\"\u001b[39m\u001b[33mhttp.client.connect\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mself\u001b[39m.host, \u001b[38;5;28mself\u001b[39m.port)\n", + "\u001b[31mNewConnectionError\u001b[39m: : Failed to establish a new connection: [Errno 111] Connection refused", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[31mMaxRetryError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/adapters.py:667\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 666\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m667\u001b[39m resp = \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 668\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 669\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 670\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 671\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 672\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 673\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 674\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 675\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 676\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 677\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 678\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 679\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 681\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connectionpool.py:841\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 839\u001b[39m new_e = ProtocolError(\u001b[33m\"\u001b[39m\u001b[33mConnection aborted.\u001b[39m\u001b[33m\"\u001b[39m, new_e)\n\u001b[32m--> \u001b[39m\u001b[32m841\u001b[39m retries = \u001b[43mretries\u001b[49m\u001b[43m.\u001b[49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 842\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m=\u001b[49m\u001b[43msys\u001b[49m\u001b[43m.\u001b[49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[32m 843\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 844\u001b[39m retries.sleep()\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/util/retry.py:519\u001b[39m, in \u001b[36mRetry.increment\u001b[39m\u001b[34m(self, method, url, response, error, _pool, _stacktrace)\u001b[39m\n\u001b[32m 518\u001b[39m reason = error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[32m--> \u001b[39m\u001b[32m519\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mreason\u001b[39;00m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[32m 521\u001b[39m log.debug(\u001b[33m\"\u001b[39m\u001b[33mIncremented Retry for (url=\u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m\"\u001b[39m, url, new_retry)\n", + "\u001b[31mMaxRetryError\u001b[39m: HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /druid/indexer/v1/supervisor/raw_scouter-opcua-orchestrated-pipeline-2/terminate (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[31mConnectionError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[20]\u001b[39m\u001b[32m, line 16\u001b[39m\n\u001b[32m 13\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m❌ Erro \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresp.status_code\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresp.text\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m 15\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[32m1\u001b[39m, \u001b[32m30\u001b[39m):\n\u001b[32m---> \u001b[39m\u001b[32m16\u001b[39m \u001b[43mterminate_supervisor\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43mf\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mraw_scouter-opcua-orchestrated-pipeline-\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mi\u001b[49m\u001b[43m+\u001b[49m\u001b[32;43m1\u001b[39;49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[20]\u001b[39m\u001b[32m, line 7\u001b[39m, in \u001b[36mterminate_supervisor\u001b[39m\u001b[34m(supervisor_id)\u001b[39m\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mterminate_supervisor\u001b[39m(supervisor_id):\n\u001b[32m 6\u001b[39m url = \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mDRUID\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/druid/indexer/v1/supervisor/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msupervisor_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/terminate\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m7\u001b[39m resp = \u001b[43mrequests\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 8\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m resp.status_code == \u001b[32m200\u001b[39m:\n\u001b[32m 9\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m✅ Supervisor \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msupervisor_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m encerrado.\u001b[39m\u001b[33m\"\u001b[39m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/api.py:115\u001b[39m, in \u001b[36mpost\u001b[39m\u001b[34m(url, data, json, **kwargs)\u001b[39m\n\u001b[32m 103\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpost\u001b[39m(url, data=\u001b[38;5;28;01mNone\u001b[39;00m, json=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m 104\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33mr\u001b[39m\u001b[33;03m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[32m 105\u001b[39m \n\u001b[32m 106\u001b[39m \u001b[33;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 112\u001b[39m \u001b[33;03m :rtype: requests.Response\u001b[39;00m\n\u001b[32m 113\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m115\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpost\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[43m=\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/api.py:59\u001b[39m, in \u001b[36mrequest\u001b[39m\u001b[34m(method, url, **kwargs)\u001b[39m\n\u001b[32m 55\u001b[39m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[32m 56\u001b[39m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[32m 57\u001b[39m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[32m 58\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m sessions.Session() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[32m---> \u001b[39m\u001b[32m59\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/sessions.py:589\u001b[39m, in \u001b[36mSession.request\u001b[39m\u001b[34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[39m\n\u001b[32m 584\u001b[39m send_kwargs = {\n\u001b[32m 585\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mtimeout\u001b[39m\u001b[33m\"\u001b[39m: timeout,\n\u001b[32m 586\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mallow_redirects\u001b[39m\u001b[33m\"\u001b[39m: allow_redirects,\n\u001b[32m 587\u001b[39m }\n\u001b[32m 588\u001b[39m send_kwargs.update(settings)\n\u001b[32m--> \u001b[39m\u001b[32m589\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 591\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/sessions.py:703\u001b[39m, in \u001b[36mSession.send\u001b[39m\u001b[34m(self, request, **kwargs)\u001b[39m\n\u001b[32m 700\u001b[39m start = preferred_clock()\n\u001b[32m 702\u001b[39m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m703\u001b[39m r = \u001b[43madapter\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 705\u001b[39m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[32m 706\u001b[39m elapsed = preferred_clock() - start\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/adapters.py:700\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 696\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e.reason, _SSLError):\n\u001b[32m 697\u001b[39m \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[32m 698\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request=request)\n\u001b[32m--> \u001b[39m\u001b[32m700\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n\u001b[32m 702\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 703\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n", + "\u001b[31mConnectionError\u001b[39m: HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /druid/indexer/v1/supervisor/raw_scouter-opcua-orchestrated-pipeline-2/terminate (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))" + ] + } + ], + "source": [ + "import os\n", + "import requests\n", + "\n", + "DRUID = os.getenv(\"DRUID_URL\", \"http://localhost:8082\")\n", + "def terminate_supervisor(supervisor_id):\n", + " url = f\"{DRUID}/druid/indexer/v1/supervisor/{supervisor_id}/terminate\"\n", + " resp = requests.post(url)\n", + " if resp.status_code == 200:\n", + " print(f\"✅ Supervisor '{supervisor_id}' encerrado.\")\n", + " elif resp.status_code == 404:\n", + " print(f\"⚠️ Supervisor '{supervisor_id}' não encontrado.\")\n", + " else:\n", + " print(f\"❌ Erro {resp.status_code}: {resp.text}\")\n", + "\n", + "for i in range(1, 30):\n", + " terminate_supervisor(f\"raw_scouter-opcua-orchestrated-pipeline-{i+1}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Document deleted successfully\n" + ] + } + ], + "source": [ + "from pymongo import MongoClient\n", + "import os\n", + "# Get MongoDB connection details from environment variables\n", + "MONGODB_USERNAME = os.getenv(\"MONGODB_USERNAME\", \"root\")\n", + "MONGODB_PASSWORD = os.getenv(\"MONGODB_PASSWORD\", \"wKZDbMNU1c\") \n", + "MONGODB_URL = os.getenv(\"MONGODB_URL\", \"localhost:27018\")\n", + "MONGODB_DATABASE = os.getenv(\"MONGODB_DATABASE\", \"sientia\")\n", + "\n", + "# Create MongoDB client\n", + "client = MongoClient(\n", + " f\"mongodb://{MONGODB_USERNAME}:{MONGODB_PASSWORD}@{MONGODB_URL}\"\n", + ")\n", + "\n", + "# Get database and collection\n", + "db = client[MONGODB_DATABASE]\n", + "collection = db[\"pipelines\"] # Replace with actual collection name\n", + "\n", + "for i in range(2, 31):\n", + " # Delete a document matching specific criteria\n", + " result = collection.delete_one({\"schedule_name\": f\"scouter-opcua-pipeline-{i}\"}) # Replace with actual query\n", + "\n", + "if result.deleted_count > 0:\n", + " print(\"✅ Document deleted successfully\")\n", + "else:\n", + " print(\"⚠️ No matching document found\")\n", + "\n", + "# Close the connection\n", + "client.close()\n" + ] } ], "metadata": { diff --git a/values.yaml b/values.yaml index 6686b55..3bf6fa4 100644 --- a/values.yaml +++ b/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 1 +replicaCount: 3 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: @@ -141,7 +141,7 @@ env: - name: POSTGRES_MIN_CONNECTIONS value: "10" - name: POSTGRES_MAX_CONNECTIONS - value: "20" + value: "40" - name: KAFKA_BOOTSTRAP_SERVERS value: "kafka.kafka.svc.cluster.local:9092" From 1dcba6b48e710a5774f5755032de7bf6725a1014 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 3 Jul 2025 14:04:10 -0300 Subject: [PATCH 34/36] SIENTIAPDE-1110 Refactor Scouter workflow to update data loading method, changing from load_latest_druid_data to load_latest_data. Adjusted parameters for consistency in collection naming and improved clarity in data retrieval process. --- scouter/workflow/scouter.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/scouter/workflow/scouter.py b/scouter/workflow/scouter.py index e9ca1f2..eedebc8 100644 --- a/scouter/workflow/scouter.py +++ b/scouter/workflow/scouter.py @@ -62,28 +62,28 @@ class Scouter: retry_policy=retry_policy ) - # data = await workflow.execute_local_activity_method( - # Activities.load_latest_data, - # { - # **metadata, - # 'collection_name': f"raw_{input_data['schedule_name']}", - # 'last_data_timestamp': last_data_timestamp - # }, - # start_to_close_timeout=timedelta(seconds=60), - # retry_policy=retry_policy - # ) - data = await workflow.execute_local_activity_method( - Activities.load_latest_druid_data, + Activities.load_latest_data, { **metadata, - 'schedule_name': input_data['schedule_name'], + 'collection_name': f"raw_{input_data['schedule_name']}", 'last_data_timestamp': last_data_timestamp }, start_to_close_timeout=timedelta(seconds=60), retry_policy=retry_policy ) + # data = await workflow.execute_local_activity_method( + # Activities.load_latest_druid_data, + # { + # **metadata, + # 'schedule_name': input_data['schedule_name'], + # 'last_data_timestamp': last_data_timestamp + # }, + # start_to_close_timeout=timedelta(seconds=60), + # retry_policy=retry_policy + # ) + await workflow.execute_activity_method( Activities.put_last_data_timestamp, { From d08d1b133722c3fa58f9f7a4a80f4245ea08996f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 3 Jul 2025 16:25:55 -0300 Subject: [PATCH 35/36] SIENTIAPDE-1110 Refactor Activities class to remove Kafka and Druid dependencies, simplifying initialization. Update values.yaml to set replica count to 1 for reduced resource usage. Adjust Redis activity to set TTL to None for better data retention. Remove unused Kafka and Druid activity files and their associated tests, streamlining the codebase. --- scouter/activities/activities.py | 30 +---- scouter/activities/kafka.py | 105 ------------------ scouter/activities/pydruid.py | 71 ------------ scouter/activities/redis.py | 2 +- scouter/worker/worker.py | 10 +- scouter/workflow/scouter.py | 25 +---- tests/activities/test_activities.py | 43 ++++--- tests/activities/test_gates.py | 4 +- tests/activities/test_kafka.py | 92 --------------- tests/activities/test_redis.py | 4 +- .../sub_workflows/test_core_scouter.py | 16 +++ tests/workflow/test_scouter.py | 70 +++++++++--- values.yaml | 2 +- 13 files changed, 105 insertions(+), 369 deletions(-) delete mode 100644 scouter/activities/kafka.py delete mode 100644 scouter/activities/pydruid.py delete mode 100644 tests/activities/test_kafka.py diff --git a/scouter/activities/activities.py b/scouter/activities/activities.py index 53f7f0f..59990b4 100644 --- a/scouter/activities/activities.py +++ b/scouter/activities/activities.py @@ -1,26 +1,22 @@ -from temporalio import activity, workflow +from temporalio import workflow with workflow.unsafe.imports_passed_through(): from sientia_do.temporal.activities.postgres import Postgres from sientia_do.notifications.handlers import NotificationHandler from sientia_do.temporal.utils.logger import Logger from scouter.activities.redis import Redis - from scouter.activities.kafka import Kafka from scouter.activities.gates import Gates from scouter.activities.mongodb import MongoDB - from scouter.activities.pydruid import Druid from typing import Any -class Activities(Postgres, Redis, Kafka, Gates, MongoDB, Druid): +class Activities(Postgres, Redis, Gates, MongoDB,): """Activities class that combines multiple services with proper initialization.""" def __init__(self, postgres_config: dict[str, Any], redis_config: dict[str, Any], - kafka_config: dict[str, Any], mongodb_config: dict[str, Any], - druid_config: dict[str, Any], logger: Logger, notification_handler: NotificationHandler): @@ -49,16 +45,6 @@ class Activities(Postgres, Redis, Kafka, Gates, MongoDB, Druid): password=redis_config['password'] ) - # Initialize Kafka - Kafka.__init__( - self, - bootstrap_servers=kafka_config['bootstrap_servers'], - polling_time=kafka_config['polling_time'], - group_id=kafka_config['group_id'], - logger=logger, - notification_handler=notification_handler - ) - # Initialize Gates Gates.__init__( self, @@ -75,16 +61,6 @@ class Activities(Postgres, Redis, Kafka, Gates, MongoDB, Druid): notification_handler=notification_handler ) - # Initialize Druid - Druid.__init__( - self, - host=druid_config['host'], - port=druid_config['port'], - logger=logger, - notification_handler=notification_handler - ) - def shutdown(self): Postgres.close(self) - Kafka.close(self) - MongoDB.close(self) + MongoDB.shutdown(self) diff --git a/scouter/activities/kafka.py b/scouter/activities/kafka.py deleted file mode 100644 index ea36655..0000000 --- a/scouter/activities/kafka.py +++ /dev/null @@ -1,105 +0,0 @@ -from temporalio import workflow, activity - -with workflow.unsafe.imports_passed_through(): - from logging import Logger - from sientia_do.notifications.handlers import NotificationHandler - from sientia_do.temporal.activities.base import BaseActivity - from sientia_do.temporal.utils.logger import Logger - from typing import Any - from aiokafka import AIOKafkaConsumer - from pandas import DataFrame - import json - import asyncio - - -class Kafka(BaseActivity): - def __init__(self, bootstrap_servers: str, polling_time: int, - group_id: str, logger: Logger, notification_handler: NotificationHandler): - self.polling_time = polling_time - self.bootstrap_servers = bootstrap_servers - self.group_id = group_id - self.consumers = {} - self._consumer_tasks = {} - BaseActivity.__init__(self, logger, notification_handler) - - async def close(self): - """Closes all consumer connections.""" - self.info("Closing Kafka connectors...") - for _topic, consumer in self.consumers.items(): - await consumer.stop() - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc, tb): - await self.close() - - async def create_consumer(self, topic: str): - consumer = AIOKafkaConsumer( - bootstrap_servers=self.bootstrap_servers, - auto_offset_reset="earliest", - enable_auto_commit=True, - group_id=f"{self.group_id}-{topic}", - value_deserializer=lambda x: json.loads(x.decode("utf-8")) - ) - await consumer.start() - self.consumers[topic] = consumer - - @activity.defn(name="load_from_kafka") - async def load_from_kafka(self, input_data: dict[str, Any]) -> dict[str, Any]: - """ - Loads data from a kafka topic. Polls the topic for a given time and returns the data. - - Args: - input_data (dict[str, Any]): The data to load. Contains: - topic (str): The topic to load data from. - Returns: - dict[str, Any]: The data loaded from the topic. - """ - - metadata = input_data['metadata'] - - self.debug( - f"Loading data from topic: {input_data['topic']}", - metadata=metadata - ) - - topic = input_data["topic"] - - if topic not in self.consumers: - self.info( - f"Creating consumer for topic: {topic}", - metadata=metadata - ) - await self.create_consumer(topic) - - consumer = self.consumers[topic] - - consumer.subscribe(topics=[topic]) - - message_values = [] - - messages = await consumer.getmany(timeout_ms=self.polling_time) - - for tp, msgs in messages.items(): - msg_topic = tp.topic - if msg_topic == topic: - for msg in msgs: - message_values.append(msg.value) - - self.info( - f"Loaded {len(message_values)} messages from topic: {topic}", - metadata=metadata - ) - - self.debug( - f"Loaded data: {message_values}", - metadata=metadata - ) - - consumer.unsubscribe() - - if not message_values: - return {} - - return DataFrame(message_values).to_dict() diff --git a/scouter/activities/pydruid.py b/scouter/activities/pydruid.py deleted file mode 100644 index f19e4bb..0000000 --- a/scouter/activities/pydruid.py +++ /dev/null @@ -1,71 +0,0 @@ -from temporalio import workflow, activity - -with workflow.unsafe.imports_passed_through(): - import pandas as pd - from typing import List, Optional, Any - from datetime import datetime, timedelta - from pydruid.client import PyDruid - from pydruid.query import QueryBuilder - from sqlalchemy.engine import create_engine - from sqlalchemy import MetaData, Table, select, text - from sientia_do.temporal.activities.base import BaseActivity - from sientia_do.notifications.handlers import NotificationHandler - from sientia_do.temporal.utils.logger import Logger - - -class Druid(BaseActivity): - def __init__(self, host: str, port: int, - logger: Logger, notification_handler: NotificationHandler): - - self.host = host - self.port = port - self.druid_engine = create_engine( - f'druid://{self.host}:{self.port}/druid/v2/sql/') - logger.info( - f"Druid client initialized with host: {self.host}, port: {self.port}") - - BaseActivity.__init__(self, logger=logger, - notification_handler=notification_handler) - - def shutdown(self): - self.client.close() - - def __del__(self): - self.shutdown() - - @activity.defn(name="load_latest_druid_data") - async def load_latest_druid_data(self, input_data: dict[str, Any]) -> dict[str, Any]: - """ - Loads the latest data from Druid. - """ - metadata = input_data['metadata'] - datasource = f"raw_{input_data['schedule_name']}" - last_data_timestamp = input_data['last_data_timestamp'] - last_data_timestamp = last_data_timestamp if last_data_timestamp is not None else '1970-01-01 00:00:00' - self.debug( - f"Loading data from Druid: {input_data}", metadata=metadata) - - query = f'"__time" > TIMESTAMP \'{last_data_timestamp}\'' - - self.info( - f"Loading data from Druid: {datasource} with query: {query}" - ) - - places = Table(datasource, MetaData(), autoload_with=self.druid_engine) - stmt = select(places).where(text(query)) - - result = pd.read_sql(stmt, self.druid_engine) - - result["inserted_at"] = pd.to_datetime(result["__time"]).dt.strftime( - "%Y-%m-%d %H:%M:%S.%f") - - result.drop(columns=["__time"], inplace=True) - - self.info( - f"Loaded {len(result)} rows from Druid" - ) - - self.debug( - f"Druid query result: {result}", metadata=metadata) - - return result.to_dict() diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index b4ad730..d9f54e4 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -55,7 +55,7 @@ class Redis(RedisBase): metadata=metadata ) - self.set(key, last_data_timestamp) + self.set(key, last_data_timestamp, ttl=None) return last_data_timestamp diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index b99c05c..d0e4fdd 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -14,10 +14,8 @@ with workflow.unsafe.imports_passed_through(): import asyncio from scouter.utils.connectors_config import ( build_postgres_config, - build_kafka_config, build_redis_config, - build_mongodb_config, - build_druid_config + build_mongodb_config ) @@ -42,10 +40,8 @@ async def main(): logger=logger, notification_handler=notification_handler, postgres_config=build_postgres_config(), - kafka_config=build_kafka_config(), redis_config=build_redis_config(), - mongodb_config=build_mongodb_config(), - druid_config=build_druid_config() + mongodb_config=build_mongodb_config() ) logger.info('Starting Faker Activities...') @@ -73,10 +69,8 @@ async def main(): workflows=[Scouter, CoreScouter], activities=[ activities.load_latest_data, - activities.load_latest_druid_data, activities.get_last_data_timestamp, activities.put_last_data_timestamp, - activities.load_from_kafka, activities.data_quality_gate, activities.aggregate_data, activities.group_and_hold_data, diff --git a/scouter/workflow/scouter.py b/scouter/workflow/scouter.py index eedebc8..ffcb625 100644 --- a/scouter/workflow/scouter.py +++ b/scouter/workflow/scouter.py @@ -41,16 +41,6 @@ class Scouter: } } - # data = await workflow.execute_activity_method( - # Activities.load_from_kafka, - # { - # **metadata, - # 'topic': input_data['topic'] - # }, - # retry_policy=retry_policy, - # start_to_close_timeout=timedelta(seconds=60) - # ) - last_data_timestamp = await workflow.execute_local_activity_method( Activities.get_last_data_timestamp, { @@ -73,16 +63,8 @@ class Scouter: retry_policy=retry_policy ) - # data = await workflow.execute_local_activity_method( - # Activities.load_latest_druid_data, - # { - # **metadata, - # 'schedule_name': input_data['schedule_name'], - # 'last_data_timestamp': last_data_timestamp - # }, - # start_to_close_timeout=timedelta(seconds=60), - # retry_policy=retry_policy - # ) + if data == {}: + return await workflow.execute_activity_method( Activities.put_last_data_timestamp, @@ -96,9 +78,6 @@ class Scouter: retry_policy=retry_policy ) - if data == {}: - return - input_data['data'] = data input_data['metadata'] = metadata diff --git a/tests/activities/test_activities.py b/tests/activities/test_activities.py index 2150dca..2740d18 100644 --- a/tests/activities/test_activities.py +++ b/tests/activities/test_activities.py @@ -2,16 +2,16 @@ from unittest.mock import patch, MagicMock, ANY from pytest import mark from sientia_do.temporal.activities.postgres import Postgres from scouter.activities.activities import Activities +from scouter.activities.mongodb import MongoDB from scouter.activities.redis import Redis -from scouter.activities.kafka import Kafka from scouter.activities.gates import Gates +@patch('scouter.activities.activities.MongoDB.__init__') @patch('scouter.activities.activities.Postgres.__init__') @patch('scouter.activities.activities.Redis.__init__') -@patch('scouter.activities.activities.Kafka.__init__') @patch('scouter.activities.activities.Gates.__init__') -def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgres_init): +def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mongodb_init): postgres_config = { 'host': 'localhost', @@ -30,10 +30,9 @@ def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgr 'password': 'redis' } - kafka_config = { - 'bootstrap_servers': 'localhost:9092', - 'polling_time': 1000, - 'group_id': 'test-group' + mongodb_config = { + 'connection_string': 'mongodb://localhost:27017', + 'database_name': 'test_database' } logger = MagicMock() @@ -42,7 +41,7 @@ def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgr activities = Activities( postgres_config=postgres_config, redis_config=redis_config, - kafka_config=kafka_config, + mongodb_config=mongodb_config, logger=logger, notification_handler=notification_handler ) @@ -50,7 +49,7 @@ def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgr assert isinstance(activities, Activities) assert isinstance(activities, Postgres) assert isinstance(activities, Redis) - assert isinstance(activities, Kafka) + assert isinstance(activities, MongoDB) assert isinstance(activities, Gates) mock_postgres_init.assert_called_once_with( @@ -76,11 +75,10 @@ def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgr notification_handler=notification_handler ) - mock_kafka_init.assert_called_once_with( + mock_mongodb_init.assert_called_once_with( ANY, - bootstrap_servers=kafka_config['bootstrap_servers'], - polling_time=kafka_config['polling_time'], - group_id=kafka_config['group_id'], + connection_string=mongodb_config['connection_string'], + database_name=mongodb_config['database_name'], logger=logger, notification_handler=notification_handler ) @@ -94,12 +92,12 @@ def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgr @patch('scouter.activities.activities.Postgres.__init__') @patch('scouter.activities.activities.Redis.__init__') -@patch('scouter.activities.activities.Kafka.__init__') @patch('scouter.activities.activities.Gates.__init__') +@patch('scouter.activities.activities.MongoDB.__init__') @patch('scouter.activities.activities.Postgres.close') -@patch('scouter.activities.activities.Kafka.close') -def test_shutdown(mock_kafka_close, mock_postgres_close, - _mock_gates_init, _mock_redis_init, _mock_kafka_init, _mock_postgres_init): +@patch('scouter.activities.activities.MongoDB.shutdown') +def test_shutdown(mock_mongodb_close, mock_postgres_close, _mock_mongodb_init, + _mock_gates_init, _mock_redis_init, _mock_postgres_init): postgres_config = { 'host': 'localhost', 'port': 5432, @@ -117,10 +115,9 @@ def test_shutdown(mock_kafka_close, mock_postgres_close, 'password': 'redis' } - kafka_config = { - 'bootstrap_servers': 'localhost:9092', - 'polling_time': 1000, - 'group_id': 'test-group' + mongodb_config = { + 'connection_string': 'mongodb://localhost:27017', + 'database_name': 'test_database' } logger = MagicMock() @@ -129,7 +126,7 @@ def test_shutdown(mock_kafka_close, mock_postgres_close, activities = Activities( postgres_config=postgres_config, redis_config=redis_config, - kafka_config=kafka_config, + mongodb_config=mongodb_config, logger=logger, notification_handler=notification_handler ) @@ -137,4 +134,4 @@ def test_shutdown(mock_kafka_close, mock_postgres_close, activities.shutdown() mock_postgres_close.assert_called_once() - mock_kafka_close.assert_called_once() + mock_mongodb_close.assert_called_once() diff --git a/tests/activities/test_gates.py b/tests/activities/test_gates.py index 245b176..ba87f5a 100644 --- a/tests/activities/test_gates.py +++ b/tests/activities/test_gates.py @@ -290,8 +290,8 @@ async def test_aggregate_data(gates_fixture): 'value': None, 'timestamp': '2023-01-04'}, ], 'model_tags': { - 'name1': {'aggr_function': 'avg'}, - 'name2': {'aggr_function': 'max'}, + 'name1': {'aggr_func': 'avg'}, + 'name2': {'aggr_func': 'max'}, }, **metadata } diff --git a/tests/activities/test_kafka.py b/tests/activities/test_kafka.py deleted file mode 100644 index 681c9dd..0000000 --- a/tests/activities/test_kafka.py +++ /dev/null @@ -1,92 +0,0 @@ -from unittest.mock import MagicMock, patch, ANY -from pytest import fixture, mark -from pandas import DataFrame -from scouter.activities.kafka import Kafka - - -@fixture -@patch("scouter.activities.kafka.KafkaConsumer") -def kafka(_kafka_consumer): - return Kafka( - bootstrap_servers="localhost:9092", - polling_time=1000, - group_id="test-group", - logger=MagicMock(), - notification_handler=MagicMock() - ) - - -@patch("scouter.activities.kafka.KafkaConsumer") -def test___init__(kafka_consumer): - kafka = Kafka( - bootstrap_servers="localhost:9092", - polling_time=1000, - group_id="test-group", - logger=MagicMock(), - notification_handler=MagicMock() - ) - - assert kafka.polling_time == 1000 - assert kafka.kafka_connector == kafka_consumer.return_value - - kafka_consumer.assert_called_once_with( - bootstrap_servers="localhost:9092", - auto_offset_reset="earliest", - enable_auto_commit=True, - group_id="test-group", - value_deserializer=ANY - ) - - -metadata = { - 'metadata': { - 'model_id': 'test_model_id', - 'model_name': 'test_model', - 'schedule_name': 'test_schedule', - 'workflow_name': 'scouter' - } -} - - -@mark.asyncio -async def test_load_from_kafka(kafka): - input_data = {"topic": "test-topic", **metadata} - - data = [ - ("test-topic", [ - MagicMock( - value=f"test-value-{i}" - ) for i in range(10) - ]) - ] - - kafka.kafka_connector.poll.return_value = MagicMock( - items=MagicMock(return_value=data) - ) - - expected = DataFrame([d.value for d in data[0][1]]).to_dict() - - result = await kafka.load_from_kafka(input_data) - - assert result == expected - - kafka.kafka_connector.subscribe.assert_called_once_with(["test-topic"]) - - kafka.kafka_connector.poll.assert_called_once_with(timeout_ms=1000) - - -@mark.asyncio -async def test_load_from_kafka_empty(kafka): - input_data = {"topic": "test-topic", **metadata} - - kafka.kafka_connector.poll.return_value = MagicMock( - items=MagicMock(return_value=[]) - ) - - result = await kafka.load_from_kafka(input_data) - - assert result == {} - - kafka.kafka_connector.subscribe.assert_called_once_with(["test-topic"]) - - kafka.kafka_connector.poll.assert_called_once_with(timeout_ms=1000) diff --git a/tests/activities/test_redis.py b/tests/activities/test_redis.py index a266c71..f4d8977 100644 --- a/tests/activities/test_redis.py +++ b/tests/activities/test_redis.py @@ -87,7 +87,7 @@ async def test_group_and_hold_data_new_key(redis_activity): # Verify set was called with correct arguments redis_activity.set.assert_called_once() args, kwargs = redis_activity.set.call_args - assert args[0] == 'test_pipeline_test_schedule' + assert args[0] == 'held_data_test_pipeline_test_schedule' assert args[1] == { 'sensor1': 25.5, 'sensor2': 30.0, @@ -139,7 +139,7 @@ async def test_group_and_hold_data_update_existing(redis_activity): # Verify set was called with correct arguments redis_activity.set.assert_called_once() args, kwargs = redis_activity.set.call_args - assert args[0] == 'test_workflow_test_schedule' + assert args[0] == 'held_data_test_workflow_test_schedule' assert args[1] == { 'sensor1': 25.5, 'sensor2': 28.0, diff --git a/tests/workflow/sub_workflows/test_core_scouter.py b/tests/workflow/sub_workflows/test_core_scouter.py index 905b864..c92a175 100644 --- a/tests/workflow/sub_workflows/test_core_scouter.py +++ b/tests/workflow/sub_workflows/test_core_scouter.py @@ -16,6 +16,14 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter): 'filtered_data', 'grouped_data', 'held_data'] await core_scouter.run( input_data={ + 'metadata': { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow' + } + }, 'workflow_name': 'test_workflow', 'schedule_name': 'test_schedule', 'model_name': 'test_model', @@ -97,6 +105,14 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter mock_workflow.execute_local_activity_method.return_value = {} await core_scouter.run( input_data={ + 'metadata': { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow' + } + }, 'workflow_name': 'test_workflow', 'schedule_name': 'test_schedule', 'model_name': 'test_model', diff --git a/tests/workflow/test_scouter.py b/tests/workflow/test_scouter.py index 9727293..8d1b3a3 100644 --- a/tests/workflow/test_scouter.py +++ b/tests/workflow/test_scouter.py @@ -1,4 +1,4 @@ -from unittest.mock import AsyncMock, patch, ANY +from unittest.mock import AsyncMock, patch, ANY, call from pytest import fixture, mark from scouter.workflow.scouter import Scouter from scouter.activities.activities import Activities @@ -13,7 +13,10 @@ def scouter(): @patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock) async def test_scouter_workflow(mock_workflow, scouter): - mock_workflow.execute_activity_method.return_value = 'test_data' + mock_workflow.execute_local_activity_method.side_effect = [ + 'test_last_data_timestamp', + 'test_data' + ] await scouter.run( input_data={ 'topic': 'test_topic', @@ -32,11 +35,41 @@ async def test_scouter_workflow(mock_workflow, scouter): } } + mock_workflow.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.get_last_data_timestamp, + { + **expected_metadata, + 'workflow_name': 'scouter', + 'schedule_name': 'test_schedule' + }, + retry_policy=ANY, + start_to_close_timeout=ANY + ) + ]) + + mock_workflow.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.load_latest_data, + { + **expected_metadata, + 'collection_name': "raw_test_schedule", + 'last_data_timestamp': 'test_last_data_timestamp' + }, + retry_policy=ANY, + start_to_close_timeout=ANY + ) + ]) + mock_workflow.execute_activity_method.assert_called_once_with( - Activities.load_from_kafka, + Activities.put_last_data_timestamp, { **expected_metadata, - 'topic': 'test_topic' + 'data': 'test_data', + 'workflow_name': 'scouter', + 'schedule_name': 'test_schedule' }, retry_policy=ANY, start_to_close_timeout=ANY @@ -45,6 +78,7 @@ async def test_scouter_workflow(mock_workflow, scouter): mock_workflow.execute_child_workflow.assert_called_once_with( 'core_scouter', { + 'metadata': expected_metadata, 'topic': 'test_topic', 'data': 'test_data', 'workflow_name': 'scouter', @@ -58,7 +92,10 @@ async def test_scouter_workflow(mock_workflow, scouter): @mark.asyncio @patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock) async def test_scouter_workflow_empty(mock_workflow, scouter): - mock_workflow.execute_activity_method.return_value = {} + mock_workflow.execute_local_activity_method.side_effect = [ + 'test_last_data_timestamp', + {} + ] await scouter.run( input_data={ 'topic': 'test_topic', @@ -77,14 +114,19 @@ async def test_scouter_workflow_empty(mock_workflow, scouter): } } - mock_workflow.execute_activity_method.assert_called_once_with( - Activities.load_from_kafka, - { - **expected_metadata, - 'topic': 'test_topic' - }, - retry_policy=ANY, - start_to_close_timeout=ANY - ) + mock_workflow.execute_local_activity_method.assert_has_calls( + [ + call( + Activities.load_latest_data, + { + **expected_metadata, + 'collection_name': "raw_test_schedule", + 'last_data_timestamp': 'test_last_data_timestamp' + }, + retry_policy=ANY, + start_to_close_timeout=ANY + ) + ]) + mock_workflow.execute_activity_method.assert_not_called() mock_workflow.execute_child_workflow.assert_not_called() diff --git a/values.yaml b/values.yaml index 3bf6fa4..ded30e4 100644 --- a/values.yaml +++ b/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 3 +replicaCount: 1 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: From 08240efc8b37f8731ecf7aca7541f3119804c6fb Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 4 Jul 2025 11:01:20 -0300 Subject: [PATCH 36/36] SIENTIAPDE-1110 Enhance Gates and Redis activities by adding metadata parameter to apply_aggregation and notification methods. Refactor notification handling to use send_notification for improved consistency. Update tests to reflect changes in notification method calls and ensure proper functionality with new metadata integration. --- scouter/activities/gates.py | 18 ++- scouter/activities/redis.py | 81 +++++++++-- tests/activities/test_gates.py | 32 +++-- tests/activities/test_mongo.py | 192 ++++++++++++++++++++++++++ tests/activities/test_redis.py | 84 +++++++++++ tests/utils/test_connectors_config.py | 52 +++++++ 6 files changed, 425 insertions(+), 34 deletions(-) create mode 100644 tests/activities/test_mongo.py diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py index fbdb5e0..cef6bd0 100644 --- a/scouter/activities/gates.py +++ b/scouter/activities/gates.py @@ -16,7 +16,8 @@ quality_gate_filters = { class Gates(BaseActivity): - def apply_aggregation(self, group: DataFrame, aggr_function: str) -> float | None | str: + def apply_aggregation(self, group: DataFrame, aggr_function: str, + metadata: dict[str, Any]) -> float | None | str: """ Apply aggregation function to a group of data. @@ -48,7 +49,8 @@ class Gates(BaseActivity): elif aggr_function == 'min': return group['value'].min() else: - self.notification_handler.build_and_send_notification( + self.send_notification( + metadata=metadata, notification_id="AGGREGATION_ISSUES", message=f"Invalid aggregation function: {aggr_function}", block="aggregate_data", @@ -100,7 +102,8 @@ class Gates(BaseActivity): # Get the latest timestamp latest_timestamp = group['timestamp'].max() - aggr_value = self.apply_aggregation(group, aggr_function) + aggr_value = self.apply_aggregation( + group, aggr_function, metadata) if aggr_value == 'continue': continue @@ -145,7 +148,8 @@ class Gates(BaseActivity): except Exception as e: trace = traceback.format_exc() - self.notification_handler.build_and_send_notification( + self.send_notification( + metadata=metadata, notification_id="AGGREGATION_ISSUES", message=f"Error aggregating data: {e}", block="aggregate_data", @@ -202,7 +206,8 @@ class Gates(BaseActivity): except Exception as e: trace = traceback.format_exc() - self.notification_handler.build_and_send_notification( + self.send_notification( + metadata=metadata, notification_id="DATA_QUALITY_GATE_ISSUES", message=f"Error applying filter {filter_name}: {e}", block="data_quality_gate", @@ -219,7 +224,8 @@ class Gates(BaseActivity): message = f"{len(filtered_data)} rows has quality issues: {filter_name}: {policy}" attachment = filtered_data.to_string() - self.notification_handler.build_and_send_notification( + self.send_notification( + metadata=metadata, notification_id=f"DATA_QUALITY_GATE_ISSUES__{filter_name}", message=message, block="data_quality_gate", diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index d9f54e4..0336cb1 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -1,8 +1,10 @@ +import traceback from temporalio import workflow, activity with workflow.unsafe.imports_passed_through(): from logging import Logger from sientia_do.notifications.handlers import NotificationHandler + from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.redis_base import Redis as RedisBase from sientia_do.temporal.utils.logger import Logger from typing import Any @@ -26,7 +28,18 @@ class Redis(RedisBase): metadata = input_data['metadata'] key = f"last_data_timestamp_{input_data['workflow_name']}_{input_data['schedule_name']}" - data_hold = self.get(key) + try: + data_hold = self.get(key) + except Exception as e: + self.send_notification( + metadata=metadata, + notification_id="REDIS_GET_ERROR", + message=f"Error getting last data timestamp: {e}", + block="get_last_data_timestamp", + level=NotificationLevel.ERROR, + attachment_content=traceback.format_exc() + ) + raise e self.debug( f"Last collected timestamp: {data_hold}", @@ -48,6 +61,12 @@ class Redis(RedisBase): data = DataFrame(input_data['data']) + if data.empty: + self.warning("No data to insert", + metadata=metadata + ) + return None + last_data_timestamp = data['inserted_at'].max() self.debug( @@ -55,7 +74,19 @@ class Redis(RedisBase): metadata=metadata ) - self.set(key, last_data_timestamp, ttl=None) + try: + self.set(key, last_data_timestamp, ttl=None) + except Exception as e: + self.send_notification( + metadata=metadata, + notification_id="REDIS_SET_ERROR", + message=f"Error setting last data timestamp: {e}", + + block="put_last_data_timestamp", + level=NotificationLevel.ERROR, + attachment_content=traceback.format_exc() + ) + raise e return last_data_timestamp @@ -83,7 +114,18 @@ class Redis(RedisBase): key = f"held_data_{input_data['workflow_name']}_{input_data['schedule_name']}" - data_hold = self.get(key) + try: + data_hold = self.get(key) + except Exception as e: + self.send_notification( + metadata=metadata, + notification_id="REDIS_GET_ERROR", + message=f"Error getting held data: {e}", + block="group_and_hold_data", + level=NotificationLevel.ERROR, + attachment_content=traceback.format_exc() + ) + raise e if not data_hold: data_hold = {} @@ -93,22 +135,33 @@ class Redis(RedisBase): ) return data_hold - for _, row in data.iterrows(): - value = row['value'] + try: + for _, row in data.iterrows(): + value = row['value'] - data_hold[row['name']] = value + data_hold[row['name']] = value - data_hold['timestamp'] = data['timestamp'].max() if not data.empty else \ - datetime.now().strftime("%Y-%m-%d %H:%M:%S") + data_hold['timestamp'] = data['timestamp'].max() if not data.empty else \ + datetime.now().strftime("%Y-%m-%d %H:%M:%S") - self.set(key, data_hold, ttl=retention_time) + self.set(key, data_hold, ttl=retention_time) - data_hold_df = DataFrame(data_hold, index=[0]) - data_hold_melted = data_hold_df.melt( - id_vars='timestamp', var_name='variable', value_name='value') - data_hold_melted['model_id'] = input_data['model_id'] + data_hold_df = DataFrame(data_hold, index=[0]) + data_hold_melted = data_hold_df.melt( + id_vars='timestamp', var_name='variable', value_name='value') + data_hold_melted['model_id'] = input_data['model_id'] - data_hold_melted.reset_index(drop=True, inplace=True) + data_hold_melted.reset_index(drop=True, inplace=True) + except Exception as e: + self.send_notification( + metadata=metadata, + notification_id="REDIS_SET_ERROR", + message=f"Error setting held data: {e}", + block="group_and_hold_data", + level=NotificationLevel.ERROR, + attachment_content=traceback.format_exc() + ) + raise e self.debug( f"Data grouped and held successfully:\n {data_hold_melted.to_string()}", diff --git a/tests/activities/test_gates.py b/tests/activities/test_gates.py index ba87f5a..366829a 100644 --- a/tests/activities/test_gates.py +++ b/tests/activities/test_gates.py @@ -11,7 +11,9 @@ def gates_fixture(): """Fixture to create a Gates instance with mocked dependencies.""" logger = Mock() notification_handler = MagicMock() - return Gates(logger=logger, notification_handler=notification_handler) + gates = Gates(logger=logger, notification_handler=notification_handler) + gates.send_notification = MagicMock() + return gates metadata = { @@ -51,7 +53,7 @@ async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture): # Verify assert len(result['tag']) == 2 assert 'tag2' not in result['tag'] - gates_fixture.notification_handler.build_and_send_notification.assert_called_once() + gates_fixture.send_notification.assert_called_once() @pytest.mark.asyncio @@ -84,7 +86,7 @@ async def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture): # Verify data is kept but notification is sent assert len(result['tag']) == 3 # All rows kept - gates_fixture.notification_handler.build_and_send_notification.assert_called_once() + gates_fixture.send_notification.assert_called_once() @pytest.mark.asyncio @@ -117,7 +119,7 @@ async def test_data_quality_gate_with_multiple_filters(gates_fixture): assert result == {'tag': {0: 'tag1', 3: 'tag4'}, 'name': {0: 'tag1', 3: 'tag4'}, 'value': { 0: 1.0, 3: 4.0}, 'timestamp': {0: '2023-01-01', 3: '2023-01-04'}} # Should be called twice (once for each filter) - assert gates_fixture.notification_handler.build_and_send_notification.call_count == 2 + assert gates_fixture.send_notification.call_count == 2 @pytest.mark.asyncio @@ -184,8 +186,8 @@ async def test_data_quality_gate_with_filter_error(gates_fixture): # Verify error notification is sent and data is unchanged assert len(result['tag']) == 1 - gates_fixture.notification_handler.build_and_send_notification.assert_called_once() - call_args = gates_fixture.notification_handler.build_and_send_notification.call_args[1] + gates_fixture.send_notification.assert_called_once() + call_args = gates_fixture.send_notification.call_args[1] assert call_args['notification_id'] == "DATA_QUALITY_GATE_ISSUES" assert call_args['level'] == NotificationLevel.ERROR assert "Filter error" in call_args['message'] @@ -214,7 +216,7 @@ async def test_data_quality_gate_with_empty_data(gates_fixture): # Verify empty result and no notifications assert len(result['tag']) == 0 - gates_fixture.notification_handler.build_and_send_notification.assert_not_called() + gates_fixture.send_notification.assert_not_called() @pytest.mark.asyncio @@ -240,7 +242,7 @@ async def test_data_quality_gate_with_no_filters(gates_fixture): # Verify data is unchanged and no notifications assert len(result['tag']) == 1 - gates_fixture.notification_handler.build_and_send_notification.assert_not_called() + gates_fixture.send_notification.assert_not_called() @pytest.mark.parametrize( @@ -265,14 +267,15 @@ async def test_data_quality_gate_with_no_filters(gates_fixture): ) def test_apply_aggregation(gates_fixture, group_data, aggr_function, expected_result): """Test apply_aggregation method with various scenarios.""" - result = gates_fixture.apply_aggregation(group_data, aggr_function) + result = gates_fixture.apply_aggregation( + group_data, aggr_function, metadata) assert result == expected_result # Check notification was sent for invalid function if aggr_function == 'invalid': - gates_fixture.notification_handler.build_and_send_notification.assert_called_once() + gates_fixture.send_notification.assert_called_once() else: - gates_fixture.notification_handler.build_and_send_notification.assert_not_called() + gates_fixture.send_notification.assert_not_called() @pytest.mark.asyncio @@ -308,7 +311,7 @@ async def test_aggregate_data(gates_fixture): # Verify assert result == expected_result - gates_fixture.notification_handler.build_and_send_notification.assert_not_called() + gates_fixture.send_notification.assert_not_called() @pytest.mark.asyncio @@ -342,7 +345,7 @@ async def test_aggregate_data_with_continue(gates_fixture): # Verify assert result == expected_result - gates_fixture.notification_handler.build_and_send_notification.assert_not_called() + gates_fixture.send_notification.assert_not_called() @pytest.mark.asyncio @@ -373,7 +376,8 @@ async def test_aggregate_data_raise_exception(gates_fixture): await gates_fixture.aggregate_data(input_data) except Exception as e: assert str(e) == "Test exception" - gates_fixture.notification_handler.build_and_send_notification.assert_called_once_with( + gates_fixture.send_notification.assert_called_once_with( + metadata=metadata['metadata'], notification_id="AGGREGATION_ISSUES", message="Error aggregating data: Test exception", block="aggregate_data", diff --git a/tests/activities/test_mongo.py b/tests/activities/test_mongo.py new file mode 100644 index 0000000..171c6fa --- /dev/null +++ b/tests/activities/test_mongo.py @@ -0,0 +1,192 @@ +from datetime import datetime +from unittest.mock import ANY, MagicMock, patch +from pytest import fixture, mark +from sientia_do.notifications.models import NotificationLevel +from scouter.activities.mongodb import MongoDB, clear_mongo_id + + +def test_clear_mongo_id(): + """Test clear_mongo_id""" + data = [ + {'_id': '1', 'name': 'test1'}, + {'_id': '2', 'name': [{ + '_id': '3', + 'name': 'test3' + }]} + ] + + result = clear_mongo_id(data) + + assert result == [{'name': 'test1'}, {'name': [{'name': 'test3'}]}] + + +@patch('scouter.activities.mongodb.MongoClient') +def test_mongodb___init__(mock_mongo_client): + """Test MongoDB __init__""" + mongo = MongoDB( + connection_string='mongodb://localhost:27017', + database_name='test_db', + logger=MagicMock(), + notification_handler=MagicMock() + ) + + mock_mongo_client.assert_called_once_with( + 'mongodb://localhost:27017', + serverSelectionTimeoutMS=5000 + ) + + mock_mongo_client.return_value.server_info.assert_called_once() + + mock_mongo_client.return_value.__getitem__.assert_called_once_with( + 'test_db') + + assert mongo.client is not None + assert mongo.database is not None + + +@fixture +@patch('scouter.activities.mongodb.MongoClient') +def mongodb_activity(mock_mongo_client): + """Test MongoDB activity""" + mongo = MongoDB( + connection_string='mongodb://localhost:27017', + database_name='test_db', + logger=MagicMock(), + notification_handler=MagicMock() + ) + + return mongo + + +def test_shutdown_success(mongodb_activity): + """Test shutdown""" + mongodb_activity.shutdown() + + mongodb_activity.client.close.assert_called_once() + + +def test_shutdown_error(mongodb_activity): + """Test shutdown""" + mongodb_activity.client.close = MagicMock(side_effect=Exception('test')) + + mongodb_activity.shutdown() + + mongodb_activity.client.close.assert_called_once() + + +@mark.asyncio +async def test_load_latest_data_none_last_data_timestamp(mongodb_activity): + """Test load_latest_data""" + collection = MagicMock() + mongodb_activity.database.__getitem__.return_value = collection + + collection.find.return_value = [ + { + 'name': 'test1', + 'value': 1, + 'inserted_at': datetime.strptime( + '2023-01-01 12:00:00.000000', '%Y-%m-%d %H:%M:%S.%f') + } + ] + + result = await mongodb_activity.load_latest_data({ + 'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, + 'collection_name': 'test_collection', + 'last_data_timestamp': None + }) + + mongodb_activity.database.__getitem__.assert_called_once_with( + 'test_collection') + + collection.find.assert_called_once_with( + {}, + {"_id": 0} + ) + + assert result == { + 'name': { + 0: 'test1' + }, + 'value': { + 0: 1 + }, + 'inserted_at': { + 0: '2023-01-01 12:00:00.000000' + } + } + + +@mark.asyncio +async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): + """Test load_latest_data""" + collection = MagicMock() + mongodb_activity.database.__getitem__.return_value = collection + + collection.find.return_value = [ + { + 'name': 'test1', + 'value': 1, + 'inserted_at': datetime.strptime( + '2023-01-01 12:00:00.000000', '%Y-%m-%d %H:%M:%S.%f') + } + ] + + result = await mongodb_activity.load_latest_data({ + 'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, + 'collection_name': 'test_collection', + 'last_data_timestamp': '2023-01-01 12:00:00.000000' + }) + + mongodb_activity.database.__getitem__.assert_called_once_with( + 'test_collection') + + collection.find.assert_called_once_with( + { + 'inserted_at': { + '$gt': datetime.strptime( + '2023-01-01 12:00:00.000000', '%Y-%m-%d %H:%M:%S.%f') + } + }, + {"_id": 0} + ) + + assert result == { + 'name': { + 0: 'test1' + }, + 'value': { + 0: 1 + }, + 'inserted_at': { + 0: '2023-01-01 12:00:00.000000' + } + } + + +@mark.asyncio +async def test_load_latest_data_error(mongodb_activity): + """Test load_latest_data""" + collection = MagicMock() + mongodb_activity.send_notification = MagicMock() + mongodb_activity.database.__getitem__.return_value = collection + + collection.find.side_effect = Exception('test') + + try: + await mongodb_activity.load_latest_data({ + 'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, + 'collection_name': 'test_collection', + 'last_data_timestamp': '2023-01-01 12:00:00.000000' + }) + except Exception as e: + assert str(e) == 'test' + + mongodb_activity.send_notification.assert_called_once_with( + metadata={'workflow_name': 'test_pipeline', + 'schedule_name': 'test_schedule'}, + notification_id='MONGO_LOAD_ERROR', + message='Error loading data from MongoDB: test', + block='load_latest_data', + level=NotificationLevel.ERROR, + attachment_content=ANY + ) diff --git a/tests/activities/test_redis.py b/tests/activities/test_redis.py index f4d8977..672d035 100644 --- a/tests/activities/test_redis.py +++ b/tests/activities/test_redis.py @@ -51,6 +51,90 @@ metadata = { } +@pytest.mark.asyncio +async def test_get_last_data_timestamp_none(redis_activity): + """Test get_last_data_timestamp""" + test_data = { + **metadata, + 'workflow_name': 'test_pipeline', + 'schedule_name': 'test_schedule' + } + + redis_activity.get = MagicMock(return_value=None) + + result = await redis_activity.get_last_data_timestamp(test_data) + + assert result is None + + +@pytest.mark.asyncio +async def test_get_last_data_timestamp_not_none(redis_activity): + """Test get_last_data_timestamp""" + test_data = { + **metadata, + 'workflow_name': 'test_pipeline', + 'schedule_name': 'test_schedule' + } + + redis_activity.get = MagicMock(return_value='2023-01-01 12:00:00') + + result = await redis_activity.get_last_data_timestamp(test_data) + + redis_activity.get.assert_called_once_with( + 'last_data_timestamp_test_pipeline_test_schedule' + ) + + assert result == '2023-01-01 12:00:00' + + +@pytest.mark.asyncio +async def test_put_last_data_timestamp_empty_dataframe(redis_activity): + """Test put_last_data_timestamp with empty dataframe""" + test_data = { + **metadata, + 'workflow_name': 'test_pipeline', + 'schedule_name': 'test_schedule', + 'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records') + } + + redis_activity.set = MagicMock() + + result = await redis_activity.put_last_data_timestamp(test_data) + + assert result is None + + redis_activity.set.assert_not_called() + + +@pytest.mark.asyncio +async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity): + """Test put_last_data_timestamp with not empty dataframe""" + + data = DataFrame({ + 'name': ['sensor1', 'sensor2'], + 'value': [25.5, 30.0], + 'inserted_at': ['2023-01-01 12:00:00', '2023-01-01 12:00:01'] + }) + test_data = { + **metadata, + 'workflow_name': 'test_pipeline', + 'schedule_name': 'test_schedule', + 'data': data.to_dict('records') + } + + redis_activity.set = MagicMock() + + result = await redis_activity.put_last_data_timestamp(test_data) + + assert result == '2023-01-01 12:00:01' + + redis_activity.set.assert_called_once_with( + 'last_data_timestamp_test_pipeline_test_schedule', + '2023-01-01 12:00:01', + ttl=None + ) + + @pytest.mark.asyncio async def test_group_and_hold_data_new_key(redis_activity): """Test group_and_hold_data with a new key""" diff --git a/tests/utils/test_connectors_config.py b/tests/utils/test_connectors_config.py index bd1694b..e3c4bbd 100644 --- a/tests/utils/test_connectors_config.py +++ b/tests/utils/test_connectors_config.py @@ -2,6 +2,8 @@ import os from unittest.mock import patch import pytest from scouter.utils.connectors_config import ( + build_druid_config, + build_mongodb_config, build_postgres_config, build_kafka_config, build_redis_config @@ -113,3 +115,53 @@ def test_build_redis_config_with_env_vars(): 'username': 'test', 'password': 'test' } + + +def test_build_mongodb_config_defaults(): + """Test that build_mongodb_config returns default values when no env vars are set""" + config = build_mongodb_config() + + assert config == { + 'connection_string': 'mongodb://sientia:sientia@localhost:27017', # NOSONAR + 'database_name': 'sientia' + } + + +def test_build_mongodb_config_with_env_vars(): + """Test that build_mongodb_config uses env vars when set""" + with patch.dict(os.environ, { + 'MONGODB_URL': 'mongodb.example.com:27017', + 'MONGODB_DATABASE_NAME': 'test_db', + 'MONGODB_USERNAME': 'test', + 'MONGODB_PASSWORD': 'test' + }): + config = build_mongodb_config() + + assert config == { + 'connection_string': 'mongodb://test:test@mongodb.example.com:27017', + 'database_name': 'test_db' + } + + +def test_build_druid_config_defaults(): + """Test that build_druid_config returns default values when no env vars are set""" + config = build_druid_config() + + assert config == { + 'host': 'localhost', + 'port': 8082 + } + + +def test_build_druid_config_with_env_vars(): + """Test that build_druid_config uses env vars when set""" + with patch.dict(os.environ, { + 'DRUID_HOST': 'druid.example.com', + 'DRUID_PORT': '8083' + }): + config = build_druid_config() + + assert config == { + 'host': 'druid.example.com', + 'port': 8083 + }