gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1163

Browse Source 
Refactor Ingestor and Manager classes to use CoreNotificationHandler and improve metadata handling

- Updated Ingestor class to initialize logger using get_logger and replaced notification handler initialization with metadata dictionary.
- Refactored DataManager, IngestorManager, OpcManager, and ResourceManager classes to inherit from BaseActivity, allowing for consistent logger and notification handler usage.
- Enhanced notification handling by integrating metadata into notification methods across various managers.
- Added .vscode/ to .gitignore to exclude VSCode configuration files.
This commit is contained in:
vitor-aignosi
2025-07-16 10:54:34 -03:00
parent 0885544bc7
commit 5132162eaa
6 changed files with 116 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -173,3 +173,6 @@ cython_debug/
# PyPI configuration file # PyPI configuration file
.pypirc .pypirc
# VSCode
.vscode/

60
ingestor/ingestor.py
View File

@@ -1,4 +1,3 @@
from logging import Formatter, StreamHandler, getLogger
from os import getenv from os import getenv
from copy import deepcopy from copy import deepcopy
from typing import Dict, Any from typing import Dict, Any
@@ -62,8 +61,7 @@ class Ingestor:
self.mongo_connection_string = f"mongodb://{mongo_username}:{mongo_password}@{mongo_url}" self.mongo_connection_string = f"mongodb://{mongo_username}:{mongo_password}@{mongo_url}"
self.kafka_servers = kafka_servers.split(",") self.kafka_servers = kafka_servers.split(",")
self.logger = None self.logger = get_logger(__name__)
self.init_logger()
self.notification_handler = NotificationHandler( self.notification_handler = NotificationHandler(
connection_string=self.mongo_connection_string, connection_string=self.mongo_connection_string,
database=self.mongo_database, database=self.mongo_database,
@@ -71,11 +69,12 @@ class Ingestor:
project_name="OPC_INGESTOR" project_name="OPC_INGESTOR"
) )
self.notification_handler.base_notification.pipeline = 'OPC_INGESTOR' self.metadata = {
self.notification_handler.base_notification.trigger = 'INGESTOR' 'schedule_name': 'OPC_INGESTOR',
self.notification_handler.base_notification.model_name = '-' 'trigger': 'INGESTOR',
self.notification_handler.base_notification.model_id = '-' 'model_name': '-',
'model_id': '-'
}
self.ingestor_manager = None self.ingestor_manager = None
def shutdown(self): def shutdown(self):
@@ -85,28 +84,6 @@ class Ingestor:
def __del__(self): def __del__(self):
self.shutdown() self.shutdown()
def init_logger(self):
"""
Initializes a logger instance for the class.
This method sets up a logger with a specified log level, a stream handler,
and a formatter. The log level is determined by the environment variable
"LOG_LEVEL", defaulting to "INFO" if not set. The logger is then attached
to the instance for use throughout the class.
Attributes:
self.logger (logging.Logger): The configured logger instance.
"""
logger = getLogger(__name__)
logger.setLevel(getenv("LOG_LEVEL", "INFO"))
handler = StreamHandler()
formatter = Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
self.logger = logger
def handle_acquired_tags(self, acquired): def handle_acquired_tags(self, acquired):
""" """
Handles the acquired tags by subscribing to them if available. Handles the acquired tags by subscribing to them if available.
@@ -150,22 +127,23 @@ class Ingestor:
""" """
self.ingestor_manager = IngestorManager( self.ingestor_manager = IngestorManager(
self.kafka_servers, kafka_servers=self.kafka_servers,
{ redis_config={
'host': self.redis_host, 'host': self.redis_host,
'port': self.redis_port, 'port': self.redis_port,
'username': self.redis_username, 'username': self.redis_username,
'password': self.redis_password, 'password': self.redis_password,
}, },
self.lease_ttl, lease_ttl=self.lease_ttl,
self.heartbeat_ttl, heartbeat_ttl=self.heartbeat_ttl,
self.pod_id, pod_id=self.pod_id,
self.poll_interval, poll_interval=self.poll_interval,
self.mongo_connection_string, mongo_connection_string=self.mongo_connection_string,
self.mongo_database, mongo_database=self.mongo_database,
self.logger, metadata=self.metadata,
self.notification_handler, logger=self.logger,
self.export_to_kafka, notification_handler=self.notification_handler,
export_to_kafka=self.export_to_kafka,
) )
# Declare ingestor ative # Declare ingestor ative

19
ingestor/managers/data_manager.py
View File

@@ -1,25 +1,26 @@
import json import json
from logging import Logger
from time import sleep from time import sleep
from datetime import datetime, timezone from datetime import datetime, timezone
from pymongo import MongoClient from pymongo import MongoClient
from kafka import KafkaProducer from kafka import KafkaProducer
from kafka.errors import NoBrokersAvailable from kafka.errors import NoBrokersAvailable
from sientia_do.notifications.handlers import NotificationHandler from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel from sientia_do.notifications.models import NotificationLevel
from sientia_do. from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.temporal.utils.logger import Logger
import traceback import traceback
import ingestor.metrics as metrics import ingestor.metrics as metrics
import os import os
class DataManager(): class DataManager(BaseActivity):
def __init__( def __init__(
self, self,
kafka_servers: str, kafka_servers: str,
mongo_connection_string: str, mongo_connection_string: str,
mongo_database: str, mongo_database: str,
export_to_kafka: bool, export_to_kafka: bool,
metadata: dict,
logger: Logger, logger: Logger,
notification_handler: NotificationHandler, notification_handler: NotificationHandler,
) -> None: ) -> None:
@@ -92,8 +93,8 @@ class DataManager():
f"DataManager initialized with MongoDB servers: {self.connection_string}" f"DataManager initialized with MongoDB servers: {self.connection_string}"
) )
self.logger = logger BaseActivity.__init__(self, logger=logger,
self.notification_handler = notification_handler notification_handler=notification_handler)
def shutdown(self): def shutdown(self):
"""Closes the Kafka producer connection.""" """Closes the Kafka producer connection."""
@@ -164,7 +165,8 @@ class DataManager():
metrics.KAFKA_MESSAGES_ERRORS.labels( metrics.KAFKA_MESSAGES_ERRORS.labels(
pod_id=self.pod_id, topic=topic).inc() pod_id=self.pod_id, topic=topic).inc()
trace = traceback.format_exc() trace = traceback.format_exc()
self.notification_handler.build_and_send_notification( self.send_notification(
metadata=self.metadata,
notification_id=f"KAFKA_PRODUCER_ERROR_{topic}", notification_id=f"KAFKA_PRODUCER_ERROR_{topic}",
message=f"Error publishing message to topic {topic}: {e}", message=f"Error publishing message to topic {topic}: {e}",
block="kafka_producer", block="kafka_producer",
@@ -187,7 +189,8 @@ class DataManager():
except Exception as e: except Exception as e:
trace = traceback.format_exc() trace = traceback.format_exc()
self.notification_handler.build_and_send_notification( self.send_notification(
metadata=self.metadata,
notification_id=f"MONGO_PRODUCER_ERROR_{topic}", notification_id=f"MONGO_PRODUCER_ERROR_{topic}",
message=f"Error inserting message to MongoDB: {e}", message=f"Error inserting message to MongoDB: {e}",
block="mongo_producer", block="mongo_producer",

65
ingestor/managers/ingestor_manager.py
View File

@@ -1,20 +1,22 @@
from logging import Logger
import traceback import traceback
from typing import Dict, List from typing import Dict, List
from copy import deepcopy from copy import deepcopy
from sientia_do.notifications.handlers import NotificationHandler from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.temporal.utils.logger import Logger
from ingestor.managers.data_manager import DataManager from ingestor.managers.data_manager import DataManager
from ingestor.managers.opc_manager import OpcManager from ingestor.managers.opc_manager import OpcManager
from ingestor.managers.resource_manager import ResourceManager from ingestor.managers.resource_manager import ResourceManager
import ingestor.metrics as metrics import ingestor.metrics as metrics
class IngestorManager(): class IngestorManager(BaseActivity):
def __init__(self, def __init__(self,
kafka_servers: str, redis_data: dict, kafka_servers: str, redis_data: dict,
lease_ttl: int, heartbeat_ttl: int, pod_id: str, lease_ttl: int, heartbeat_ttl: int, pod_id: str,
poll_interval: int, mongo_connection_string: str, mongo_database: str, poll_interval: int, mongo_connection_string: str, mongo_database: str,
metadata: dict,
logger: Logger, notification_handler: NotificationHandler, logger: Logger, notification_handler: NotificationHandler,
export_to_kafka: bool = False): export_to_kafka: bool = False):
@@ -24,23 +26,39 @@ class IngestorManager():
redis_password = redis_data.get('password', None) redis_password = redis_data.get('password', None)
self.data_manager = DataManager( self.data_manager = DataManager(
kafka_servers, mongo_connection_string, mongo_database, kafka_servers=kafka_servers,
export_to_kafka, logger, notification_handler) mongo_connection_string=mongo_connection_string,
mongo_database=mongo_database,
export_to_kafka=export_to_kafka,
metadata=metadata,
logger=logger,
notification_handler=notification_handler
)
self.opc_managers = {} self.opc_managers = {}
self.resource_manager = ResourceManager( self.resource_manager = ResourceManager(
redis_host, redis_port, lease_ttl, heartbeat_ttl, pod_id, redis_username, redis_password host=redis_host,
port=redis_port,
lease_ttl=lease_ttl,
heartbeat_ttl=heartbeat_ttl,
pod_id=pod_id,
metadata=metadata,
logger=logger,
notification_handler=notification_handler,
username=redis_username,
password=redis_password,
) )
self.number_of_slots = 0 self.number_of_slots = 0
self.poll_interval = poll_interval self.poll_interval = poll_interval
self.logger = logger
self.managed_tags = {} self.managed_tags = {}
self.opc_servers = {} self.opc_servers = {}
self.notification_handler = notification_handler
self.pod_id = pod_id self.pod_id = pod_id
self.metadata = metadata
def initialize_opc_from_config(self, server_config: dict, BaseActivity.__init__(self, logger=logger,
data_manager: DataManager, logger: Logger) -> OpcManager | None: notification_handler=notification_handler)
def initialize_opc_from_config(self, server_config: dict) -> OpcManager | None:
""" """
Initializes an OPC Manager instance using the provided server configuration. Initializes an OPC Manager instance using the provided server configuration.
Args: Args:
@@ -63,12 +81,17 @@ class IngestorManager():
self.logger.info( self.logger.info(
f"Initializing OpcManager at {server_config['url']}") f"Initializing OpcManager at {server_config['url']}")
manager = OpcManager( manager = OpcManager(
server_config['name'], server_config['url'], name=server_config['name'],
data_manager, logger, server_config['server_uri'], url=server_config['url'],
self.notification_handler, self.pod_id, server_config.get( data_manager=self.data_manager,
'cert_path'), logger=self.logger,
server_config.get('private_key_path'), server_uri=server_config['server_uri'],
server_config.get('server_cert_path') notification_handler=self.notification_handler,
pod_id=self.pod_id,
metadata=self.metadata,
cert_path=server_config.get('cert_path'),
private_key_path=server_config.get('private_key_path'),
server_cert_path=server_config.get('server_cert_path')
) )
manager.config = server_config manager.config = server_config
@@ -76,7 +99,8 @@ class IngestorManager():
except Exception as e: except Exception as e:
trace = traceback.format_exc() trace = traceback.format_exc()
self.notification_handler.build_and_send_notification( self.send_notification(
metadata=self.metadata,
notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}', notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}',
message=f'Error initializing OPC manager: {e}', message=f'Error initializing OPC manager: {e}',
block="opc_manager", block="opc_manager",
@@ -135,7 +159,7 @@ class IngestorManager():
f"Initializing OPC manager for server {server}" f"Initializing OPC manager for server {server}"
) )
server_instance = self.initialize_opc_from_config( server_instance = self.initialize_opc_from_config(
server_config, self.data_manager, self.logger server_config
) )
elif server_instance.config != server_config: elif server_instance.config != server_config:
@@ -145,7 +169,7 @@ class IngestorManager():
server_instance.disconnect() server_instance.disconnect()
del self.opc_managers[server] del self.opc_managers[server]
server_instance = self.initialize_opc_from_config( server_instance = self.initialize_opc_from_config(
server_config, self.data_manager, self.logger server_config
) )
else: else:
self.logger.debug( self.logger.debug(
@@ -416,7 +440,8 @@ class IngestorManager():
metrics.OPC_SUBSCRIPTION_ERRORS.labels( metrics.OPC_SUBSCRIPTION_ERRORS.labels(
pod_id=self.pod_id, server=server, slot=slot).inc() pod_id=self.pod_id, server=server, slot=slot).inc()
trace = traceback.format_exc() trace = traceback.format_exc()
self.notification_handler.build_and_send_notification( self.send_notification(
metadata=self.metadata,
notification_id=f'OPC_SUBSCRIPTION_ERROR_{slot}:{server}', notification_id=f'OPC_SUBSCRIPTION_ERROR_{slot}:{server}',
message=f'Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}', message=f'Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}',
block="opc_manager", block="opc_manager",

26
ingestor/managers/opc_manager.py
View File

@@ -1,24 +1,23 @@
import json import json
from logging import Logger
from pathlib import Path from pathlib import Path
from typing import Callable
from asyncua.crypto.security_policies import SecurityPolicyBasic256 from asyncua.crypto.security_policies import SecurityPolicyBasic256
from asyncua.sync import Client from asyncua.sync import Client
from sientia_do.notifications.models import NotificationLevel from sientia_do.notifications.models import NotificationLevel
from sientia_do.notifications.handlers import NotificationHandler from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.temporal.utils.logger import Logger
from ingestor.managers.data_manager import DataManager from ingestor.managers.data_manager import DataManager
import ingestor.metrics as metrics import ingestor.metrics as metrics
class OpcManager(): class OpcManager(BaseActivity):
def __init__(self, name: str, url: str, data_manager: DataManager, def __init__(self, name: str, url: str, data_manager: DataManager,
logger: Logger, server_uri: str, notification_handler: NotificationHandler, pod_id: str, logger: Logger, server_uri: str, notification_handler: NotificationHandler, pod_id: str, metadata: dict,
cert_path: str = None, private_key_path: str = None, server_cert_path: str = None): cert_path: str = None, private_key_path: str = None, server_cert_path: str = None):
self.url = url self.url = url
self.name = name self.name = name
self.server_uri = server_uri self.server_uri = server_uri
self.data_queue = {} self.data_queue = {}
self.logger = logger
self.non_receive_count = 0 self.non_receive_count = 0
self.client = None self.client = None
self.cert_path = cert_path self.cert_path = cert_path
@@ -27,13 +26,17 @@ class OpcManager():
self.nodes = {} self.nodes = {}
self.subscriptions = {} self.subscriptions = {}
self.data_manager = data_manager self.data_manager = data_manager
self.notification_handler = notification_handler
self.pod_id = pod_id self.pod_id = pod_id
self.metadata = metadata
metrics.OPC_CONNECTION_STATUS.labels( metrics.OPC_CONNECTION_STATUS.labels(
pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0) pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0)
metrics.OPC_TAGS_SUBSCRIBED.labels( metrics.OPC_TAGS_SUBSCRIBED.labels(
pod_id=self.pod_id, server_name=self.name).set(0) pod_id=self.pod_id, server_name=self.name).set(0)
BaseActivity.__init__(self, logger=logger,
notification_handler=notification_handler)
def __str__(self): def __str__(self):
return f"OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n" \ return f"OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n" \
f"nodes={self.nodes}, subscriptions={self.subscriptions}" f"nodes={self.nodes}, subscriptions={self.subscriptions}"
@@ -296,7 +299,8 @@ class OpcManager():
if self.nodes[node]['cycle_rule']['cycle_count'] >= 5: if self.nodes[node]['cycle_rule']['cycle_count'] >= 5:
name = config['tag_name'] name = config['tag_name']
cycles = self.nodes[node]['cycle_rule']['cycle_count'] cycles = self.nodes[node]['cycle_rule']['cycle_count']
self.notification_handler.build_and_send_notification( self.send_notification(
metadata=self.metadata,
notification_id=f'TAG_{node}:{name}_LISTENNING_STOPPED', notification_id=f'TAG_{node}:{name}_LISTENNING_STOPPED',
message=f'{cycles} cycles without receive from {node}:{name}', message=f'{cycles} cycles without receive from {node}:{name}',
block="opc_manager", block="opc_manager",
@@ -314,7 +318,8 @@ class OpcManager():
metrics.OPC_CYCLES_WITHOUT_DATA.labels( metrics.OPC_CYCLES_WITHOUT_DATA.labels(
pod_id=self.pod_id, server_name=self.name).set(self.non_receive_count) pod_id=self.pod_id, server_name=self.name).set(self.non_receive_count)
if self.non_receive_count >= 5: if self.non_receive_count >= 5:
self.notification_handler.build_and_send_notification( self.send_notification(
metadata=self.metadata,
notification_id=f'OPC_LISTENNING_STOPPED__{self.name}', notification_id=f'OPC_LISTENNING_STOPPED__{self.name}',
message=f'{self.non_receive_count} cycles without ' message=f'{self.non_receive_count} cycles without '
f'receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}', f'receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}',
@@ -324,7 +329,8 @@ class OpcManager():
if self.non_receive_count >= 15: if self.non_receive_count >= 15:
metrics.OPC_RECONNECTIONS_TOTAL.labels( metrics.OPC_RECONNECTIONS_TOTAL.labels(
pod_id=self.pod_id, server_name=self.name).inc() pod_id=self.pod_id, server_name=self.name).inc()
self.notification_handler.build_and_send_notification( self.send_notification(
metadata=self.metadata,
notification_id=f'OPC_CONNECTION_RETRY__{self.name}', notification_id=f'OPC_CONNECTION_RETRY__{self.name}',
message=f'Retrying to connect to server {self.name}', message=f'Retrying to connect to server {self.name}',
block="opc_manager", block="opc_manager",

26
ingestor/managers/resource_manager.py
View File

@@ -3,9 +3,13 @@ from typing import List
from redis import Redis from redis import Redis
from time import time from time import time
import ingestor.metrics as metrics import ingestor.metrics as metrics
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.utils.logger import Logger
from sientia_do.temporal.activities.base import BaseActivity
class ResourceManager: class ResourceManager(BaseActivity):
def __init__( def __init__(
self, self,
host: str, host: str,
@@ -13,6 +17,9 @@ class ResourceManager:
lease_ttl: int, lease_ttl: int,
heartbeat_ttl: int, heartbeat_ttl: int,
pod_id: str, pod_id: str,
metadata: dict,
logger: Logger,
notification_handler: NotificationHandler,
username: str | None = None, username: str | None = None,
password: str | None = None, password: str | None = None,
) -> None: ) -> None:
@@ -28,12 +35,16 @@ class ResourceManager:
self.redis.ping() self.redis.ping()
metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1) metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1)
except Exception as e: except Exception as e:
print(f"Failed to connect to Redis: {e}") logger.error(f"Failed to connect to Redis: {e}")
metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0) metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
raise raise
self.lease_ttl = lease_ttl self.lease_ttl = lease_ttl
self.heartbeat_ttl = heartbeat_ttl self.heartbeat_ttl = heartbeat_ttl
self.metadata = metadata
BaseActivity.__init__(self, logger=logger,
notification_handler=notification_handler)
def _execute_redis_op(self, operation_name: str, func, *args, **kwargs): def _execute_redis_op(self, operation_name: str, func, *args, **kwargs):
"""Wrapper to execute Redis operations and record metrics.""" """Wrapper to execute Redis operations and record metrics."""
@@ -52,7 +63,13 @@ class ResourceManager:
metrics.REDIS_OPERATIONS_ERRORS.labels( metrics.REDIS_OPERATIONS_ERRORS.labels(
pod_id=self.pod_id, operation=operation_name pod_id=self.pod_id, operation=operation_name
).inc() ).inc()
print(f"Error in Redis operation '{operation_name}': {e}") self.send_notification(
metadata=self.metadata,
notification_id=f"REDIS_OPERATION_ERROR_{operation_name}",
message=f"Error in Redis operation '{operation_name}': {e}",
block="redis_manager",
level=NotificationLevel.ERROR,
)
raise raise
def get(self, key: str) -> dict: def get(self, key: str) -> dict:
@@ -151,7 +168,8 @@ class ResourceManager:
None None
""" """
self._execute_redis_op("delete", self.redis.delete, f"lease:opc_tags:{tag_id}") self._execute_redis_op("delete", self.redis.delete,
f"lease:opc_tags:{tag_id}")
def get_all_ingestors(self) -> List[str]: def get_all_ingestors(self) -> List[str]:
""" """