gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Update dependencies, improve CI workflow, and enhance code formatting

Browse Source 
- Updated the `sientia-dataops-library` dependency version from 1.4.3 to 1.4.6 in `requirements.txt`.
- Modified the GitHub Actions workflow to install development and runtime dependencies separately, improving clarity and organization.
- Added code formatting and linting checks using Ruff, along with type checking using mypy, to ensure code quality.
- Updated `.gitignore` to include additional cache directories and log files.
- Refactored code in various files for consistency in string formatting and improved logging messages.
This commit is contained in:
Vitor Pimentel dos Santos
2025-10-17 12:58:24 -03:00
parent 464c9aae9b
commit e2462af31c
21 changed files with 1536 additions and 1368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
ingestor/managers/data_manager.py
View File

@@ -1,16 +1,18 @@
import json
import os
import traceback
from time import sleep
from pymongo import MongoClient
from kafka import KafkaProducer
from kafka.errors import NoBrokersAvailable
from pymongo import MongoClient
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.constants import now
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.observability.logger import Logger
import traceback
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.temporal.constants import now
import ingestor.metrics as metrics
import os
class DataManager(BaseActivity):
@@ -85,49 +87,43 @@ class DataManager(BaseActivity):
- KAFKA_CONNECTION_STATUS: Set to 1 on successful connection, 0 on failure
"""
self.pod_id = os.getenv("HOSTNAME", "localhost")
self.pod_id = os.getenv('HOSTNAME', 'localhost')
self.kafka_producer = None
self.export_to_kafka = export_to_kafka
if self.export_to_kafka:
for i in range(0, 3):
logger.info(
f"Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}"
f'Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}'
)
try:
self.kafka_producer = KafkaProducer(
bootstrap_servers=kafka_servers,
value_serializer=lambda v: json.dumps(v).encode(
"utf-8"
'utf-8'
), # Serialize JSON messages
key_serializer=lambda k: str(
k).encode("utf-8") if k else None,
key_serializer=lambda k: str(k).encode('utf-8') if k else None,
)
# Kafka connected
metrics.KAFKA_CONNECTION_STATUS.labels(
pod_id=self.pod_id).set(1)
metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1)
break
except NoBrokersAvailable:
logger.error(
f"Kafka servers {kafka_servers} are not available. Retrying..."
)
logger.error(f'Kafka servers {kafka_servers} are not available. Retrying...')
sleep(5)
else:
# Kafka not connected
metrics.KAFKA_CONNECTION_STATUS.labels(
pod_id=self.pod_id).set(0)
metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
logger.error(
f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts."
f'Failed to connect to Kafka servers {kafka_servers} after 3 attempts.'
)
raise NoBrokersAvailable(
f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts."
f'Failed to connect to Kafka servers {kafka_servers} after 3 attempts.'
)
logger.info(
f"DataManager initialized with Kafka servers: {kafka_servers}")
logger.info(f'DataManager initialized with Kafka servers: {kafka_servers}')
logger.info(
f"Trying to initializing DataManager with MongoDB servers: {mongo_connection_string}"
f'Trying to initializing DataManager with MongoDB servers: {mongo_connection_string}'
)
self.connection_string = mongo_connection_string
@@ -140,13 +136,11 @@ class DataManager(BaseActivity):
self.mongo_db = self.mongo_client[self.database]
logger.info(
f"DataManager initialized with MongoDB servers: {self.connection_string}"
)
logger.info(f'DataManager initialized with MongoDB servers: {self.connection_string}')
BaseActivity.__init__(self, logger=logger,
notification_handler=notification_handler,
set_error_counter=True)
BaseActivity.__init__(
self, logger=logger, notification_handler=notification_handler, set_error_counter=True
)
def shutdown(self):
"""
@@ -165,22 +159,19 @@ class DataManager(BaseActivity):
self.kafka_producer.flush(timeout=10)
self.kafka_producer.close()
# Mark as disconnected
metrics.KAFKA_CONNECTION_STATUS.labels(
pod_id=self.pod_id).set(0)
metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
except Exception as e:
self.logger.error(f"Error closing Kafka producer: {e}")
self.logger.error(f'Error closing Kafka producer: {e}')
else:
self.logger.warning(
"Kafka producer is already closed or not initialized.")
self.logger.warning('Kafka producer is already closed or not initialized.')
if self.mongo_client:
try:
self.mongo_client.close()
except Exception as e:
self.logger.error(f"Error closing MongoDB client: {e}")
self.logger.error(f'Error closing MongoDB client: {e}')
else:
self.logger.warning(
"MongoDB client is already closed or not initialized.")
self.logger.warning('MongoDB client is already closed or not initialized.')
def __del__(self):
self.shutdown()
@@ -197,7 +188,7 @@ class DataManager(BaseActivity):
msg: Kafka message object containing delivery details
"""
self.logger.debug(
f"Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}"
f'Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}'
)
def delivery_error(self, err: str):
@@ -210,7 +201,7 @@ class DataManager(BaseActivity):
Args:
err: Error information from the failed delivery attempt
"""
self.logger.error(f"Delivery failed for record : {err}")
self.logger.error(f'Delivery failed for record : {err}')
def publish(self, topic: str, data: dict) -> None:
"""
@@ -229,26 +220,22 @@ class DataManager(BaseActivity):
if self.export_to_kafka:
try:
self.logger.debug(
f"Publishing message to topic {topic}: {data}")
self.logger.debug(f'Publishing message to topic {topic}: {data}')
self.kafka_producer.send(topic=topic, value=data).add_callback(
self.delivery_report
).add_errback(self.delivery_error)
self.kafka_producer.flush(timeout=10)
metrics.KAFKA_MESSAGES_SENT.labels(
pod_id=self.pod_id, topic=topic).inc()
metrics.KAFKA_MESSAGES_SENT.labels(pod_id=self.pod_id, topic=topic).inc()
except Exception as e:
metrics.KAFKA_MESSAGES_ERRORS.labels(
pod_id=self.pod_id, topic=topic).inc()
metrics.KAFKA_MESSAGES_ERRORS.labels(pod_id=self.pod_id, topic=topic).inc()
trace = traceback.format_exc()
self.send_notification(
metadata=self.metadata,
notification_id=f"KAFKA_PRODUCER_ERROR_{topic}",
message=f"Error publishing message to topic {topic}: {e}",
block="kafka_producer",
notification_id=f'KAFKA_PRODUCER_ERROR_{topic}',
message=f'Error publishing message to topic {topic}: {e}',
block='kafka_producer',
level=NotificationLevel.ERROR,
attachment_content=trace,
)
@@ -260,25 +247,22 @@ class DataManager(BaseActivity):
collection.insert_one(
{
**data,
"inserted_at": now(),
'inserted_at': now(),
}
)
self.logger.debug(
f"Message inserted into MongoDB collection {topic}: {data}")
self.logger.debug(f'Message inserted into MongoDB collection {topic}: {data}')
metrics.TAG_WRITTEN_COUNT.labels(
pod_id=self.pod_id,
tag_name=data["name"],
collection_name=topic
pod_id=self.pod_id, tag_name=data['name'], collection_name=topic
).inc()
except Exception as e:
trace = traceback.format_exc()
self.send_notification(
metadata=self.metadata,
notification_id=f"MONGO_PRODUCER_ERROR_{topic}",
message=f"Error inserting message to MongoDB: {e}",
block="mongo_producer",
notification_id=f'MONGO_PRODUCER_ERROR_{topic}',
message=f'Error inserting message to MongoDB: {e}',
block='mongo_producer',
level=NotificationLevel.ERROR,
attachment_content=trace,
)