gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SAM0123-231: Remove Kafka integration from Ingestor service.

Browse Source
This commit is contained in:
vitor-aignosi
2026-07-23 12:25:29 -03:00
parent c157d1649a
commit b10c226e4e
8 changed files with 18 additions and 414 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
tests/unit/managers/test_data_manager.py
View File

@@ -1,6 +1,5 @@
from unittest.mock import ANY, AsyncMock, MagicMock, patch
from kafka.errors import NoBrokersAvailable
from pytest import fixture, mark
from sientia_do.notifications.models import NotificationLevel
@@ -18,14 +17,11 @@ metadata = {
@fixture
@patch('ingestor.managers.data_manager.KafkaProducer')
@patch('ingestor.managers.data_manager.MongoDBRepository')
def data_manager(mongodb_repository, kafka):
def data_manager(mongodb_repository):
data_manager = DataManager(
kafka_servers='localhost:9092',
mongo_connection_string='mongodb://localhost:27017',
mongo_database='sientia',
export_to_kafka=True,
logger=MagicMock(),
notification_handler=MagicMock(),
metadata=metadata['metadata'],
@@ -38,141 +34,34 @@ def data_manager(mongodb_repository, kafka):
return data_manager
@patch('ingestor.managers.data_manager.KafkaProducer')
@patch('ingestor.managers.data_manager.MongoDBRepository')
def test___init___success(mongodb_repository, kafka):
def test___init___success(mongodb_repository):
logger_mock = MagicMock()
data_manager = DataManager(
metadata=metadata['metadata'],
kafka_servers='localhost:9092',
mongo_connection_string='mongodb://localhost:27017',
mongo_database='sientia',
export_to_kafka=True,
logger=logger_mock,
notification_handler=MagicMock(),
metrics_controller=MagicMock(),
)
kafka.assert_called_once_with(
bootstrap_servers='localhost:9092', value_serializer=ANY, key_serializer=ANY
)
assert data_manager.kafka_producer is not None
logger_mock.info.assert_any_call(
'Trying (0) to initializing DataManager with Kafka servers: localhost:9092'
'DataManager initialized with MongoDB servers: mongodb://localhost:27017'
)
logger_mock.info.assert_any_call('DataManager initialized with Kafka servers: localhost:9092')
logger_mock.error.assert_not_called()
assert logger_mock.info.call_count == 4
assert data_manager.connection_string == 'mongodb://localhost:27017'
@patch('ingestor.managers.data_manager.KafkaProducer')
@patch('ingestor.managers.data_manager.MongoDBRepository')
def test___init___second_attempt(mongodb_repository, kafka):
kafka.side_effect = [NoBrokersAvailable, MagicMock()]
logger_mock = MagicMock()
data_manager = DataManager(
kafka_servers='localhost:9092',
mongo_connection_string='mongodb://localhost:27017',
mongo_database='sientia',
export_to_kafka=True,
logger=logger_mock,
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=MagicMock(),
)
kafka.assert_any_call(
bootstrap_servers='localhost:9092', value_serializer=ANY, key_serializer=ANY
)
assert kafka.call_count == 2
assert data_manager.kafka_producer is not None
logger_mock.info.assert_any_call(
'Trying (0) to initializing DataManager with Kafka servers: localhost:9092'
)
logger_mock.info.assert_any_call(
'Trying (1) to initializing DataManager with Kafka servers: localhost:9092'
)
logger_mock.info.assert_any_call('DataManager initialized with Kafka servers: localhost:9092')
logger_mock.error.assert_called_once_with(
'Kafka servers localhost:9092 are not available. Retrying...'
)
assert logger_mock.info.call_count == 5
@patch('ingestor.managers.data_manager.KafkaProducer')
@patch('ingestor.managers.data_manager.MongoDBRepository')
def test___init___failure_max_attempts(mongodb_repository, kafka):
kafka.side_effect = NoBrokersAvailable
logger_mock = MagicMock()
try:
DataManager(
kafka_servers='localhost:9092',
mongo_connection_string='mongodb://localhost:27017',
mongo_database='sientia',
export_to_kafka=True,
logger=logger_mock,
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=MagicMock(),
)
except NoBrokersAvailable as e:
assert (
str(e)
== 'NoBrokersAvailable: Failed to connect to Kafka servers localhost:9092 after 3 attempts.'
)
assert kafka.call_count == 3
logger_mock.info.assert_any_call(
'Trying (0) to initializing DataManager with Kafka servers: localhost:9092'
)
logger_mock.info.assert_any_call(
'Trying (1) to initializing DataManager with Kafka servers: localhost:9092'
)
logger_mock.info.assert_any_call(
'Trying (2) to initializing DataManager with Kafka servers: localhost:9092'
)
logger_mock.error.assert_called_with(
'Failed to connect to Kafka servers localhost:9092 after 3 attempts.'
)
assert logger_mock.info.call_count == 3
else:
raise AssertionError('Expected NoBrokersAvailable exception was not raised.')
def test_shutdown_has_producer(data_manager):
flush_mock = MagicMock()
def test_shutdown(data_manager):
close_mock = MagicMock()
data_manager.kafka_producer.flush = flush_mock
data_manager.kafka_producer.close = close_mock
data_manager.mongo_repository.close = close_mock
data_manager.shutdown()
flush_mock.assert_called_once()
close_mock.assert_called_once()
def test_shutdown_no_producer(data_manager):
data_manager.kafka_producer = None
data_manager.shutdown()
data_manager.logger.warning.assert_any_call(
'Kafka producer is already closed or not initialized.'
)
def test_shutdown_exception(data_manager):
data_manager.kafka_producer.flush = MagicMock(side_effect=Exception('Test error'))
data_manager.kafka_producer.close = MagicMock()
data_manager.shutdown()
data_manager.logger.error.assert_called_once_with('Error closing Kafka producer: Test error')
def test_shutdown_exception_mongo(data_manager):
data_manager.mongo_repository.close = MagicMock(side_effect=Exception('Test error'))
@@ -187,88 +76,24 @@ def test___del__(data_manager):
data_manager.shutdown.assert_called_once()
def test_delivery_report(data_manager):
msg = MagicMock()
msg.topic = 'test_topic'
msg.partition = 0
msg.offset = 1
data_manager.delivery_report(msg)
data_manager.logger.debug.assert_called_once_with(
f'Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}'
)
def test_delivery_error(data_manager):
err = 'Test error'
data_manager.delivery_error(err)
data_manager.logger.error.assert_called_once_with(f'Delivery failed for record : {err}')
@mark.asyncio
async def test_publish(data_manager):
topic = 'test_topic'
data = {'key': 'value'}
# Mock the send method of the Kafka producer
send_mock = MagicMock()
data_manager.kafka_producer.send = send_mock
# Call the publish method
await data_manager.publish(topic, data)
# Check if the send method was called with the correct arguments
send_mock.assert_called_once_with(topic=topic, value=data)
send_mock.return_value.add_callback.assert_called_once()
data_manager.kafka_producer.flush.assert_called_once()
def test_publish_no_kafka(data_manager):
data_manager.export_to_kafka = False
topic = 'test_topic'
data = {'key': 'value'}
data_manager.publish(topic, data)
data_manager.kafka_producer.send.assert_not_called()
@patch('ingestor.managers.data_manager.traceback')
@mark.asyncio
async def test_publish_error(traceback, data_manager):
topic = 'test_topic'
data = {'key': 'value', 'name': 'test_tag'}
# Mock the send method of the Kafka producer to raise an exception
send_mock = MagicMock(side_effect=Exception('Test error'))
data_manager.kafka_producer.send = send_mock
data_manager.mongo_repository.insert = AsyncMock()
data_manager.mongo_repository = AsyncMock()
# Call the publish method
await data_manager.publish(topic, data)
# Check if the send method was called with the correct arguments
send_mock.assert_called_once_with(topic=topic, value=data)
# Check if the error was logged
data_manager.send_notification_async.assert_called_once_with(
notification_id=f'KAFKA_PRODUCER_ERROR_{topic}',
message=f'Error publishing message to topic {topic}: Test error',
block='kafka_producer',
level=NotificationLevel.ERROR,
attachment_content=traceback.format_exc.return_value,
data_manager.mongo_repository.insert.assert_called_once_with(
collection_name=topic,
document={**data, 'inserted_at': ANY},
metadata=metadata['metadata'],
)
@mark.asyncio
async def test_publish_error_mongo(data_manager):
data_manager.export_to_kafka = False
data_manager.mongo_repository.insert = AsyncMock(side_effect=Exception('Test error'))
await data_manager.publish('test_topic', {'key': 'value'})

6
tests/unit/managers/test_ingestor_manager.py
View File

@@ -20,14 +20,12 @@ metadata = {
@patch('ingestor.managers.ingestor_manager.ResourceManager')
def ingestor_manager(data_manager_mock, resource_manager_mock):
ingestor = IngestorManager(
kafka_servers='localhost:9092',
redis_data={'host': 'localhost', 'port': 6379},
lease_ttl=60,
heartbeat_ttl=60,
poll_interval=5,
mongo_connection_string='mongodb://localhost:27017',
mongo_database='sientia',
export_to_kafka=False,
logger=MagicMock(),
notification_handler=MagicMock(),
metadata=metadata['metadata'],
@@ -49,14 +47,12 @@ def test___init__(
notification_handler_mock, resource_manager_mock, data_manager_mock, opc_manager_mock
):
ingestor = IngestorManager(
kafka_servers='localhost:9092',
redis_data={'host': 'localhost', 'port': 6379},
lease_ttl=60,
heartbeat_ttl=60,
poll_interval=5,
mongo_connection_string='mongodb://localhost:27017',
mongo_database='sientia',
export_to_kafka=False,
logger=MagicMock(),
notification_handler=MagicMock(),
metadata=metadata['metadata'],
@@ -65,10 +61,8 @@ def test___init__(
opc_manager_mock.assert_not_called()
data_manager_mock.assert_called_once_with(
kafka_servers='localhost:9092',
mongo_connection_string='mongodb://localhost:27017',
mongo_database='sientia',
export_to_kafka=False,
metadata=metadata['metadata'],
logger=ingestor.logger,
notification_handler=ingestor.notification_handler,

6
tests/unit/test_ingestor.py
View File

@@ -9,8 +9,6 @@ from ingestor.ingestor import Ingestor
@patch('ingestor.ingestor.NotificationHandler')
def test___init__(notification_handler, getenv):
getenv.side_effect = [
'localhost:9092,localhost:35', # KAFKA_SERVERS
'true', # EXPORT_TO_KAFKA
'localhost', # REDIS_HOST
'63790', # REDIS_PORT
'user', # REDIS_USERNAME
@@ -27,7 +25,6 @@ def test___init__(notification_handler, getenv):
ingestor = Ingestor()
getenv.assert_any_call('KAFKA_SERVERS', 'localhost:9092')
getenv.assert_any_call('REDIS_HOST', 'localhost')
getenv.assert_any_call('REDIS_PORT', '6379')
getenv.assert_any_call('REDIS_USERNAME', None)
@@ -37,7 +34,6 @@ def test___init__(notification_handler, getenv):
getenv.assert_any_call('HOSTNAME', 'localhost')
getenv.assert_any_call('POLL_INTERVAL', '5')
assert ingestor.kafka_servers == ['localhost:9092', 'localhost:35']
assert ingestor.redis_host == 'localhost'
assert ingestor.redis_port == 63790
assert ingestor.redis_username == 'user'
@@ -124,7 +120,6 @@ async def test_prepare_ingestor(ingestor_manager_mock, ingestor):
await ingestor.prepare_ingestor()
ingestor_manager_mock.assert_called_once_with(
kafka_servers=','.join(ingestor.kafka_servers),
redis_data={
'host': ingestor.redis_host,
'port': ingestor.redis_port,
@@ -139,7 +134,6 @@ async def test_prepare_ingestor(ingestor_manager_mock, ingestor):
metadata=ingestor.metadata,
logger=ingestor.logger,
notification_handler=ingestor.notification_handler,
export_to_kafka=ingestor.export_to_kafka,
metrics_controller=ingestor.metrics_controller,
)
ingestor_manager.declare_active.assert_called_once()

24
tests/unit/test_metrics.py
View File

@@ -183,30 +183,6 @@ def test_opc_reconnections_total():
assert set(metrics.OPC_RECONNECTIONS_TOTAL._labelnames) == {'pod_id', 'server_name'}
def test_kafka_messages_sent():
"""Verify the definition of KAFKA_MESSAGES_SENT."""
assert metrics.KAFKA_MESSAGES_SENT is not None
assert isinstance(metrics.KAFKA_MESSAGES_SENT, Counter)
assert metrics.KAFKA_MESSAGES_SENT._name == 'kafka_messages_sent' # REMOVED _total
assert set(metrics.KAFKA_MESSAGES_SENT._labelnames) == {'pod_id', 'topic'}
def test_kafka_messages_errors():
"""Verify the definition of KAFKA_MESSAGES_ERRORS."""
assert metrics.KAFKA_MESSAGES_ERRORS is not None
assert isinstance(metrics.KAFKA_MESSAGES_ERRORS, Counter)
assert metrics.KAFKA_MESSAGES_ERRORS._name == 'kafka_messages_errors' # REMOVED _total
assert set(metrics.KAFKA_MESSAGES_ERRORS._labelnames) == {'pod_id', 'topic'}
def test_kafka_connection_status():
"""Verify the definition of KAFKA_CONNECTION_STATUS."""
assert metrics.KAFKA_CONNECTION_STATUS is not None
assert isinstance(metrics.KAFKA_CONNECTION_STATUS, Gauge)
assert metrics.KAFKA_CONNECTION_STATUS._name == 'kafka_connection_status'
assert set(metrics.KAFKA_CONNECTION_STATUS._labelnames) == {'pod_id'}
def test_notifications_sent():
"""Verify the definition of NOTIFICATIONS_SENT."""
assert metrics.NOTIFICATIONS_SENT is not None