gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1110

Browse Source 
Refactor Activities class to remove Kafka and Druid dependencies, simplifying initialization. Update values.yaml to set replica count to 1 for reduced resource usage. Adjust Redis activity to set TTL to None for better data retention. Remove unused Kafka and Druid activity files and their associated tests, streamlining the codebase.
This commit is contained in:
vitor-aignosi
2025-07-03 16:25:55 -03:00
parent 1dcba6b48e
commit d08d1b1337
13 changed files with 105 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
tests/activities/test_activities.py
View File

@@ -2,16 +2,16 @@ from unittest.mock import patch, MagicMock, ANY
from pytest import mark
from sientia_do.temporal.activities.postgres import Postgres
from scouter.activities.activities import Activities
from scouter.activities.mongodb import MongoDB
from scouter.activities.redis import Redis
from scouter.activities.kafka import Kafka
from scouter.activities.gates import Gates
@patch('scouter.activities.activities.MongoDB.__init__')
@patch('scouter.activities.activities.Postgres.__init__')
@patch('scouter.activities.activities.Redis.__init__')
@patch('scouter.activities.activities.Kafka.__init__')
@patch('scouter.activities.activities.Gates.__init__')
def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgres_init):
def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mongodb_init):
postgres_config = {
'host': 'localhost',
@@ -30,10 +30,9 @@ def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgr
'password': 'redis'
}
kafka_config = {
'bootstrap_servers': 'localhost:9092',
'polling_time': 1000,
'group_id': 'test-group'
mongodb_config = {
'connection_string': 'mongodb://localhost:27017',
'database_name': 'test_database'
}
logger = MagicMock()
@@ -42,7 +41,7 @@ def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgr
activities = Activities(
postgres_config=postgres_config,
redis_config=redis_config,
kafka_config=kafka_config,
mongodb_config=mongodb_config,
logger=logger,
notification_handler=notification_handler
)
@@ -50,7 +49,7 @@ def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgr
assert isinstance(activities, Activities)
assert isinstance(activities, Postgres)
assert isinstance(activities, Redis)
assert isinstance(activities, Kafka)
assert isinstance(activities, MongoDB)
assert isinstance(activities, Gates)
mock_postgres_init.assert_called_once_with(
@@ -76,11 +75,10 @@ def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgr
notification_handler=notification_handler
)
mock_kafka_init.assert_called_once_with(
mock_mongodb_init.assert_called_once_with(
ANY,
bootstrap_servers=kafka_config['bootstrap_servers'],
polling_time=kafka_config['polling_time'],
group_id=kafka_config['group_id'],
connection_string=mongodb_config['connection_string'],
database_name=mongodb_config['database_name'],
logger=logger,
notification_handler=notification_handler
)
@@ -94,12 +92,12 @@ def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgr
@patch('scouter.activities.activities.Postgres.__init__')
@patch('scouter.activities.activities.Redis.__init__')
@patch('scouter.activities.activities.Kafka.__init__')
@patch('scouter.activities.activities.Gates.__init__')
@patch('scouter.activities.activities.MongoDB.__init__')
@patch('scouter.activities.activities.Postgres.close')
@patch('scouter.activities.activities.Kafka.close')
def test_shutdown(mock_kafka_close, mock_postgres_close,
_mock_gates_init, _mock_redis_init, _mock_kafka_init, _mock_postgres_init):
@patch('scouter.activities.activities.MongoDB.shutdown')
def test_shutdown(mock_mongodb_close, mock_postgres_close, _mock_mongodb_init,
_mock_gates_init, _mock_redis_init, _mock_postgres_init):
postgres_config = {
'host': 'localhost',
'port': 5432,
@@ -117,10 +115,9 @@ def test_shutdown(mock_kafka_close, mock_postgres_close,
'password': 'redis'
}
kafka_config = {
'bootstrap_servers': 'localhost:9092',
'polling_time': 1000,
'group_id': 'test-group'
mongodb_config = {
'connection_string': 'mongodb://localhost:27017',
'database_name': 'test_database'
}
logger = MagicMock()
@@ -129,7 +126,7 @@ def test_shutdown(mock_kafka_close, mock_postgres_close,
activities = Activities(
postgres_config=postgres_config,
redis_config=redis_config,
kafka_config=kafka_config,
mongodb_config=mongodb_config,
logger=logger,
notification_handler=notification_handler
)
@@ -137,4 +134,4 @@ def test_shutdown(mock_kafka_close, mock_postgres_close,
activities.shutdown()
mock_postgres_close.assert_called_once()
mock_kafka_close.assert_called_once()
mock_mongodb_close.assert_called_once()

4
tests/activities/test_gates.py
View File

@@ -290,8 +290,8 @@ async def test_aggregate_data(gates_fixture):
'value': None, 'timestamp': '2023-01-04'},
],
'model_tags': {
'name1': {'aggr_function': 'avg'},
'name2': {'aggr_function': 'max'},
'name1': {'aggr_func': 'avg'},
'name2': {'aggr_func': 'max'},
},
**metadata
}

92
tests/activities/test_kafka.py
View File

@@ -1,92 +0,0 @@
from unittest.mock import MagicMock, patch, ANY
from pytest import fixture, mark
from pandas import DataFrame
from scouter.activities.kafka import Kafka
@fixture
@patch("scouter.activities.kafka.KafkaConsumer")
def kafka(_kafka_consumer):
return Kafka(
bootstrap_servers="localhost:9092",
polling_time=1000,
group_id="test-group",
logger=MagicMock(),
notification_handler=MagicMock()
)
@patch("scouter.activities.kafka.KafkaConsumer")
def test___init__(kafka_consumer):
kafka = Kafka(
bootstrap_servers="localhost:9092",
polling_time=1000,
group_id="test-group",
logger=MagicMock(),
notification_handler=MagicMock()
)
assert kafka.polling_time == 1000
assert kafka.kafka_connector == kafka_consumer.return_value
kafka_consumer.assert_called_once_with(
bootstrap_servers="localhost:9092",
auto_offset_reset="earliest",
enable_auto_commit=True,
group_id="test-group",
value_deserializer=ANY
)
metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
}
}
@mark.asyncio
async def test_load_from_kafka(kafka):
input_data = {"topic": "test-topic", **metadata}
data = [
("test-topic", [
MagicMock(
value=f"test-value-{i}"
) for i in range(10)
])
]
kafka.kafka_connector.poll.return_value = MagicMock(
items=MagicMock(return_value=data)
)
expected = DataFrame([d.value for d in data[0][1]]).to_dict()
result = await kafka.load_from_kafka(input_data)
assert result == expected
kafka.kafka_connector.subscribe.assert_called_once_with(["test-topic"])
kafka.kafka_connector.poll.assert_called_once_with(timeout_ms=1000)
@mark.asyncio
async def test_load_from_kafka_empty(kafka):
input_data = {"topic": "test-topic", **metadata}
kafka.kafka_connector.poll.return_value = MagicMock(
items=MagicMock(return_value=[])
)
result = await kafka.load_from_kafka(input_data)
assert result == {}
kafka.kafka_connector.subscribe.assert_called_once_with(["test-topic"])
kafka.kafka_connector.poll.assert_called_once_with(timeout_ms=1000)

4
tests/activities/test_redis.py
View File

@@ -87,7 +87,7 @@ async def test_group_and_hold_data_new_key(redis_activity):
# Verify set was called with correct arguments
redis_activity.set.assert_called_once()
args, kwargs = redis_activity.set.call_args
assert args[0] == 'test_pipeline_test_schedule'
assert args[0] == 'held_data_test_pipeline_test_schedule'
assert args[1] == {
'sensor1': 25.5,
'sensor2': 30.0,
@@ -139,7 +139,7 @@ async def test_group_and_hold_data_update_existing(redis_activity):
# Verify set was called with correct arguments
redis_activity.set.assert_called_once()
args, kwargs = redis_activity.set.call_args
assert args[0] == 'test_workflow_test_schedule'
assert args[0] == 'held_data_test_workflow_test_schedule'
assert args[1] == {
'sensor1': 25.5,
'sensor2': 28.0,

16
tests/workflow/sub_workflows/test_core_scouter.py
View File

@@ -16,6 +16,14 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
'filtered_data', 'grouped_data', 'held_data']
await core_scouter.run(
input_data={
'metadata': {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
}
},
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'model_name': 'test_model',
@@ -97,6 +105,14 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
mock_workflow.execute_local_activity_method.return_value = {}
await core_scouter.run(
input_data={
'metadata': {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
}
},
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'model_name': 'test_model',

70
tests/workflow/test_scouter.py
View File

@@ -1,4 +1,4 @@
from unittest.mock import AsyncMock, patch, ANY
from unittest.mock import AsyncMock, patch, ANY, call
from pytest import fixture, mark
from scouter.workflow.scouter import Scouter
from scouter.activities.activities import Activities
@@ -13,7 +13,10 @@ def scouter():
@patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock)
async def test_scouter_workflow(mock_workflow, scouter):
mock_workflow.execute_activity_method.return_value = 'test_data'
mock_workflow.execute_local_activity_method.side_effect = [
'test_last_data_timestamp',
'test_data'
]
await scouter.run(
input_data={
'topic': 'test_topic',
@@ -32,11 +35,41 @@ async def test_scouter_workflow(mock_workflow, scouter):
}
}
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.get_last_data_timestamp,
{
**expected_metadata,
'workflow_name': 'scouter',
'schedule_name': 'test_schedule'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.load_latest_data,
{
**expected_metadata,
'collection_name': "raw_test_schedule",
'last_data_timestamp': 'test_last_data_timestamp'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
mock_workflow.execute_activity_method.assert_called_once_with(
Activities.load_from_kafka,
Activities.put_last_data_timestamp,
{
**expected_metadata,
'topic': 'test_topic'
'data': 'test_data',
'workflow_name': 'scouter',
'schedule_name': 'test_schedule'
},
retry_policy=ANY,
start_to_close_timeout=ANY
@@ -45,6 +78,7 @@ async def test_scouter_workflow(mock_workflow, scouter):
mock_workflow.execute_child_workflow.assert_called_once_with(
'core_scouter',
{
'metadata': expected_metadata,
'topic': 'test_topic',
'data': 'test_data',
'workflow_name': 'scouter',
@@ -58,7 +92,10 @@ async def test_scouter_workflow(mock_workflow, scouter):
@mark.asyncio
@patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock)
async def test_scouter_workflow_empty(mock_workflow, scouter):
mock_workflow.execute_activity_method.return_value = {}
mock_workflow.execute_local_activity_method.side_effect = [
'test_last_data_timestamp',
{}
]
await scouter.run(
input_data={
'topic': 'test_topic',
@@ -77,14 +114,19 @@ async def test_scouter_workflow_empty(mock_workflow, scouter):
}
}
mock_workflow.execute_activity_method.assert_called_once_with(
Activities.load_from_kafka,
{
**expected_metadata,
'topic': 'test_topic'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.load_latest_data,
{
**expected_metadata,
'collection_name': "raw_test_schedule",
'last_data_timestamp': 'test_last_data_timestamp'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
mock_workflow.execute_activity_method.assert_not_called()
mock_workflow.execute_child_workflow.assert_not_called()