gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1110

Browse Source 
Update sientia-dataops-library version to 1.2.0 in requirements.txt; refactor logging in activities to use a unified Logger instance and include metadata in log messages across various activities.
This commit is contained in:
vitor-aignosi
2025-06-26 15:15:31 -03:00
parent 9355af1709
commit 4a90b77786
15 changed files with 238 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
tests/activities/test_activities.py
View File

@@ -92,12 +92,14 @@ def test___init__(mock_gates_init, mock_kafka_init, mock_redis_init, mock_postgr
)
@mark.asyncio
@patch('scouter.activities.activities.Postgres.__init__')
@patch('scouter.activities.activities.Redis.__init__')
@patch('scouter.activities.activities.Kafka.__init__')
async def test_prepare_activity(_mock_kafka_init,
_mock_redis_init, _mock_postgres_init):
@patch('scouter.activities.activities.Gates.__init__')
@patch('scouter.activities.activities.Postgres.close')
@patch('scouter.activities.activities.Kafka.close')
def test_shutdown(mock_kafka_close, mock_postgres_close,
_mock_gates_init, _mock_redis_init, _mock_kafka_init, _mock_postgres_init):
postgres_config = {
'host': 'localhost',
'port': 5432,
@@ -132,20 +134,7 @@ async def test_prepare_activity(_mock_kafka_init,
notification_handler=notification_handler
)
input_data = {
'workflow_name': 'test-workflow-name',
'schedule_name': 'test-schedule-name',
'model_name': 'test-model-name',
'model_id': 'test-model-id'
}
activities.shutdown()
await activities.prepare_activity(input_data)
assert activities.notification_handler.base_notification.pipeline == input_data[
'workflow_name']
assert activities.notification_handler.base_notification.trigger == input_data[
'schedule_name']
assert activities.notification_handler.base_notification.model_name == input_data[
'model_name']
assert activities.notification_handler.base_notification.model_id == input_data[
'model_id']
mock_postgres_close.assert_called_once()
mock_kafka_close.assert_called_once()

24
tests/activities/test_faker.py
View File

@@ -1,4 +1,3 @@
from logging import Logger
from unittest.mock import MagicMock, patch, call
import pytest
from sientia_do.notifications.handlers import NotificationHandler
@@ -22,7 +21,7 @@ def mock_datetime():
@pytest.fixture
def faker_instance(mock_kafka_producer):
logger = MagicMock(spec=Logger)
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
return Faker(
bootstrap_servers='localhost:9092',
@@ -37,6 +36,15 @@ async def test_faker_init(faker_instance, mock_kafka_producer):
assert faker_instance.producer is not None
assert len(faker_instance.tags) == 6
metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
}
}
@pytest.mark.asyncio
async def test_generate_and_send_data_default_count(faker_instance,
@@ -56,7 +64,7 @@ async def test_generate_and_send_data_default_count(faker_instance,
]
# Call the method
await faker_instance.generate_and_send_data({'topic': 'test_topic'})
await faker_instance.generate_and_send_data({'topic': 'test_topic', **metadata})
# Verify the producer was called 3 times (default count)
assert mock_kafka_producer.send.call_count == 3
@@ -92,7 +100,8 @@ async def test_generate_and_send_data_custom_count(faker_instance, mock_kafka_pr
# Call the method with custom count
await faker_instance.generate_and_send_data({
'topic': 'test_topic',
'num_messages': 2
'num_messages': 2,
**metadata
})
# Verify the producer was called 2 times
@@ -104,7 +113,7 @@ async def test_generate_and_send_data_custom_count(faker_instance, mock_kafka_pr
async def test_generate_and_send_data_no_topic(faker_instance):
"""Test that ValueError is raised when no topic is provided"""
with pytest.raises(ValueError, match="Topic must be specified in input_data"):
await faker_instance.generate_and_send_data({})
await faker_instance.generate_and_send_data({**metadata})
@pytest.mark.asyncio
@@ -112,7 +121,7 @@ async def test_generate_and_send_data_no_topic(faker_instance):
async def test_generate_and_send_data_random_values(_random, faker_instance, mock_kafka_producer):
"""Test that random values are within expected ranges"""
# Call the method
await faker_instance.generate_and_send_data({'topic': 'test_topic'})
await faker_instance.generate_and_send_data({'topic': 'test_topic', **metadata})
# Get the call arguments
call_args = mock_kafka_producer.send.call_args[1]['value']
@@ -132,7 +141,8 @@ async def test_generate_and_send_data_generate_null_values(
mock_kafka_producer):
# Call the method
await faker_instance.generate_and_send_data({'topic': 'test_topic',
'num_messages': 1})
'num_messages': 1,
**metadata})
# Get the call arguments
call_args = mock_kafka_producer.send.call_args[1]['value']

47
tests/activities/test_gates.py
View File

@@ -14,6 +14,16 @@ def gates_fixture():
return Gates(logger=logger, notification_handler=notification_handler)
metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
}
}
@pytest.mark.asyncio
async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture):
"""Test data_quality_gate with NULL_VALUES_FILTER and DISCARD policy."""
@@ -31,7 +41,8 @@ async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture):
'tag1': {'data_range': [0, 100]},
'tag2': {'data_range': [0, 100]},
'tag3': {'data_range': [0, 100]}
}
},
**metadata
}
# Execute
@@ -60,7 +71,8 @@ async def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture):
'tag1': {'data_range': [0, 100]},
'tag2': {'data_range': [0, 100]},
'tag3': {'data_range': [0, 100]}
}
},
**metadata
}
# Mock the out_of_bounds_filter to return rows with out of bounds values
@@ -95,7 +107,8 @@ async def test_data_quality_gate_with_multiple_filters(gates_fixture):
'tag2': {'data_range': [0, 100]},
'tag3': {'data_range': [0, 100]},
'tag4': {'data_range': [0, 100]}
}
},
**metadata
}
result = await gates_fixture.data_quality_gate(input_data)
@@ -111,6 +124,7 @@ async def test_data_quality_gate_with_multiple_filters(gates_fixture):
async def test_data_quality_gate_with_unknown_filter(gates_fixture):
"""Test data_quality_gate with an unknown filter."""
# Setup test data with unknown filter
gates_fixture.warning = MagicMock()
input_data = {
'filters': {
'UNKNOWN_FILTER': 'DISCARD'
@@ -123,7 +137,8 @@ async def test_data_quality_gate_with_unknown_filter(gates_fixture):
},
'model_tags': {
'tag1': {'data_range': [0, 100]}
}
},
**metadata
}
# Execute
@@ -131,8 +146,10 @@ async def test_data_quality_gate_with_unknown_filter(gates_fixture):
# Verify data is unchanged and warning is logged
assert len(result['tag']) == 1
gates_fixture.logger.warning.assert_called_once_with(
"Filter UNKNOWN_FILTER not found")
gates_fixture.warning.assert_called_once_with(
"Filter UNKNOWN_FILTER not found",
metadata=metadata['metadata']
)
@pytest.mark.asyncio
@@ -151,7 +168,8 @@ async def test_data_quality_gate_with_filter_error(gates_fixture):
},
'model_tags': {
'tag1': {'data_range': [0, 100]}
}
},
**metadata
}
# Mock the filter to raise an exception
@@ -187,7 +205,8 @@ async def test_data_quality_gate_with_empty_data(gates_fixture):
'value': [],
'timestamp': []
},
'model_tags': {}
'model_tags': {},
**metadata
}
# Execute
@@ -212,7 +231,8 @@ async def test_data_quality_gate_with_no_filters(gates_fixture):
},
'model_tags': {
'tag1': {'data_range': [0, 100]}
}
},
**metadata
}
# Execute
@@ -272,7 +292,8 @@ async def test_aggregate_data(gates_fixture):
'model_tags': {
'name1': {'aggr_function': 'avg'},
'name2': {'aggr_function': 'max'},
}
},
**metadata
}
# Expected result
@@ -309,7 +330,8 @@ async def test_aggregate_data_with_continue(gates_fixture):
'model_tags': {
'name1': {'aggr_function': 'avg'},
'name2': {'aggr_function': 'max'},
}
},
**metadata
}
# Expected result
@@ -343,7 +365,8 @@ async def test_aggregate_data_raise_exception(gates_fixture):
'model_tags': {
'name1': {'aggr_function': 'avg'},
'name2': {'aggr_function': 'max'},
}
},
**metadata
}
try:

14
tests/activities/test_kafka.py
View File

@@ -38,9 +38,19 @@ def test___init__(kafka_consumer):
)
metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
}
}
@mark.asyncio
async def test_load_from_kafka(kafka):
input_data = {"topic": "test-topic"}
input_data = {"topic": "test-topic", **metadata}
data = [
("test-topic", [
@@ -67,7 +77,7 @@ async def test_load_from_kafka(kafka):
@mark.asyncio
async def test_load_from_kafka_empty(kafka):
input_data = {"topic": "test-topic"}
input_data = {"topic": "test-topic", **metadata}
kafka.kafka_connector.poll.return_value = MagicMock(
items=MagicMock(return_value=[])

14
tests/activities/test_redis.py
View File

@@ -41,11 +41,22 @@ def test_redis_initialization(mock_redis_init):
)
metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
}
}
@pytest.mark.asyncio
async def test_group_and_hold_data_new_key(redis_activity):
"""Test group_and_hold_data with a new key"""
# Setup
test_data = {
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule',
'retention_time': 3600,
@@ -97,6 +108,7 @@ async def test_group_and_hold_data_update_existing(redis_activity):
# New data to update with
test_data = {
**metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'retention_time': 3600,
@@ -142,6 +154,7 @@ async def test_group_and_hold_data_with_none_values(redis_activity):
"""Test handling of None values in group_and_hold_data"""
# Setup test data with None values
test_data = {
**metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'retention_time': 3600,
@@ -170,6 +183,7 @@ async def test_group_and_hold_data_empty_dataframe(redis_activity):
"""Test group_and_hold_data with empty DataFrame"""
# Setup test with empty data
test_data = {
**metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'retention_time': 3600,

25
tests/workflow/sub_workflows/test_core_scouter.py
View File

@@ -30,10 +30,20 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
}
}
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {}
@@ -45,6 +55,7 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
call(
Activities.aggregate_data,
{
**expected_metadata,
'data': 'filtered_data',
'model_tags': {}
},
@@ -55,6 +66,7 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': 'grouped_data',
@@ -69,6 +81,7 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
call(
Activities.export_data_to_postgres,
{
**expected_metadata,
'schema': 'test_schema',
'table_name': 'test_table',
'data': 'held_data'},
@@ -98,10 +111,20 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
}
}
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {}
@@ -113,6 +136,7 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
call(
Activities.aggregate_data,
{
**expected_metadata,
'data': {},
'model_tags': {}
},
@@ -123,6 +147,7 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': {},

36
tests/workflow/test_scouter.py
View File

@@ -23,21 +23,19 @@ async def test_scouter_workflow(mock_workflow, scouter):
}
)
mock_workflow.execute_local_activity_method.assert_called_once_with(
Activities.prepare_activity,
{
'workflow_name': 'scouter',
'schedule_name': 'test_schedule',
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'model_id': 'test_model_id'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
}
}
mock_workflow.execute_activity_method.assert_called_once_with(
Activities.load_from_kafka,
{
**expected_metadata,
'topic': 'test_topic'
},
retry_policy=ANY,
@@ -70,21 +68,19 @@ async def test_scouter_workflow_empty(mock_workflow, scouter):
}
)
mock_workflow.execute_local_activity_method.assert_called_once_with(
Activities.prepare_activity,
{
'workflow_name': 'scouter',
'schedule_name': 'test_schedule',
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'model_id': 'test_model_id'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
}
}
mock_workflow.execute_activity_method.assert_called_once_with(
Activities.load_from_kafka,
{
**expected_metadata,
'topic': 'test_topic'
},
retry_policy=ANY,