gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1316

Browse Source 
Update .gitignore and refactor metrics.py, activities.py, and gates.py for improved clarity and consistency. Added coverage.xml and cache directories to .gitignore. Standardized string formatting and parameter handling in metrics and activities classes, enhancing code readability. Removed the deprecated faker.py file and adjusted related tests accordingly.
This commit is contained in:
vitor-aignosi
2025-10-16 13:31:12 -03:00
parent 8bdbf049b8
commit 97eb5bc904
27 changed files with 1101 additions and 1336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
tests/activities/test_activities.py
View File

@@ -1,10 +1,11 @@
from unittest.mock import patch, MagicMock, ANY
from pytest import mark
from unittest.mock import ANY, MagicMock, patch
from sientia_do.temporal.activities.postgres import Postgres
from scouter.activities.activities import Activities
from scouter.activities.gates import Gates
from scouter.activities.mongodb import MongoDB
from scouter.activities.redis import Redis
from scouter.activities.gates import Gates
@patch('scouter.activities.activities.MongoDB.__init__')
@@ -12,7 +13,6 @@ from scouter.activities.gates import Gates
@patch('scouter.activities.activities.Redis.__init__')
@patch('scouter.activities.activities.Gates.__init__')
def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mongodb_init):
postgres_config = {
'host': 'localhost',
'port': 5432,
@@ -20,19 +20,14 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
'password': 'postgres',
'dbname': 'postgres',
'min_connections': 1,
'max_connections': 10
'max_connections': 10,
}
redis_config = {
'host': 'localhost',
'port': 6379,
'username': 'redis',
'password': 'redis'
}
redis_config = {'host': 'localhost', 'port': 6379, 'username': 'redis', 'password': 'redis'}
mongodb_config = {
'connection_string': 'mongodb://localhost:27017',
'database_name': 'test_database'
'database_name': 'test_database',
}
logger = MagicMock()
@@ -43,7 +38,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
redis_config=redis_config,
mongodb_config=mongodb_config,
logger=logger,
notification_handler=notification_handler
notification_handler=notification_handler,
)
assert isinstance(activities, Activities)
@@ -62,7 +57,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
min_connections=postgres_config['min_connections'],
max_connections=postgres_config['max_connections'],
logger=logger,
notification_handler=notification_handler
notification_handler=notification_handler,
)
mock_redis_init.assert_called_once_with(
@@ -72,7 +67,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
username=redis_config['username'],
password=redis_config['password'],
logger=logger,
notification_handler=notification_handler
notification_handler=notification_handler,
)
mock_mongodb_init.assert_called_once_with(
@@ -80,13 +75,11 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
connection_string=mongodb_config['connection_string'],
database_name=mongodb_config['database_name'],
logger=logger,
notification_handler=notification_handler
notification_handler=notification_handler,
)
mock_gates_init.assert_called_once_with(
ANY,
logger=logger,
notification_handler=notification_handler
ANY, logger=logger, notification_handler=notification_handler
)
@@ -96,8 +89,14 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
@patch('scouter.activities.activities.MongoDB.__init__')
@patch('scouter.activities.activities.Postgres.close')
@patch('scouter.activities.activities.MongoDB.shutdown')
def test_shutdown(mock_mongodb_close, mock_postgres_close, _mock_mongodb_init,
_mock_gates_init, _mock_redis_init, _mock_postgres_init):
def test_shutdown(
mock_mongodb_close,
mock_postgres_close,
_mock_mongodb_init,
_mock_gates_init,
_mock_redis_init,
_mock_postgres_init,
):
postgres_config = {
'host': 'localhost',
'port': 5432,
@@ -105,19 +104,14 @@ def test_shutdown(mock_mongodb_close, mock_postgres_close, _mock_mongodb_init,
'password': 'postgres',
'dbname': 'postgres',
'min_connections': 1,
'max_connections': 10
'max_connections': 10,
}
redis_config = {
'host': 'localhost',
'port': 6379,
'username': 'redis',
'password': 'redis'
}
redis_config = {'host': 'localhost', 'port': 6379, 'username': 'redis', 'password': 'redis'}
mongodb_config = {
'connection_string': 'mongodb://localhost:27017',
'database_name': 'test_database'
'database_name': 'test_database',
}
logger = MagicMock()
@@ -128,7 +122,7 @@ def test_shutdown(mock_mongodb_close, mock_postgres_close, _mock_mongodb_init,
redis_config=redis_config,
mongodb_config=mongodb_config,
logger=logger,
notification_handler=notification_handler
notification_handler=notification_handler,
)
activities.shutdown()

153
tests/activities/test_faker.py
View File

@@ -1,153 +0,0 @@
from unittest.mock import MagicMock, patch, call
import pytest
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from scouter.activities.faker import Faker
@pytest.fixture
def mock_kafka_producer():
with patch('scouter.activities.faker.KafkaProducer') as mock:
producer = MagicMock()
mock.return_value = producer
yield producer
@pytest.fixture
def mock_datetime():
with patch('scouter.activities.faker.datetime') as mock_dt:
mock_dt.now.return_value.strftime.return_value = '2025-05-14 14:54:24'
yield mock_dt
@pytest.fixture
def faker_instance(mock_kafka_producer):
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
return Faker(
bootstrap_servers='localhost:9092',
logger=logger,
notification_handler=notification_handler
)
@pytest.mark.asyncio
async def test_faker_init(faker_instance, mock_kafka_producer):
"""Test Faker initialization with correct parameters"""
assert faker_instance.producer is not None
assert len(faker_instance.tags) == 6
metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
}
}
@pytest.mark.asyncio
async def test_generate_and_send_data_default_count(faker_instance,
mock_kafka_producer, mock_datetime):
"""Test generating data with default message count"""
# Mock random.choice to control the output
with patch('random.choice') as mock_choice, \
patch('random.uniform', return_value=42.5), \
patch('random.randint', return_value=3), \
patch('random.random', return_value=0.5):
# Setup mock for tag and name selection
mock_choice.side_effect = [
'ns=1;i=1001',
'ns=1;i=1002',
'ns=1;i=1003'
]
# Call the method
await faker_instance.generate_and_send_data({'topic': 'test_topic', **metadata})
# Verify the producer was called 3 times (default count)
assert mock_kafka_producer.send.call_count == 3
mock_kafka_producer.flush.assert_called_once()
# Verify the message format
expected_data = [{
'tag': 'ns=1;i=1001',
'name': 'Temperature Sensor',
'timestamp': '2025-05-14 14:54:24',
'value': 42.5
}, {
'tag': 'ns=1;i=1002',
'name': 'Vibration Meter',
'timestamp': '2025-05-14 14:54:24',
'value': 42.5
}, {
'tag': 'ns=1;i=1003',
'name': 'Pressure Gauge',
'timestamp': '2025-05-14 14:54:24',
'value': 42.5
}]
mock_kafka_producer.send.assert_has_calls([
call('test_topic', value=expected_data[0]),
call('test_topic', value=expected_data[1]),
call('test_topic', value=expected_data[2])
])
@pytest.mark.asyncio
async def test_generate_and_send_data_custom_count(faker_instance, mock_kafka_producer):
"""Test generating data with custom message count"""
# Call the method with custom count
await faker_instance.generate_and_send_data({
'topic': 'test_topic',
'num_messages': 2,
**metadata
})
# Verify the producer was called 2 times
assert mock_kafka_producer.send.call_count == 2
mock_kafka_producer.flush.assert_called_once()
@pytest.mark.asyncio
async def test_generate_and_send_data_no_topic(faker_instance):
"""Test that ValueError is raised when no topic is provided"""
with pytest.raises(ValueError, match="Topic must be specified in input_data"):
await faker_instance.generate_and_send_data({**metadata})
@pytest.mark.asyncio
@patch('scouter.activities.faker.random.random', return_value=0.5)
async def test_generate_and_send_data_random_values(_random, faker_instance, mock_kafka_producer):
"""Test that random values are within expected ranges"""
# Call the method
await faker_instance.generate_and_send_data({'topic': 'test_topic', **metadata})
# Get the call arguments
call_args = mock_kafka_producer.send.call_args[1]['value']
# Verify the data structure
assert 'tag' in call_args
assert call_args['tag'] in faker_instance.tags
assert 'value' in call_args
assert 0 <= call_args['value'] <= 100
@pytest.mark.asyncio
@patch('scouter.activities.faker.random.random', return_value=0.05)
async def test_generate_and_send_data_generate_null_values(
_random_mock,
faker_instance,
mock_kafka_producer):
# Call the method
await faker_instance.generate_and_send_data({'topic': 'test_topic',
'num_messages': 1,
**metadata})
# Get the call arguments
call_args = mock_kafka_producer.send.call_args[1]['value']
assert call_args['value'] is None
assert call_args['tag'] in faker_instance.tags
assert 'name' in call_args
assert 'timestamp' in call_args

201
tests/activities/test_gates.py
View File

@@ -1,8 +1,11 @@
from unittest.mock import Mock, patch, MagicMock, ANY
from typing import Any
from unittest.mock import ANY, MagicMock, Mock, patch
import numpy as np
import pandas as pd
import pytest
from sientia_do.notifications.models import NotificationLevel
from scouter.activities.gates import Gates
@@ -21,7 +24,7 @@ metadata = {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
'workflow_name': 'scouter',
}
}
@@ -31,11 +34,7 @@ async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture):
"""Test data_quality_gate with NULL_VALUES_FILTER and DISCARD policy."""
# Setup test data
input_data = {
'filters': {
'NULL_VALUES_FILTER': {
'policy': 'DISCARD'
}
},
'filters': {'NULL_VALUES_FILTER': {'policy': 'DISCARD'}},
'data': {
'name': ['tag1', 'tag2', 'tag3'],
'tag': ['tag1', 'tag2', 'tag3'],
@@ -45,9 +44,9 @@ async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture):
'model_tags': {
'tag1': {'data_range': [0, 100]},
'tag2': {'data_range': [0, 100]},
'tag3': {'data_range': [0, 100]}
'tag3': {'data_range': [0, 100]},
},
**metadata
**metadata,
}
# Execute
@@ -64,29 +63,26 @@ async def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture):
"""Test data_quality_gate with OUT_OF_BOUNDS_FILTER and KEEP policy."""
# Setup test data with out of bounds values
input_data = {
'filters': {
'OUT_OF_BOUNDS_FILTER': {
'policy': 'KEEP'
}
},
'filters': {'OUT_OF_BOUNDS_FILTER': {'policy': 'KEEP'}},
'data': {
'name': ['tag1', 'tag2', 'tag3'],
'tag': ['tag1', 'tag2', 'tag3'],
'value': [1.0, 200.0, 3.0],
'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03']
'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03'],
},
'model_tags': {
'tag1': {'data_range': [0, 100]},
'tag2': {'data_range': [0, 100]},
'tag3': {'data_range': [0, 100]}
'tag3': {'data_range': [0, 100]},
},
**metadata
**metadata,
}
# Mock the out_of_bounds_filter to return rows with out of bounds values
with patch('scouter.activities.gates.quality_gate_filters', {
'OUT_OF_BOUNDS_FILTER': lambda df: df[df['tag'] == 'tag2']
}):
with patch(
'scouter.activities.gates.quality_gate_filters',
{'OUT_OF_BOUNDS_FILTER': lambda df: df[df['tag'] == 'tag2']},
):
# Execute
result = await gates_fixture.data_quality_gate(input_data)
@@ -101,33 +97,33 @@ async def test_data_quality_gate_with_multiple_filters(gates_fixture):
# Setup test data
input_data = {
'filters': {
'NULL_VALUES_FILTER': {
'policy': 'DISCARD'
},
'OUT_OF_BOUNDS_FILTER': {
'policy': 'DISCARD'
}
'NULL_VALUES_FILTER': {'policy': 'DISCARD'},
'OUT_OF_BOUNDS_FILTER': {'policy': 'DISCARD'},
},
'data': {
'tag': ['tag1', 'tag2', 'tag3', 'tag4'],
'name': ['tag1', 'tag2', 'tag3', 'tag4'],
'value': [1.0, None, 300.0, 4.0],
'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03', '2023-01-04']
'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03', '2023-01-04'],
},
'model_tags': {
'tag1': {'data_range': [0, 100]},
'tag2': {'data_range': [0, 100]},
'tag3': {'data_range': [0, 100]},
'tag4': {'data_range': [0, 100]}
'tag4': {'data_range': [0, 100]},
},
**metadata
**metadata,
}
result = await gates_fixture.data_quality_gate(input_data)
# Verify only tag1 and tag4 remain (tag2 has null, tag3 is out of bounds)
assert result == {'tag': {0: 'tag1', 3: 'tag4'}, 'name': {0: 'tag1', 3: 'tag4'}, 'value': {
0: 1.0, 3: 4.0}, 'timestamp': {0: '2023-01-01', 3: '2023-01-04'}}
assert result == {
'tag': {0: 'tag1', 3: 'tag4'},
'name': {0: 'tag1', 3: 'tag4'},
'value': {0: 1.0, 3: 4.0},
'timestamp': {0: '2023-01-01', 3: '2023-01-04'},
}
# Should be called twice (once for each filter)
assert gates_fixture.send_notification.call_count == 2
@@ -138,21 +134,10 @@ async def test_data_quality_gate_with_unknown_filter(gates_fixture):
# Setup test data with unknown filter
gates_fixture.warning = MagicMock()
input_data = {
'filters': {
'UNKNOWN_FILTER': {
'policy': 'DISCARD'
}
},
'data': {
'tag': ['tag1'],
'name': ['tag1'],
'value': [1.0],
'timestamp': ['2023-01-01']
},
'model_tags': {
'tag1': {'data_range': [0, 100]}
},
**metadata
'filters': {'UNKNOWN_FILTER': {'policy': 'DISCARD'}},
'data': {'tag': ['tag1'], 'name': ['tag1'], 'value': [1.0], 'timestamp': ['2023-01-01']},
'model_tags': {'tag1': {'data_range': [0, 100]}},
**metadata,
}
# Execute
@@ -161,8 +146,7 @@ async def test_data_quality_gate_with_unknown_filter(gates_fixture):
# Verify data is unchanged and warning is logged
assert len(result['tag']) == 1
gates_fixture.warning.assert_called_once_with(
"Filter UNKNOWN_FILTER not found",
metadata=metadata['metadata']
'Filter UNKNOWN_FILTER not found', metadata=metadata['metadata']
)
@@ -171,30 +155,19 @@ async def test_data_quality_gate_with_filter_error(gates_fixture):
"""Test data_quality_gate when a filter raises an exception."""
# Setup test data
input_data = {
'filters': {
'NULL_VALUES_FILTER': {
'policy': 'DISCARD'
}
},
'data': {
'tag': ['tag1'],
'name': ['tag1'],
'value': [1.0],
'timestamp': ['2023-01-01']
},
'model_tags': {
'tag1': {'data_range': [0, 100]}
},
**metadata
'filters': {'NULL_VALUES_FILTER': {'policy': 'DISCARD'}},
'data': {'tag': ['tag1'], 'name': ['tag1'], 'value': [1.0], 'timestamp': ['2023-01-01']},
'model_tags': {'tag1': {'data_range': [0, 100]}},
**metadata,
}
# Mock the filter to raise an exception
def failing_filter(_, _model_tags):
raise ValueError("Filter error")
raise ValueError('Filter error')
with patch('scouter.activities.gates.quality_gate_filters', {
'NULL_VALUES_FILTER': failing_filter
}):
with patch(
'scouter.activities.gates.quality_gate_filters', {'NULL_VALUES_FILTER': failing_filter}
):
# Execute
result = await gates_fixture.data_quality_gate(input_data)
@@ -202,9 +175,9 @@ async def test_data_quality_gate_with_filter_error(gates_fixture):
assert len(result['tag']) == 1
gates_fixture.send_notification.assert_called_once()
call_args = gates_fixture.send_notification.call_args[1]
assert call_args['notification_id'] == "DATA_QUALITY_GATE_ISSUES"
assert call_args['notification_id'] == 'DATA_QUALITY_GATE_ISSUES'
assert call_args['level'] == NotificationLevel.ERROR
assert "Filter error" in call_args['message']
assert 'Filter error' in call_args['message']
@pytest.mark.asyncio
@@ -212,19 +185,10 @@ async def test_data_quality_gate_with_empty_data(gates_fixture):
"""Test data_quality_gate with empty input data."""
# Setup empty input data
input_data = {
'filters': {
'NULL_VALUES_FILTER': {
'policy': 'DISCARD'
}
},
'data': {
'tag': [],
'name': [],
'value': [],
'timestamp': []
},
'filters': {'NULL_VALUES_FILTER': {'policy': 'DISCARD'}},
'data': {'tag': [], 'name': [], 'value': [], 'timestamp': []},
'model_tags': {},
**metadata
**metadata,
}
# Execute
@@ -241,16 +205,9 @@ async def test_data_quality_gate_with_no_filters(gates_fixture):
# Setup test data with no filters
input_data = {
'filters': {},
'data': {
'tag': ['tag1'],
'name': ['tag1'],
'value': [1.0],
'timestamp': ['2023-01-01']
},
'model_tags': {
'tag1': {'data_range': [0, 100]}
},
**metadata
'data': {'tag': ['tag1'], 'name': ['tag1'], 'value': [1.0], 'timestamp': ['2023-01-01']},
'model_tags': {'tag1': {'data_range': [0, 100]}},
**metadata,
}
# Execute
@@ -262,7 +219,7 @@ async def test_data_quality_gate_with_no_filters(gates_fixture):
@pytest.mark.parametrize(
"group_data, aggr_function, expected_result",
'group_data, aggr_function, expected_result',
[
# Single value case
(pd.DataFrame({'value': [10.0]}), 'avg', 10.0),
@@ -273,18 +230,16 @@ async def test_data_quality_gate_with_no_filters(gates_fixture):
(pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'min', 1.0),
(pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'lts', 4.0),
# With NaN values
(pd.DataFrame({'value': [1.0, np.nan, 3.0, 4.0]}),
'avg', 2.6666666666666665),
(pd.DataFrame({'value': [1.0, np.nan, 3.0, 4.0]}), 'avg', 2.6666666666666665),
# Empty group after dropping NaN
(pd.DataFrame({'value': [np.nan, np.nan]}), 'avg', None),
# Invalid aggregation function
(pd.DataFrame({'value': [1.0, 2.0]}), 'invalid', 'continue'),
]
],
)
def test_apply_aggregation(gates_fixture, group_data, aggr_function, expected_result):
"""Test apply_aggregation method with various scenarios."""
result = gates_fixture.apply_aggregation(
group_data, aggr_function, metadata)
result = gates_fixture.apply_aggregation(group_data, aggr_function, metadata)
assert result == expected_result
# Check notification was sent for invalid function
@@ -305,22 +260,23 @@ async def test_aggregate_data(gates_fixture):
{'tag': 'tag2', 'name': 'name2', 'value': 4.0, 'timestamp': '2023-01-01'},
{'tag': 'tag2', 'name': 'name2', 'value': 5.0, 'timestamp': '2023-01-02'},
{'tag': 'tag2', 'name': 'name2', 'value': 6.0, 'timestamp': '2023-01-03'},
{'tag': 'tag1', 'name': 'name1',
'value': None, 'timestamp': '2023-01-04'},
{'tag': 'tag1', 'name': 'name1', 'value': None, 'timestamp': '2023-01-04'},
],
'model_tags': {
'name1': {'aggr_func': 'avg'},
'name2': {'aggr_func': 'max'},
},
**metadata
**metadata,
}
# Expected result
expected_result = {'tag': {0: 'tag1', 1: 'tag2'},
'name': {0: 'name1', 1: 'name2'},
'value': {0: 2.0, 1: 6.0},
'timestamp': {0: '2023-01-04', 1: '2023-01-03'},
'aggregation_function': {0: 'avg', 1: 'max'}}
expected_result = {
'tag': {0: 'tag1', 1: 'tag2'},
'name': {0: 'name1', 1: 'name2'},
'value': {0: 2.0, 1: 6.0},
'timestamp': {0: '2023-01-04', 1: '2023-01-03'},
'aggregation_function': {0: 'avg', 1: 'max'},
}
# Execute
result = await gates_fixture.aggregate_data(input_data)
@@ -332,7 +288,6 @@ async def test_aggregate_data(gates_fixture):
@pytest.mark.asyncio
async def test_aggregate_data_with_continue(gates_fixture):
gates_fixture.apply_aggregation = MagicMock(return_value='continue')
input_data = {
@@ -343,18 +298,17 @@ async def test_aggregate_data_with_continue(gates_fixture):
{'tag': 'tag2', 'name': 'name2', 'value': 4.0, 'timestamp': '2023-01-01'},
{'tag': 'tag2', 'name': 'name2', 'value': 5.0, 'timestamp': '2023-01-02'},
{'tag': 'tag2', 'name': 'name2', 'value': 6.0, 'timestamp': '2023-01-03'},
{'tag': 'tag1', 'name': 'name1',
'value': None, 'timestamp': '2023-01-04'},
{'tag': 'tag1', 'name': 'name1', 'value': None, 'timestamp': '2023-01-04'},
],
'model_tags': {
'name1': {'aggr_function': 'avg'},
'name2': {'aggr_function': 'max'},
},
**metadata
**metadata,
}
# Expected result
expected_result = {}
expected_result: dict[str, Any] = {}
# Execute
result = await gates_fixture.aggregate_data(input_data)
@@ -366,9 +320,7 @@ async def test_aggregate_data_with_continue(gates_fixture):
@pytest.mark.asyncio
async def test_aggregate_data_raise_exception(gates_fixture):
gates_fixture.apply_aggregation = MagicMock(
side_effect=Exception("Test exception"))
gates_fixture.apply_aggregation = MagicMock(side_effect=Exception('Test exception'))
input_data = {
'data': [
@@ -378,42 +330,39 @@ async def test_aggregate_data_raise_exception(gates_fixture):
{'tag': 'tag2', 'name': 'name2', 'value': 4.0, 'timestamp': '2023-01-01'},
{'tag': 'tag2', 'name': 'name2', 'value': 5.0, 'timestamp': '2023-01-02'},
{'tag': 'tag2', 'name': 'name2', 'value': 6.0, 'timestamp': '2023-01-03'},
{'tag': 'tag1', 'name': 'name1',
'value': None, 'timestamp': '2023-01-04'},
{'tag': 'tag1', 'name': 'name1', 'value': None, 'timestamp': '2023-01-04'},
],
'model_tags': {
'name1': {'aggr_function': 'avg'},
'name2': {'aggr_function': 'max'},
},
**metadata
**metadata,
}
try:
await gates_fixture.aggregate_data(input_data)
except Exception as e:
assert str(e) == "Test exception"
assert str(e) == 'Test exception'
gates_fixture.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id="AGGREGATION_ISSUES",
message="Error aggregating data: Test exception",
block="aggregate_data",
notification_id='AGGREGATION_ISSUES',
message='Error aggregating data: Test exception',
block='aggregate_data',
level=NotificationLevel.ERROR,
attachment_content=ANY
attachment_content=ANY,
)
else:
assert False
raise AssertionError('Exception not raised')
@pytest.mark.asyncio
@patch('scouter.activities.gates.metrics')
async def test_write_metrics(mock_metrics, gates_fixture):
"""Test write_metrics method."""
input_data = {
'metadata': metadata['metadata']
}
input_data = {'metadata': metadata['metadata']}
await gates_fixture.write_metrics(input_data)
mock_metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels.assert_called_once_with(
pod_id=gates_fixture.pod_id,
model_name=metadata['metadata']['model_name'],
pipeline_name=metadata['metadata']['workflow_name']
pipeline_name=metadata['metadata']['workflow_name'],
)

117
tests/activities/test_mongo.py
View File

@@ -1,8 +1,10 @@
from datetime import datetime
from unittest.mock import ANY, MagicMock, call, patch
from unittest.mock import ANY, MagicMock, patch
from pytest import fixture, mark
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
from scouter.activities.mongodb import MongoDB, clear_mongo_id
@@ -10,15 +12,9 @@ def test_clear_mongo_id():
"""Test clear_mongo_id"""
data = [
{'_id': '1', 'name': 'test1'},
{'_id': '2', 'name': [{
'_id': '3',
'name': 'test3'
}]},
{'_id': '4', 'name': {
'_id': '5',
'name': 'test2'
}},
[{'_id': '6', 'name': 'test2'}]
{'_id': '2', 'name': [{'_id': '3', 'name': 'test3'}]},
{'_id': '4', 'name': {'_id': '5', 'name': 'test2'}},
[{'_id': '6', 'name': 'test2'}],
]
result = clear_mongo_id(data)
@@ -27,7 +23,7 @@ def test_clear_mongo_id():
{'name': 'test1'},
{'name': [{'name': 'test3'}]},
{'name': {'name': 'test2'}},
[{'name': 'test2'}]
[{'name': 'test2'}],
]
@@ -38,18 +34,16 @@ def test_mongodb___init__(mock_mongo_client):
connection_string='mongodb://localhost:27017',
database_name='test_db',
logger=MagicMock(),
notification_handler=MagicMock()
notification_handler=MagicMock(),
)
mock_mongo_client.assert_called_once_with(
'mongodb://localhost:27017',
serverSelectionTimeoutMS=5000
'mongodb://localhost:27017', serverSelectionTimeoutMS=5000
)
mock_mongo_client.return_value.server_info.assert_called_once()
mock_mongo_client.return_value.__getitem__.assert_called_once_with(
'test_db')
mock_mongo_client.return_value.__getitem__.assert_called_once_with('test_db')
assert mongo.client is not None
assert mongo.database is not None
@@ -63,7 +57,7 @@ def mongodb_activity(mock_mongo_client):
connection_string='mongodb://localhost:27017',
database_name='test_db',
logger=MagicMock(),
notification_handler=MagicMock()
notification_handler=MagicMock(),
)
return mongo
@@ -96,34 +90,27 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
'name': 'test1',
'value': 1,
'inserted_at': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ)
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
}
]
result = await mongodb_activity.load_latest_data({
'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
'collection_name': 'test_collection',
'last_data_timestamp': None
})
mongodb_activity.database.__getitem__.assert_called_once_with(
'test_collection')
collection.find.assert_called_once_with(
{},
{"_id": 0}
result = await mongodb_activity.load_latest_data(
{
'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
'collection_name': 'test_collection',
'last_data_timestamp': None,
}
)
mongodb_activity.database.__getitem__.assert_called_once_with('test_collection')
collection.find.assert_called_once_with({}, {'_id': 0})
assert result == {
'name': {
0: 'test1'
},
'value': {
0: 1
},
'inserted_at': {
0: '2023-01-01 12:00:00.000000+0000'
}
'name': {0: 'test1'},
'value': {0: 1},
'inserted_at': {0: '2023-01-01 12:00:00.000000+0000'},
}
@@ -138,39 +125,36 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
'name': 'test1',
'value': 1,
'inserted_at': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ)
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
}
]
result = await mongodb_activity.load_latest_data({
'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
'collection_name': 'test_collection',
'last_data_timestamp': '2023-01-01 12:00:00.000000+0000'
})
result = await mongodb_activity.load_latest_data(
{
'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
'collection_name': 'test_collection',
'last_data_timestamp': '2023-01-01 12:00:00.000000+0000',
}
)
mongodb_activity.database.__getitem__.assert_called_once_with(
'test_collection')
mongodb_activity.database.__getitem__.assert_called_once_with('test_collection')
collection.find.assert_called_once_with(
{
'inserted_at': {
'$gt': datetime.strptime(
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ)
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
)
}
},
{"_id": 0}
{'_id': 0},
)
assert result == {
'name': {
0: 'test1'
},
'value': {
0: 1
},
'inserted_at': {
0: '2023-01-01 12:00:00.000000+0000'
}
'name': {0: 'test1'},
'value': {0: 1},
'inserted_at': {0: '2023-01-01 12:00:00.000000+0000'},
}
@@ -184,20 +168,21 @@ async def test_load_latest_data_error(mongodb_activity):
collection.find.side_effect = Exception('test')
try:
await mongodb_activity.load_latest_data({
'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
'collection_name': 'test_collection',
'last_data_timestamp': '2023-01-01 12:00:00.000000+0000'
})
await mongodb_activity.load_latest_data(
{
'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
'collection_name': 'test_collection',
'last_data_timestamp': '2023-01-01 12:00:00.000000+0000',
}
)
except Exception as e:
assert str(e) == 'test'
mongodb_activity.send_notification.assert_called_once_with(
metadata={'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule'},
metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
notification_id='MONGO_LOAD_ERROR',
message='Error loading data from MongoDB: test',
block='load_latest_data',
level=NotificationLevel.ERROR,
attachment_content=ANY
attachment_content=ANY,
)

292
tests/activities/test_redis.py
View File

@@ -1,10 +1,12 @@
from unittest.mock import MagicMock, patch, ANY
from datetime import datetime
import pytest
from unittest.mock import ANY, MagicMock, patch
import numpy as np
import pytest
from pandas import DataFrame
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from scouter.activities.redis import Redis
@@ -13,9 +15,14 @@ from scouter.activities.redis import Redis
def redis_activity(_mock_redis_init):
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
activity = Redis(host='localhost', port=6379,
logger=logger, notification_handler=notification_handler,
username='test', password='test')
activity = Redis(
host='localhost',
port=6379,
logger=logger,
notification_handler=notification_handler,
username='test',
password='test',
)
activity.redis_client = MagicMock()
activity.logger = logger
@@ -29,17 +36,16 @@ def test_redis_initialization(mock_redis_init):
"""Test Redis activity initialization"""
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
Redis(host='localhost', port=6379,
logger=logger, notification_handler=notification_handler,
username='test', password='test')
Redis(
host='localhost',
port=6379,
logger=logger,
notification_handler=notification_handler,
username='test',
password='test',
)
mock_redis_init.assert_called_once_with(
ANY,
'localhost',
6379,
'test',
'test',
logger,
notification_handler
ANY, 'localhost', 6379, 'test', 'test', logger, notification_handler
)
@@ -48,7 +54,7 @@ metadata = {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
'workflow_name': 'scouter',
}
}
@@ -56,11 +62,7 @@ metadata = {
@pytest.mark.asyncio
async def test_get_last_data_timestamp_none(redis_activity):
"""Test get_last_data_timestamp"""
test_data = {
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule'
}
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
redis_activity.get = MagicMock(return_value=None)
@@ -72,19 +74,13 @@ async def test_get_last_data_timestamp_none(redis_activity):
@pytest.mark.asyncio
async def test_get_last_data_timestamp_not_none(redis_activity):
"""Test get_last_data_timestamp"""
test_data = {
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule'
}
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
redis_activity.get = MagicMock(return_value='2023-01-01 12:00:00')
result = await redis_activity.get_last_data_timestamp(test_data)
redis_activity.get.assert_called_once_with(
'last_data_timestamp:test_pipeline:test_schedule'
)
redis_activity.get.assert_called_once_with('last_data_timestamp:test_pipeline:test_schedule')
assert result == '2023-01-01 12:00:00'
@@ -92,17 +88,12 @@ async def test_get_last_data_timestamp_not_none(redis_activity):
@pytest.mark.asyncio
async def test_get_last_data_timestamp_error(redis_activity):
"""Test get_last_data_timestamp error"""
test_data = {
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule'
}
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
redis_activity.send_notification = MagicMock()
redis_activity.get = MagicMock(side_effect=Exception('test'))
try:
await redis_activity.get_last_data_timestamp(test_data)
except Exception as e:
@@ -110,15 +101,15 @@ async def test_get_last_data_timestamp_error(redis_activity):
redis_activity.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id="REDIS_GET_ERROR",
message="Error getting last data timestamp: test",
block="get_last_data_timestamp",
notification_id='REDIS_GET_ERROR',
message='Error getting last data timestamp: test',
block='get_last_data_timestamp',
level=NotificationLevel.ERROR,
attachment_content=ANY
attachment_content=ANY,
)
else:
assert False, "Expected exception"
raise AssertionError('Expected exception')
@pytest.mark.asyncio
@@ -128,7 +119,7 @@ async def test_put_last_data_timestamp_empty_dataframe(redis_activity):
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule',
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records')
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
}
redis_activity.set = MagicMock()
@@ -144,16 +135,18 @@ async def test_put_last_data_timestamp_empty_dataframe(redis_activity):
async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
"""Test put_last_data_timestamp with not empty dataframe"""
data = DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'inserted_at': ['2023-01-01 12:00:00', '2023-01-01 12:00:01']
})
data = DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'inserted_at': ['2023-01-01 12:00:00', '2023-01-01 12:00:01'],
}
)
test_data = {
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule',
'data': data.to_dict('records')
'data': data.to_dict('records'),
}
redis_activity.set = MagicMock()
@@ -163,9 +156,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
assert result == '2023-01-01 12:00:01'
redis_activity.set.assert_called_once_with(
'last_data_timestamp:test_pipeline:test_schedule',
'2023-01-01 12:00:01',
ttl=18000
'last_data_timestamp:test_pipeline:test_schedule', '2023-01-01 12:00:01', ttl=18000
)
@@ -176,11 +167,13 @@ async def test_put_last_data_timestamp_error(redis_activity):
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule',
'data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'inserted_at': ['2023-01-01 12:00:00'] * 2
}).to_dict('records')
'data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'inserted_at': ['2023-01-01 12:00:00'] * 2,
}
).to_dict('records'),
}
redis_activity.send_notification = MagicMock()
@@ -194,15 +187,15 @@ async def test_put_last_data_timestamp_error(redis_activity):
redis_activity.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id="REDIS_SET_ERROR",
message="Error setting last data timestamp: test",
block="put_last_data_timestamp",
notification_id='REDIS_SET_ERROR',
message='Error setting last data timestamp: test',
block='put_last_data_timestamp',
level=NotificationLevel.ERROR,
attachment_content=ANY
attachment_content=ANY,
)
else:
assert False, "Expected exception"
raise AssertionError('Expected exception')
@pytest.mark.asyncio
@@ -215,15 +208,14 @@ async def test_group_and_hold_data_new_key(redis_activity):
'schedule_name': 'test_schedule',
'retention_time': 3600,
'model_id': 1,
'data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2
}).to_dict('records'),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2,
}
).to_dict('records'),
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
# Mock get to return None for new key
@@ -238,7 +230,7 @@ async def test_group_and_hold_data_new_key(redis_activity):
'timestamp': {0: '2023-01-01 12:00:00', 1: '2023-01-01 12:00:00'},
'variable': {0: 'sensor1', 1: 'sensor2'},
'value': {0: 25.5, 1: 30.0},
'model_id': {0: 1, 1: 1}
'model_id': {0: 1, 1: 1},
}
assert result == expected_result
@@ -246,11 +238,7 @@ async def test_group_and_hold_data_new_key(redis_activity):
redis_activity.set.assert_called_once()
args, kwargs = redis_activity.set.call_args
assert args[0] == 'held_data_test_pipeline_test_schedule'
assert args[1] == {
'sensor1': 25.5,
'sensor2': 30.0,
'timestamp': '2023-01-01 12:00:00'
}
assert args[1] == {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}
assert kwargs['ttl'] == 3600
@@ -258,11 +246,7 @@ async def test_group_and_hold_data_new_key(redis_activity):
async def test_group_and_hold_data_update_existing(redis_activity):
"""Test updating existing data with group_and_hold_data"""
# Setup initial data in Redis
existing_data = {
'sensor1': 20.0,
'sensor2': 28.0,
'timestamp': '2023-01-01 11:00:00'
}
existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
# New data to update with
test_data = {
@@ -271,16 +255,14 @@ async def test_group_and_hold_data_update_existing(redis_activity):
'schedule_name': 'test_schedule',
'retention_time': 3600,
'model_id': 1,
'data': DataFrame({
'name': ['sensor1', 'sensor3'],
'value': [25.5, 42.0],
'timestamp': ['2023-01-01 12:00:00'] * 2
}).to_dict('records'),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2',
'sensor3': 'sensor3'
}
'data': DataFrame(
{
'name': ['sensor1', 'sensor3'],
'value': [25.5, 42.0],
'timestamp': ['2023-01-01 12:00:00'] * 2,
}
).to_dict('records'),
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2', 'sensor3': 'sensor3'},
}
# Mock get to return existing data
@@ -295,7 +277,7 @@ async def test_group_and_hold_data_update_existing(redis_activity):
'timestamp': {0: '2023-01-01 12:00:00', 1: '2023-01-01 12:00:00', 2: '2023-01-01 12:00:00'},
'variable': {0: 'sensor1', 1: 'sensor2', 2: 'sensor3'},
'value': {0: 25.5, 1: 28.0, 2: 42.0},
'model_id': {0: 1, 1: 1, 2: 1}
'model_id': {0: 1, 1: 1, 2: 1},
}
assert result == expected_result
@@ -307,7 +289,7 @@ async def test_group_and_hold_data_update_existing(redis_activity):
'sensor1': 25.5,
'sensor2': 28.0,
'sensor3': 42.0,
'timestamp': '2023-01-01 12:00:00'
'timestamp': '2023-01-01 12:00:00',
}
assert kwargs['ttl'] == 3600
@@ -322,15 +304,14 @@ async def test_group_and_hold_data_with_none_values(redis_activity):
'schedule_name': 'test_schedule',
'retention_time': 3600,
'model_id': 1,
'data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [None, 30.0],
'timestamp': [datetime(2023, 1, 1, 12, 0, 0)] * 2
}).to_dict('records'),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [None, 30.0],
'timestamp': [datetime(2023, 1, 1, 12, 0, 0)] * 2,
}
).to_dict('records'),
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
# Mock get to return None for new key
@@ -355,10 +336,7 @@ async def test_group_and_hold_data_empty_dataframe(redis_activity):
'schedule_name': 'test_schedule',
'retention_time': 3600,
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
redis_activity.get = MagicMock(return_value=None)
@@ -379,10 +357,7 @@ async def test_group_and_hold_data_error_get(redis_activity):
'retention_time': 3600,
'model_id': 1,
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
redis_activity.get = MagicMock(side_effect=Exception('test'))
@@ -396,15 +371,15 @@ async def test_group_and_hold_data_error_get(redis_activity):
redis_activity.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id="REDIS_GET_ERROR",
message="Error getting held data: test",
block="group_and_hold_data",
notification_id='REDIS_GET_ERROR',
message='Error getting held data: test',
block='group_and_hold_data',
level=NotificationLevel.ERROR,
attachment_content=ANY
attachment_content=ANY,
)
else:
assert False, "Expected exception"
raise AssertionError('Expected exception')
@pytest.mark.asyncio
@@ -417,17 +392,10 @@ async def test_group_and_hold_data_error_set(redis_activity):
'retention_time': 3600,
'model_id': 1,
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
existing_data = {
'sensor1': 20.0,
'sensor2': 28.0,
'timestamp': '2023-01-01 11:00:00'
}
existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
# Mock get to return existing data
redis_activity.get = MagicMock(return_value=existing_data)
@@ -450,67 +418,65 @@ async def test_store_data_package(redis_activity):
**metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'held_data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2
}).to_dict(),
'data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2
}).to_dict(),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'held_data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2,
}
).to_dict(),
'data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2,
}
).to_dict(),
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
await redis_activity.store_data_package(test_data)
redis_activity.set.assert_called_once_with(
ANY,
{
'data': test_data['data'],
'held_data': test_data['held_data']
},
ttl=120)
ANY, {'data': test_data['data'], 'held_data': test_data['held_data']}, ttl=120
)
@pytest.mark.asyncio
async def test_store_data_package_error(redis_activity):
"""Test store_data_package error"""
redis_activity.set = MagicMock(side_effect=Exception('test'))
redis_activity.set = MagicMock(side_effect=ValueError('test'))
redis_activity.send_notification = MagicMock()
test_data = {
**metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'held_data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2
}).to_dict(),
'data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2
}).to_dict(),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'held_data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2,
}
).to_dict(),
'data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2,
}
).to_dict(),
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
with pytest.raises(Exception):
with pytest.raises(ValueError):
await redis_activity.store_data_package(test_data)
redis_activity.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id="REDIS_SET_ERROR",
message="Error setting data package: test",
block="store_data_package",
notification_id='REDIS_SET_ERROR',
message='Error setting data package: test',
block='store_data_package',
level=NotificationLevel.ERROR,
attachment_content=ANY
attachment_content=ANY,
)

19
tests/test_metrics.py
View File

@@ -1,7 +1,7 @@
# tests/unit/test_metrics.py
import pytest
from prometheus_client import Counter, Gauge, Histogram
from prometheus_client import Counter, Gauge
import scouter.metrics as metrics
# --- Test Functions for Each Metric (Corrected for v0.22.0 _name behavior) ---
@@ -11,15 +11,22 @@ def test_scouter_laborious_data_written_count():
"""Verify the definition of LABORIOUS_DATA_WRITTEN_COUNT."""
assert metrics.LABORIOUS_DATA_WRITTEN_COUNT is not None
assert isinstance(metrics.LABORIOUS_DATA_WRITTEN_COUNT, Counter)
assert metrics.LABORIOUS_DATA_WRITTEN_COUNT._name == "scouter_laborious_data_written_count"
assert metrics.LABORIOUS_DATA_WRITTEN_COUNT._name == 'scouter_laborious_data_written_count'
assert set(metrics.LABORIOUS_DATA_WRITTEN_COUNT._labelnames) == {
"pod_id", "model_name", "pipeline_name"}
'pod_id',
'model_name',
'pipeline_name',
}
def test_scouter_tag_changes_monitor():
"""Verify the definition of TAG_CHANGES_MONITOR."""
assert metrics.TAG_CHANGES_MONITOR is not None
assert isinstance(metrics.TAG_CHANGES_MONITOR, Gauge)
assert metrics.TAG_CHANGES_MONITOR._name == "scouter_tag_changes_monitor"
assert metrics.TAG_CHANGES_MONITOR._name == 'scouter_tag_changes_monitor'
assert set(metrics.TAG_CHANGES_MONITOR._labelnames) == {
"pod_id", "model_name", "pipeline_name", "tag_name"}
'pod_id',
'model_name',
'pipeline_name',
'tag_name',
}

45
tests/utils/quality/test_filters.py
View File

@@ -1,8 +1,9 @@
import pytest
import pandas as pd
import numpy as np
import pandas as pd
import pytest
from pandas.testing import assert_frame_equal
from scouter.utils.quality.filters import check_data_range, out_of_bounds_filter, null_values_filter
from scouter.utils.quality.filters import check_data_range, null_values_filter, out_of_bounds_filter
# Fixtures
@@ -10,12 +11,14 @@ from scouter.utils.quality.filters import check_data_range, out_of_bounds_filter
@pytest.fixture
def sample_dataframe():
"""Fixture providing a sample DataFrame for testing."""
return pd.DataFrame({
'tag': ['temp', 'temp', 'pressure', 'pressure', 'humidity', 'wind_speed'],
'name': ['temp', 'temp', 'pressure', 'pressure', 'humidity', 'wind_speed'],
'value': [25, 35, 95, 105, 60, None],
'timestamp': pd.date_range(start='2023-01-01', periods=6)
})
return pd.DataFrame(
{
'tag': ['temp', 'temp', 'pressure', 'pressure', 'humidity', 'wind_speed'],
'name': ['temp', 'temp', 'pressure', 'pressure', 'humidity', 'wind_speed'],
'value': [25, 35, 95, 105, 60, None],
'timestamp': pd.date_range(start='2023-01-01', periods=6),
}
)
@pytest.fixture
@@ -25,7 +28,7 @@ def nodes_data_range():
'temp': {'data_range': [10, 30]},
'pressure': {'data_range': [90, 100]},
'humidity': {'data_range': [40, 80]},
'wind_speed': {'data_range': [0, 50]}
'wind_speed': {'data_range': [0, 50]},
}
@@ -59,6 +62,7 @@ def test_check_data_range(value, val_range, expected):
else:
assert result == expected
# Tests for out_of_bounds_filter
@@ -72,8 +76,8 @@ def test_out_of_bounds_filter(sample_dataframe, nodes_data_range):
'timestamp': [
pd.Timestamp('2023-01-02'),
pd.Timestamp('2023-01-04'),
pd.Timestamp('2023-01-06')
]
pd.Timestamp('2023-01-06'),
],
}
expected_df = pd.DataFrame(expected_data)
@@ -91,6 +95,7 @@ def test_out_of_bounds_filter_empty_df(nodes_data_range):
assert result.empty
assert list(result.columns) == ['tag', 'name', 'value', 'timestamp']
# Tests for null_values_filter
@@ -100,7 +105,7 @@ def test_null_values_filter(sample_dataframe, nodes_data_range):
'tag': ['wind_speed'],
'name': ['wind_speed'],
'value': [None],
'timestamp': [pd.Timestamp('2023-01-06')]
'timestamp': [pd.Timestamp('2023-01-06')],
}
expected_df = pd.DataFrame(expected_data)
@@ -113,12 +118,14 @@ def test_null_values_filter(sample_dataframe, nodes_data_range):
def test_null_values_filter_no_nulls(nodes_data_range):
"""Test with a DataFrame containing no null values."""
df = pd.DataFrame({
'tag': ['temp', 'pressure'],
'name': ['temp', 'pressure'],
'value': [25, 100],
'timestamp': pd.date_range(start='2023-01-01', periods=2)
})
df = pd.DataFrame(
{
'tag': ['temp', 'pressure'],
'name': ['temp', 'pressure'],
'value': [25, 100],
'timestamp': pd.date_range(start='2023-01-01', periods=2),
}
)
result = null_values_filter(df, nodes_data_range)
assert result.empty
assert list(result.columns) == ['tag', 'name', 'value', 'timestamp']

113
tests/utils/test_connectors_config.py
View File

@@ -1,12 +1,14 @@
import os
from unittest.mock import patch
import pytest
from scouter.utils.connectors_config import (
build_druid_config,
build_kafka_config,
build_mongodb_config,
build_postgres_config,
build_kafka_config,
build_redis_config
build_redis_config,
)
@@ -16,7 +18,7 @@ def mock_env_vars():
yield
@pytest.mark.usefixtures("mock_env_vars")
@pytest.mark.usefixtures('mock_env_vars')
def test_build_postgres_config_defaults():
"""Test that build_postgres_config returns default values when no env vars are set"""
config = build_postgres_config()
@@ -28,22 +30,25 @@ def test_build_postgres_config_defaults():
'password': 'sientia',
'dbname': 'sientia',
'min_connections': 5,
'max_connections': 20
'max_connections': 20,
}
@pytest.mark.usefixtures("mock_env_vars")
@pytest.mark.usefixtures('mock_env_vars')
def test_build_postgres_config_with_env_vars():
"""Test that build_postgres_config uses env vars when set"""
with patch.dict(os.environ, {
'POSTGRES_HOST': 'db.example.com',
'POSTGRES_PORT': '5433',
'POSTGRES_USER': 'admin',
'POSTGRES_PASSWORD': 'secret',
'POSTGRES_DBNAME': 'test_db',
'POSTGRES_MIN_CONNECTIONS': '3',
'POSTGRES_MAX_CONNECTIONS': '15'
}):
with patch.dict(
os.environ,
{
'POSTGRES_HOST': 'db.example.com',
'POSTGRES_PORT': '5433',
'POSTGRES_USER': 'admin',
'POSTGRES_PASSWORD': 'secret',
'POSTGRES_DBNAME': 'test_db',
'POSTGRES_MIN_CONNECTIONS': '3',
'POSTGRES_MAX_CONNECTIONS': '15',
},
):
config = build_postgres_config()
assert config == {
@@ -53,11 +58,11 @@ def test_build_postgres_config_with_env_vars():
'password': 'secret',
'dbname': 'test_db',
'min_connections': 3,
'max_connections': 15
'max_connections': 15,
}
@pytest.mark.usefixtures("mock_env_vars")
@pytest.mark.usefixtures('mock_env_vars')
def test_build_kafka_config_defaults():
"""Test that build_kafka_config returns default values when no env vars are set"""
config = build_kafka_config()
@@ -65,55 +70,53 @@ def test_build_kafka_config_defaults():
assert config == {
'bootstrap_servers': 'localhost:9092',
'polling_time': 1000,
'group_id': 'scouter-group'
'group_id': 'scouter-group',
}
@pytest.mark.usefixtures("mock_env_vars")
@pytest.mark.usefixtures('mock_env_vars')
def test_build_kafka_config_with_env_vars():
"""Test that build_kafka_config uses env vars when set"""
with patch.dict(os.environ, {
'KAFKA_BOOTSTRAP_SERVERS': 'kafka.example.com:9092',
'KAFKA_POLLING_TIME': '5000'
}):
with patch.dict(
os.environ,
{'KAFKA_BOOTSTRAP_SERVERS': 'kafka.example.com:9092', 'KAFKA_POLLING_TIME': '5000'},
):
config = build_kafka_config()
assert config == {
'bootstrap_servers': 'kafka.example.com:9092',
'polling_time': 5000,
'group_id': 'scouter-group'
'group_id': 'scouter-group',
}
@pytest.mark.usefixtures("mock_env_vars")
@pytest.mark.usefixtures('mock_env_vars')
def test_build_redis_config_defaults():
"""Test that build_redis_config returns default values when no env vars are set"""
config = build_redis_config()
assert config == {
'host': 'localhost',
'port': 6379,
'username': None,
'password': None
}
assert config == {'host': 'localhost', 'port': 6379, 'username': None, 'password': None}
@pytest.mark.usefixtures("mock_env_vars")
@pytest.mark.usefixtures('mock_env_vars')
def test_build_redis_config_with_env_vars():
"""Test that build_redis_config uses env vars when set"""
with patch.dict(os.environ, {
'REDIS_HOST': 'redis.example.com',
'REDIS_PORT': '6380',
'REDIS_USERNAME': 'test',
'REDIS_PASSWORD': 'test'
}):
with patch.dict(
os.environ,
{
'REDIS_HOST': 'redis.example.com',
'REDIS_PORT': '6380',
'REDIS_USERNAME': 'test',
'REDIS_PASSWORD': 'test',
},
):
config = build_redis_config()
assert config == {
'host': 'redis.example.com',
'port': 6380,
'username': 'test',
'password': 'test'
'password': 'test',
}
@@ -123,23 +126,26 @@ def test_build_mongodb_config_defaults():
assert config == {
'connection_string': 'mongodb://sientia:sientia@localhost:27017', # NOSONAR
'database_name': 'sientia'
'database_name': 'sientia',
}
def test_build_mongodb_config_with_env_vars():
"""Test that build_mongodb_config uses env vars when set"""
with patch.dict(os.environ, {
'MONGODB_URL': 'mongodb.example.com:27017',
'MONGODB_DATABASE_NAME': 'test_db',
'MONGODB_USERNAME': 'test',
'MONGODB_PASSWORD': 'test'
}):
with patch.dict(
os.environ,
{
'MONGODB_URL': 'mongodb.example.com:27017',
'MONGODB_DATABASE_NAME': 'test_db',
'MONGODB_USERNAME': 'test',
'MONGODB_PASSWORD': 'test',
},
):
config = build_mongodb_config()
assert config == {
'connection_string': 'mongodb://test:test@mongodb.example.com:27017',
'database_name': 'test_db'
'database_name': 'test_db',
}
@@ -147,21 +153,12 @@ def test_build_druid_config_defaults():
"""Test that build_druid_config returns default values when no env vars are set"""
config = build_druid_config()
assert config == {
'host': 'localhost',
'port': 8082
}
assert config == {'host': 'localhost', 'port': 8082}
def test_build_druid_config_with_env_vars():
"""Test that build_druid_config uses env vars when set"""
with patch.dict(os.environ, {
'DRUID_HOST': 'druid.example.com',
'DRUID_PORT': '8083'
}):
with patch.dict(os.environ, {'DRUID_HOST': 'druid.example.com', 'DRUID_PORT': '8083'}):
config = build_druid_config()
assert config == {
'host': 'druid.example.com',
'port': 8083
}
assert config == {'host': 'druid.example.com', 'port': 8083}

253
tests/workflow/sub_workflows/test_core_scouter.py
View File

@@ -1,9 +1,11 @@
from unittest.mock import AsyncMock, patch, call, ANY
from unittest.mock import ANY, AsyncMock, call, patch
import pytest
from scouter.workflow.sub_workflows.core_scouter import CoreScouter
from scouter.activities.activities import Activities
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from scouter.activities.activities import Activities
from scouter.workflow.sub_workflows.core_scouter import CoreScouter
@pytest.fixture
def core_scouter():
@@ -14,7 +16,10 @@ def core_scouter():
@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock)
async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
mock_workflow.execute_local_activity_method.side_effect = [
'filtered_data', 'grouped_data', 'held_data']
'filtered_data',
'grouped_data',
'held_data',
]
await core_scouter.run(
input_data={
'metadata': {
@@ -22,7 +27,7 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
'workflow_name': 'test_workflow',
}
},
'workflow_name': 'test_workflow',
@@ -36,7 +41,7 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
'table_name': 'test_table',
'retention_time': 3600,
'model_tags': {},
'debug_data_package': True
'debug_data_package': True,
}
)
@@ -45,81 +50,90 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
'workflow_name': 'test_workflow',
}
}
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)])
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.aggregate_data,
{
**expected_metadata,
'data': 'filtered_data',
'model_tags': {}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)])
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': 'grouped_data',
'model_id': 'test_model_id',
'retention_time': 3600,
'model_tags': {}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)])
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.aggregate_data,
{**expected_metadata, 'data': 'filtered_data', 'model_tags': {}},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': 'grouped_data',
'model_id': 'test_model_id',
'retention_time': 3600,
'model_tags': {},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_activity_method.assert_has_calls([
call(
Activities.export_data_to_postgres,
{
**expected_metadata,
'schema': 'test_schema',
'table_name': 'test_table',
'data': 'held_data',
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_WITH_TZ
}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
mock_workflow.execute_activity_method.assert_has_calls(
[
call(
Activities.export_data_to_postgres,
{
**expected_metadata,
'schema': 'test_schema',
'table_name': 'test_table',
'data': 'held_data',
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_WITH_TZ,
},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_activity_method.assert_has_calls([
call(
Activities.store_data_package,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'held_data': 'held_data',
'data': 'test_data'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
mock_workflow.execute_activity_method.assert_has_calls(
[
call(
Activities.store_data_package,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'held_data': 'held_data',
'data': 'test_data',
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
@pytest.mark.asyncio
@@ -133,7 +147,7 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
'workflow_name': 'test_workflow',
}
},
'workflow_name': 'test_workflow',
@@ -146,7 +160,7 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
'schema': 'test_schema',
'table_name': 'test_table',
'retention_time': 3600,
'model_tags': {}
'model_tags': {},
}
)
@@ -155,47 +169,52 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
'workflow_name': 'test_workflow',
}
}
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)])
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.aggregate_data,
{
**expected_metadata,
'data': {},
'model_tags': {}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)])
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': {},
'model_id': 'test_model_id',
'retention_time': 3600,
'model_tags': {}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)])
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.aggregate_data,
{**expected_metadata, 'data': {}, 'model_tags': {}},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': {},
'model_id': 'test_model_id',
'retention_time': 3600,
'model_tags': {},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
assert mock_workflow.execute_local_activity_method.call_count == 3

29
tests/workflow/test_fake_data.py
View File

@@ -1,29 +0,0 @@
from unittest.mock import AsyncMock, patch, ANY
from pytest import fixture, mark
from scouter.workflow.fake_data import FakeData
from scouter.activities.faker import Faker
@fixture
def fake_data():
return FakeData()
@mark.asyncio
@patch('scouter.workflow.fake_data.workflow', new_callable=AsyncMock)
async def test_fake_data_workflow(mock_workflow, fake_data):
mock_workflow.execute_activity_method.return_value = None
await fake_data.run(
{
'topic': 'test_topic'
}
)
mock_workflow.execute_activity_method.assert_called_once_with(
Faker.generate_and_send_data,
{
'topic': 'test_topic'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)

59
tests/workflow/test_scouter.py
View File

@@ -1,7 +1,9 @@
from unittest.mock import AsyncMock, patch, ANY, call
from unittest.mock import ANY, AsyncMock, call, patch
from pytest import fixture, mark
from scouter.workflow.scouter import Scouter
from scouter.activities.activities import Activities
from scouter.workflow.scouter import Scouter
@fixture
@@ -12,17 +14,16 @@ def scouter():
@mark.asyncio
@patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock)
async def test_scouter_workflow(mock_workflow, scouter):
mock_workflow.execute_local_activity_method.side_effect = [
'test_last_data_timestamp',
'test_data'
'test_data',
]
await scouter.run(
input_data={
'topic': 'test_topic',
'schedule_name': 'test_schedule',
'model_name': 'test_model',
'model_id': 'test_model_id'
'model_id': 'test_model_id',
}
)
@@ -31,7 +32,7 @@ async def test_scouter_workflow(mock_workflow, scouter):
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
'workflow_name': 'scouter',
}
}
@@ -39,15 +40,12 @@ async def test_scouter_workflow(mock_workflow, scouter):
[
call(
Activities.get_last_data_timestamp,
{
**expected_metadata,
'workflow_name': 'scouter',
'schedule_name': 'test_schedule'
},
{**expected_metadata, 'workflow_name': 'scouter', 'schedule_name': 'test_schedule'},
retry_policy=ANY,
start_to_close_timeout=ANY
start_to_close_timeout=ANY,
)
])
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
@@ -55,13 +53,14 @@ async def test_scouter_workflow(mock_workflow, scouter):
Activities.load_latest_data,
{
**expected_metadata,
'collection_name': "raw_test_schedule",
'last_data_timestamp': 'test_last_data_timestamp'
'collection_name': 'raw_test_schedule',
'last_data_timestamp': 'test_last_data_timestamp',
},
retry_policy=ANY,
start_to_close_timeout=ANY
start_to_close_timeout=ANY,
)
])
]
)
mock_workflow.execute_activity_method.assert_called_once_with(
Activities.put_last_data_timestamp,
@@ -69,10 +68,10 @@ async def test_scouter_workflow(mock_workflow, scouter):
**expected_metadata,
'data': 'test_data',
'workflow_name': 'scouter',
'schedule_name': 'test_schedule'
'schedule_name': 'test_schedule',
},
retry_policy=ANY,
start_to_close_timeout=ANY
start_to_close_timeout=ANY,
)
mock_workflow.execute_child_workflow.assert_called_once_with(
@@ -84,24 +83,21 @@ async def test_scouter_workflow(mock_workflow, scouter):
'workflow_name': 'scouter',
'schedule_name': 'test_schedule',
'model_name': 'test_model',
'model_id': 'test_model_id'
}
'model_id': 'test_model_id',
},
)
@mark.asyncio
@patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock)
async def test_scouter_workflow_empty(mock_workflow, scouter):
mock_workflow.execute_local_activity_method.side_effect = [
'test_last_data_timestamp',
{}
]
mock_workflow.execute_local_activity_method.side_effect = ['test_last_data_timestamp', {}]
await scouter.run(
input_data={
'topic': 'test_topic',
'schedule_name': 'test_schedule',
'model_name': 'test_model',
'model_id': 'test_model_id'
'model_id': 'test_model_id',
}
)
@@ -110,7 +106,7 @@ async def test_scouter_workflow_empty(mock_workflow, scouter):
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
'workflow_name': 'scouter',
}
}
@@ -120,13 +116,14 @@ async def test_scouter_workflow_empty(mock_workflow, scouter):
Activities.load_latest_data,
{
**expected_metadata,
'collection_name': "raw_test_schedule",
'last_data_timestamp': 'test_last_data_timestamp'
'collection_name': 'raw_test_schedule',
'last_data_timestamp': 'test_last_data_timestamp',
},
retry_policy=ANY,
start_to_close_timeout=ANY
start_to_close_timeout=ANY,
)
])
]
)
mock_workflow.execute_activity_method.assert_not_called()
mock_workflow.execute_child_workflow.assert_not_called()