From b230ad614efab6fdac6e340c58c42e79b1277f5c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 18 Jul 2025 16:19:06 -0300 Subject: [PATCH 1/5] SIENTIAPDE-1151 Update Redis set method to use a TTL of 5 hours for improved data retention. --- scouter/activities/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 9bad871..5174d87 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -75,7 +75,7 @@ class Redis(RedisBase): ) try: - self.set(key, last_data_timestamp, ttl=None) + self.set(key, last_data_timestamp, ttl=60*60*5) except Exception as e: self.send_notification( metadata=metadata, From a9b421b134cb36ee1dbc19e6db213407950db0fb Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 18 Jul 2025 16:21:18 -0300 Subject: [PATCH 2/5] SIENTIAPDE-1151 Update GITHUB_BRANCH in values.yaml to SIENTIAPDE-1151-criar-testes-de-stress for stress testing implementation. --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index 540eb7b..64ec56f 100644 --- a/values.yaml +++ b/values.yaml @@ -118,7 +118,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1163-alterar-dinamica-de-notificacoes-para-usar-o-mongodb-ao-inves-do-kafka" + value: "SIENTIAPDE-1151-criar-testes-de-stress" - name: PYTHON_APP value: "scouter.worker.worker" From aacd52b5c745825467750246d7ba55f1ba7bab53 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 21 Jul 2025 11:12:39 -0300 Subject: [PATCH 3/5] SIENTIAPDE-1151 Add debug logging for collected data in MongoDB activity --- scouter/activities/mongodb.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index 3189235..c4c6fe4 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -112,6 +112,11 @@ class MongoDB(BaseActivity): data = clear_mongo_id(data) + self.debug( + f"Collected: {data}", + metadata=metadata + ) + for item in data: item['inserted_at'] = item['inserted_at'].strftime( "%Y-%m-%d %H:%M:%S.%f") From b68b3b403b89bb765f789a8804e16ea7e8acd376 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 21 Jul 2025 12:03:03 -0300 Subject: [PATCH 4/5] SIENTIAPDE-1151 SIENTIAPDE-1151 Filter data in Gates activity based on model tags to enhance data accuracy. --- scouter/activities/gates.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py index 0cefdae..ea9d1bf 100644 --- a/scouter/activities/gates.py +++ b/scouter/activities/gates.py @@ -192,6 +192,10 @@ class Gates(BaseActivity): metadata=metadata ) + tags = list(model_tags.keys()) + + data = data[data['name'].isin(tags)] + for filter_name, config in filters.items(): policy = config['policy'] if filter_name not in quality_gate_filters: From f6913da932433e290a162553b0420e0bea5d2cbd Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 22 Jul 2025 08:16:21 -0300 Subject: [PATCH 5/5] SIENTIAPDE-1151 Enhance test cases in Gates and Redis activities by adding 'name' field to data and updating Redis TTL to 5 hours for better data retention. --- tests/activities/test_gates.py | 2 ++ tests/activities/test_redis.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/activities/test_gates.py b/tests/activities/test_gates.py index ab5b0a4..dec6b9d 100644 --- a/tests/activities/test_gates.py +++ b/tests/activities/test_gates.py @@ -37,6 +37,7 @@ async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture): } }, 'data': { + 'name': ['tag1', 'tag2', 'tag3'], 'tag': ['tag1', 'tag2', 'tag3'], 'value': [1.0, None, 3.0], 'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03'], @@ -69,6 +70,7 @@ async def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture): } }, 'data': { + 'name': ['tag1', 'tag2', 'tag3'], 'tag': ['tag1', 'tag2', 'tag3'], 'value': [1.0, 200.0, 3.0], 'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03'] diff --git a/tests/activities/test_redis.py b/tests/activities/test_redis.py index 99eed1f..ce3215e 100644 --- a/tests/activities/test_redis.py +++ b/tests/activities/test_redis.py @@ -164,7 +164,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity): redis_activity.set.assert_called_once_with( 'last_data_timestamp_test_pipeline_test_schedule', '2023-01-01 12:00:01', - ttl=None + ttl=18000 )