diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py index 0cefdae..ea9d1bf 100644 --- a/scouter/activities/gates.py +++ b/scouter/activities/gates.py @@ -192,6 +192,10 @@ class Gates(BaseActivity): metadata=metadata ) + tags = list(model_tags.keys()) + + data = data[data['name'].isin(tags)] + for filter_name, config in filters.items(): policy = config['policy'] if filter_name not in quality_gate_filters: diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index 3189235..c4c6fe4 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -112,6 +112,11 @@ class MongoDB(BaseActivity): data = clear_mongo_id(data) + self.debug( + f"Collected: {data}", + metadata=metadata + ) + for item in data: item['inserted_at'] = item['inserted_at'].strftime( "%Y-%m-%d %H:%M:%S.%f") diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 9bad871..5174d87 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -75,7 +75,7 @@ class Redis(RedisBase): ) try: - self.set(key, last_data_timestamp, ttl=None) + self.set(key, last_data_timestamp, ttl=60*60*5) except Exception as e: self.send_notification( metadata=metadata, diff --git a/tests/activities/test_gates.py b/tests/activities/test_gates.py index ab5b0a4..dec6b9d 100644 --- a/tests/activities/test_gates.py +++ b/tests/activities/test_gates.py @@ -37,6 +37,7 @@ async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture): } }, 'data': { + 'name': ['tag1', 'tag2', 'tag3'], 'tag': ['tag1', 'tag2', 'tag3'], 'value': [1.0, None, 3.0], 'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03'], @@ -69,6 +70,7 @@ async def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture): } }, 'data': { + 'name': ['tag1', 'tag2', 'tag3'], 'tag': ['tag1', 'tag2', 'tag3'], 'value': [1.0, 200.0, 3.0], 'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03'] diff --git a/tests/activities/test_redis.py b/tests/activities/test_redis.py index 99eed1f..ce3215e 100644 --- a/tests/activities/test_redis.py +++ b/tests/activities/test_redis.py @@ -164,7 +164,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity): redis_activity.set.assert_called_once_with( 'last_data_timestamp_test_pipeline_test_schedule', '2023-01-01 12:00:01', - ttl=None + ttl=18000 ) diff --git a/values.yaml b/values.yaml index 540eb7b..64ec56f 100644 --- a/values.yaml +++ b/values.yaml @@ -118,7 +118,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1163-alterar-dinamica-de-notificacoes-para-usar-o-mongodb-ao-inves-do-kafka" + value: "SIENTIAPDE-1151-criar-testes-de-stress" - name: PYTHON_APP value: "scouter.worker.worker"