Merge pull request #12 from Aignosi/SIENTIAPDE-1151-criar-testes-de-stress

Sientiapde 1151 criar testes de stress

scouter/activities/gates.py

@@ -192,6 +192,10 @@ class Gates(BaseActivity):
             metadata=metadata
         )
 
+        tags = list(model_tags.keys())
+
+        data = data[data['name'].isin(tags)]
+
         for filter_name, config in filters.items():
             policy = config['policy']
             if filter_name not in quality_gate_filters:

scouter/activities/mongodb.py

@@ -112,6 +112,11 @@ class MongoDB(BaseActivity):
 
             data = clear_mongo_id(data)
 
+            self.debug(
+                f"Collected: {data}",
+                metadata=metadata
+            )
+
             for item in data:
                 item['inserted_at'] = item['inserted_at'].strftime(
                     "%Y-%m-%d %H:%M:%S.%f")

scouter/activities/redis.py

@@ -75,7 +75,7 @@ class Redis(RedisBase):
         )
 
         try:
-            self.set(key, last_data_timestamp, ttl=None)
+            self.set(key, last_data_timestamp, ttl=60*60*5)
         except Exception as e:
             self.send_notification(
                 metadata=metadata,

tests/activities/test_gates.py

@@ -37,6 +37,7 @@ async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture):
             }
         },
         'data': {
+            'name': ['tag1', 'tag2', 'tag3'],
             'tag': ['tag1', 'tag2', 'tag3'],
             'value': [1.0, None, 3.0],
             'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03'],
@@ -69,6 +70,7 @@ async def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture):
             }
         },
         'data': {
+            'name': ['tag1', 'tag2', 'tag3'],
             'tag': ['tag1', 'tag2', 'tag3'],
             'value': [1.0, 200.0, 3.0],
             'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03']

tests/activities/test_redis.py

@@ -164,7 +164,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
     redis_activity.set.assert_called_once_with(
         'last_data_timestamp_test_pipeline_test_schedule',
         '2023-01-01 12:00:01',
-        ttl=None
+        ttl=18000
     )
 
 

values.yaml

@@ -118,7 +118,7 @@ env:
   - name: GITHUB_REPO_URL
     value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git"
   - name: GITHUB_BRANCH
-    value: "SIENTIAPDE-1163-alterar-dinamica-de-notificacoes-para-usar-o-mongodb-ao-inves-do-kafka"
+    value: "SIENTIAPDE-1151-criar-testes-de-stress"
   - name: PYTHON_APP
     value: "scouter.worker.worker"