From fad10bc0d0a49755088c17083edf99fc16a5ba8f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 21 Aug 2025 12:16:06 -0300 Subject: [PATCH 01/12] SIENTIAPDE-1193 Update requirements and core scouter workflow: comment out old dataops library dependency, change default namespace in worker.py, and enhance timestamp handling in core_scouter.py with DATETIME_FORMAT. --- requirements.txt | 3 ++- scouter/worker/worker.py | 2 +- scouter/workflow/sub_workflows/core_scouter.py | 9 +++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 91864f7..3d33f78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,8 @@ asyncua redis aiokafka pymongo -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.1 +# git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.1 +/home/grezewave/Documents/projects/sientia/sientia-dataops-library git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.5 pydruid[pandas] prometheus-client \ No newline at end of file diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index f636d59..95af238 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -86,7 +86,7 @@ async def main(): temporal_client = await client.Client.connect( target_host=host, - namespace=os.getenv('TEMPORAL_NAMESPACE', 'laborious'), + namespace=os.getenv('TEMPORAL_NAMESPACE', 'scouter'), runtime=new_runtime ) diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py index d6c83f4..2209b72 100644 --- a/scouter/workflow/sub_workflows/core_scouter.py +++ b/scouter/workflow/sub_workflows/core_scouter.py @@ -5,6 +5,7 @@ with workflow.unsafe.imports_passed_through(): from typing import Any from datetime import timedelta from sientia_do.temporal.policies import retry_policy + from sientia_do.temporal.constants import DATETIME_FORMAT @workflow.defn(name="core_scouter") @@ -81,7 +82,11 @@ class CoreScouter: **metadata, 'schema': input_data['schema'], 'table_name': input_data['table_name'], - 'data': held_data + 'data': held_data, + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT + } }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=60) @@ -104,7 +109,7 @@ class CoreScouter: 'data': input_data['data'], 'held_data': held_data, 'workflow_name': input_data['workflow_name'], - 'schedule_name': input_data['schedule_name'] + 'schedule_name': input_data['schedule_name'], }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=60) From eabc642febdd6fb55ead3d95a80eb20af02c4a42 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 22 Aug 2025 08:02:13 -0300 Subject: [PATCH 02/12] SIENTIAPDE-1193 Update dependencies in requirements.txt and change GITHUB_BRANCH in values.yaml to reflect new datetime handling task. --- requirements.txt | 3 +-- values.yaml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3d33f78..a6a4b25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,8 +5,7 @@ asyncua redis aiokafka pymongo -# git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.1 -/home/grezewave/Documents/projects/sientia/sientia-dataops-library +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.2 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.5 pydruid[pandas] prometheus-client \ No newline at end of file diff --git a/values.yaml b/values.yaml index 5d413f0..813f223 100644 --- a/values.yaml +++ b/values.yaml @@ -150,7 +150,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1199-revisar-e-testar-observabilidade" + value: "SIENTIAPDE-1193-conferir-como-a-escrita-de-datetime-ocorre-no-temporal" - name: PYTHON_APP value: "scouter.worker.worker" From 0e0603d2bcf63ef17bb8b49ccb5ba8501465ea0d Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 22 Aug 2025 11:12:37 -0300 Subject: [PATCH 03/12] SIENTIAPDE-1193 Update sientia-dataops-library version to 1.4.3 and refactor timestamp handling in MongoDB and Redis activities to use DATETIME_FORMAT constants for improved consistency. Enhance unit tests to reflect these changes. --- requirements.txt | 2 +- scouter/activities/mongodb.py | 5 +++-- scouter/activities/redis.py | 5 +++-- tests/activities/test_mongo.py | 7 ++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index a6a4b25..8c7edc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ asyncua redis aiokafka pymongo -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.2 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.3 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.5 pydruid[pandas] prometheus-client \ No newline at end of file diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index e36c563..878b185 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -10,6 +10,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.base import BaseActivity from sientia_do.observability.logger import Logger + from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ def clear_mongo_id(docs: list) -> list: @@ -99,7 +100,7 @@ class MongoDB(BaseActivity): else: data_filter = { "inserted_at": { - "$gt": datetime.strptime(last_data_timestamp, "%Y-%m-%d %H:%M:%S.%f") + "$gt": datetime.strptime(last_data_timestamp, DATETIME_FORMAT_MS_WITH_TZ) } } @@ -120,7 +121,7 @@ class MongoDB(BaseActivity): for item in data: item['inserted_at'] = item['inserted_at'].strftime( - "%Y-%m-%d %H:%M:%S.%f") + DATETIME_FORMAT_MS_WITH_TZ) self.info( f"Loaded {len(data)} documents from MongoDB", diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index fb3f556..284acd5 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -11,6 +11,7 @@ with workflow.unsafe.imports_passed_through(): from pandas import DataFrame from datetime import datetime from scouter import metrics + from sientia_do.temporal.constants import DATETIME_FORMAT class Redis(RedisBase): @@ -169,7 +170,7 @@ class Redis(RedisBase): (row['name'], value)) data_hold['timestamp'] = data['timestamp'].max() if not data.empty else \ - datetime.now().strftime("%Y-%m-%d %H:%M:%S") + data_hold['timestamp'] self.set(key, data_hold, ttl=retention_time) @@ -224,7 +225,7 @@ class Redis(RedisBase): data: The data used to collect the data. """ metadata = input_data['metadata'] - key = f"data_package_{input_data['workflow_name']}_{input_data['schedule_name']}_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}" + key = f"data_package_{input_data['workflow_name']}_{input_data['schedule_name']}_{datetime.now().strftime(DATETIME_FORMAT)}" data = DataFrame(input_data['data']) held_data = DataFrame(input_data['held_data']) diff --git a/tests/activities/test_mongo.py b/tests/activities/test_mongo.py index 24ae1c4..8b8cd87 100644 --- a/tests/activities/test_mongo.py +++ b/tests/activities/test_mongo.py @@ -2,6 +2,7 @@ from datetime import datetime from unittest.mock import ANY, MagicMock, call, patch from pytest import fixture, mark from sientia_do.notifications.models import NotificationLevel +from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ from scouter.activities.mongodb import MongoDB, clear_mongo_id @@ -95,7 +96,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity): 'name': 'test1', 'value': 1, 'inserted_at': datetime.strptime( - '2023-01-01 12:00:00.000000', '%Y-%m-%d %H:%M:%S.%f') + '2023-01-01 12:00:00.000000', DATETIME_FORMAT_MS_WITH_TZ) } ] @@ -137,7 +138,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): 'name': 'test1', 'value': 1, 'inserted_at': datetime.strptime( - '2023-01-01 12:00:00.000000', '%Y-%m-%d %H:%M:%S.%f') + '2023-01-01 12:00:00.000000', DATETIME_FORMAT_MS_WITH_TZ) } ] @@ -154,7 +155,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): { 'inserted_at': { '$gt': datetime.strptime( - '2023-01-01 12:00:00.000000', '%Y-%m-%d %H:%M:%S.%f') + '2023-01-01 12:00:00.000000', DATETIME_FORMAT_MS_WITH_TZ) } }, {"_id": 0} From 0b337d00f9108d73980b34bd074cbba7d8cd6b6c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 22 Aug 2025 11:24:26 -0300 Subject: [PATCH 04/12] SIENTIAPDE-1193 Refactor Redis activity to utilize the new 'now' function for timestamp generation, ensuring consistency with updated datetime handling. This change replaces the direct use of 'datetime.now()' with 'now()' from the temporal constants. --- scouter/activities/redis.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 284acd5..56a4313 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -9,9 +9,8 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.logger import Logger from typing import Any from pandas import DataFrame - from datetime import datetime from scouter import metrics - from sientia_do.temporal.constants import DATETIME_FORMAT + from sientia_do.temporal.constants import DATETIME_FORMAT, now class Redis(RedisBase): @@ -225,7 +224,7 @@ class Redis(RedisBase): data: The data used to collect the data. """ metadata = input_data['metadata'] - key = f"data_package_{input_data['workflow_name']}_{input_data['schedule_name']}_{datetime.now().strftime(DATETIME_FORMAT)}" + key = f"data_package_{input_data['workflow_name']}_{input_data['schedule_name']}_{now().strftime(DATETIME_FORMAT)}" data = DataFrame(input_data['data']) held_data = DataFrame(input_data['held_data']) From 96a28e91e5f2a4f42a176feb3c10b160e5d16b70 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 22 Aug 2025 14:05:41 -0300 Subject: [PATCH 05/12] SIENTIAPDE-1193 Enhance timestamp handling in Redis and CoreScouter workflows by updating to DATETIME_FORMAT_MS_WITH_TZ. Modify tests to reflect new timestamp format, ensuring consistency across data structures and improving overall datetime management. --- scouter/activities/redis.py | 1 + scouter/workflow/sub_workflows/core_scouter.py | 4 ++-- tests/activities/test_mongo.py | 14 +++++++------- tests/workflow/sub_workflows/test_core_scouter.py | 8 +++++++- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 56a4313..e7443a1 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -149,6 +149,7 @@ class Redis(RedisBase): # Remove possibly removed tags tags = list(model_tags.keys()) + tags.append('timestamp') self.debug( f"Tags to keep: {tags}", metadata=metadata diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py index 2209b72..3beb4d9 100644 --- a/scouter/workflow/sub_workflows/core_scouter.py +++ b/scouter/workflow/sub_workflows/core_scouter.py @@ -5,7 +5,7 @@ with workflow.unsafe.imports_passed_through(): from typing import Any from datetime import timedelta from sientia_do.temporal.policies import retry_policy - from sientia_do.temporal.constants import DATETIME_FORMAT + from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ @workflow.defn(name="core_scouter") @@ -85,7 +85,7 @@ class CoreScouter: 'data': held_data, 'timestamp_conversion': { 'column': 'timestamp', - 'format': DATETIME_FORMAT + 'format': DATETIME_FORMAT_MS_WITH_TZ } }, retry_policy=retry_policy, diff --git a/tests/activities/test_mongo.py b/tests/activities/test_mongo.py index 8b8cd87..5f91e1c 100644 --- a/tests/activities/test_mongo.py +++ b/tests/activities/test_mongo.py @@ -96,7 +96,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity): 'name': 'test1', 'value': 1, 'inserted_at': datetime.strptime( - '2023-01-01 12:00:00.000000', DATETIME_FORMAT_MS_WITH_TZ) + '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ) } ] @@ -122,7 +122,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity): 0: 1 }, 'inserted_at': { - 0: '2023-01-01 12:00:00.000000' + 0: '2023-01-01 12:00:00.000000+0000' } } @@ -138,14 +138,14 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): 'name': 'test1', 'value': 1, 'inserted_at': datetime.strptime( - '2023-01-01 12:00:00.000000', DATETIME_FORMAT_MS_WITH_TZ) + '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ) } ] result = await mongodb_activity.load_latest_data({ 'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, 'collection_name': 'test_collection', - 'last_data_timestamp': '2023-01-01 12:00:00.000000' + 'last_data_timestamp': '2023-01-01 12:00:00.000000+0000' }) mongodb_activity.database.__getitem__.assert_called_once_with( @@ -155,7 +155,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): { 'inserted_at': { '$gt': datetime.strptime( - '2023-01-01 12:00:00.000000', DATETIME_FORMAT_MS_WITH_TZ) + '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ) } }, {"_id": 0} @@ -169,7 +169,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): 0: 1 }, 'inserted_at': { - 0: '2023-01-01 12:00:00.000000' + 0: '2023-01-01 12:00:00.000000+0000' } } @@ -187,7 +187,7 @@ async def test_load_latest_data_error(mongodb_activity): await mongodb_activity.load_latest_data({ 'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}, 'collection_name': 'test_collection', - 'last_data_timestamp': '2023-01-01 12:00:00.000000' + 'last_data_timestamp': '2023-01-01 12:00:00.000000+0000' }) except Exception as e: assert str(e) == 'test' diff --git a/tests/workflow/sub_workflows/test_core_scouter.py b/tests/workflow/sub_workflows/test_core_scouter.py index 5e88c5b..f574325 100644 --- a/tests/workflow/sub_workflows/test_core_scouter.py +++ b/tests/workflow/sub_workflows/test_core_scouter.py @@ -2,6 +2,7 @@ from unittest.mock import AsyncMock, patch, call, ANY import pytest from scouter.workflow.sub_workflows.core_scouter import CoreScouter from scouter.activities.activities import Activities +from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ @pytest.fixture @@ -94,7 +95,12 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter): **expected_metadata, 'schema': 'test_schema', 'table_name': 'test_table', - 'data': 'held_data'}, + 'data': 'held_data', + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT_MS_WITH_TZ + } + }, retry_policy=ANY, start_to_close_timeout=ANY ) From 39ba82b539a630906a80edf3351c920f0a07fe4e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 22 Aug 2025 15:13:04 -0300 Subject: [PATCH 06/12] SIENTIAPDE-1193 Update image tag to 0.4.3 in values.yaml and refactor Redis key format for last data timestamp to use colons for improved readability. --- scouter/activities/redis.py | 4 ++-- values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index e7443a1..7b62ee2 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -27,7 +27,7 @@ class Redis(RedisBase): Gets the last data timestamp from redis. """ metadata = input_data['metadata'] - key = f"last_data_timestamp_{input_data['workflow_name']}_{input_data['schedule_name']}" + key = f"last_data_timestamp:{input_data['workflow_name']}:{input_data['schedule_name']}" self.info(f"Getting last data timestamp for {key}") @@ -60,7 +60,7 @@ class Redis(RedisBase): Puts the last data timestamp into redis. """ metadata = input_data['metadata'] - key = f"last_data_timestamp_{input_data['workflow_name']}_{input_data['schedule_name']}" + key = f"last_data_timestamp:{input_data['workflow_name']}:{input_data['schedule_name']}" self.info(f"Putting last data timestamp for {key}") diff --git a/values.yaml b/values.yaml index 813f223..4bb3331 100644 --- a/values.yaml +++ b/values.yaml @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.4.2" + tag: "0.4.3" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: From 54c3dd25e99d2c4ea8a04bd6c7cbcbf46e1c16b4 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 22 Aug 2025 15:30:47 -0300 Subject: [PATCH 07/12] SIENTIAPDE-1193 Update replica count in values.yaml to 1 and enhance timestamp handling in MongoDB activity by setting timezone to UTC for inserted_at field. --- scouter/activities/mongodb.py | 6 +++--- values.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index 878b185..a40cd83 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -3,7 +3,7 @@ from temporalio import workflow, activity with workflow.unsafe.imports_passed_through(): from typing import Any import traceback - from datetime import datetime + from datetime import datetime, timezone from pymongo import MongoClient from pandas import DataFrame from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler @@ -120,8 +120,8 @@ class MongoDB(BaseActivity): ) for item in data: - item['inserted_at'] = item['inserted_at'].strftime( - DATETIME_FORMAT_MS_WITH_TZ) + item['inserted_at'] = item['inserted_at'].replace( + tzinfo=timezone.utc).strftime(DATETIME_FORMAT_MS_WITH_TZ) self.info( f"Loaded {len(data)} documents from MongoDB", diff --git a/values.yaml b/values.yaml index 4bb3331..7a7d1f9 100644 --- a/values.yaml +++ b/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 3 +replicaCount: 1 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: From 7bbfa4dbacf1920f7fe6140fed8a7ca1d46cde13 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 22 Aug 2025 15:33:41 -0300 Subject: [PATCH 08/12] SIENTIAPDE-1193 Refactor timestamp handling in CoreScouter workflow by updating to DATETIME_FORMAT_WITH_TZ, ensuring consistency with recent changes in datetime management. --- scouter/workflow/sub_workflows/core_scouter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py index 3beb4d9..89bf7ff 100644 --- a/scouter/workflow/sub_workflows/core_scouter.py +++ b/scouter/workflow/sub_workflows/core_scouter.py @@ -5,7 +5,7 @@ with workflow.unsafe.imports_passed_through(): from typing import Any from datetime import timedelta from sientia_do.temporal.policies import retry_policy - from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ @workflow.defn(name="core_scouter") @@ -85,7 +85,7 @@ class CoreScouter: 'data': held_data, 'timestamp_conversion': { 'column': 'timestamp', - 'format': DATETIME_FORMAT_MS_WITH_TZ + 'format': DATETIME_FORMAT_WITH_TZ } }, retry_policy=retry_policy, From 826b559489c6b727519320d104c4f2a5d10a1ae4 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 22 Aug 2025 15:44:13 -0300 Subject: [PATCH 09/12] SIENTIAPDE-1193 Refactor CoreScouter workflow test to use updated DATETIME_FORMAT_WITH_TZ, ensuring consistency with recent changes in timestamp handling. --- tests/workflow/sub_workflows/test_core_scouter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/workflow/sub_workflows/test_core_scouter.py b/tests/workflow/sub_workflows/test_core_scouter.py index f574325..16ecff6 100644 --- a/tests/workflow/sub_workflows/test_core_scouter.py +++ b/tests/workflow/sub_workflows/test_core_scouter.py @@ -2,7 +2,7 @@ from unittest.mock import AsyncMock, patch, call, ANY import pytest from scouter.workflow.sub_workflows.core_scouter import CoreScouter from scouter.activities.activities import Activities -from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ +from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ @pytest.fixture @@ -98,7 +98,7 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter): 'data': 'held_data', 'timestamp_conversion': { 'column': 'timestamp', - 'format': DATETIME_FORMAT_MS_WITH_TZ + 'format': DATETIME_FORMAT_WITH_TZ } }, retry_policy=ANY, From 18f7ab6723e17eb52c6e5b03582f26e0669596bb Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 22 Aug 2025 15:46:14 -0300 Subject: [PATCH 10/12] SIENTIAPDE-1193 Update replica count in values.yaml from 1 to 3 to scale application deployment. --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index 7a7d1f9..4bb3331 100644 --- a/values.yaml +++ b/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 1 +replicaCount: 3 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: From 5c01d024a89c8970cb475e777500ff4dab35e1cc Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 25 Aug 2025 08:28:12 -0300 Subject: [PATCH 11/12] SIENTIAPDE-1193 Update image tag in values.yaml from 0.4.3 to 0.4.4 for version increment. --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index 4bb3331..e4a8620 100644 --- a/values.yaml +++ b/values.yaml @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.4.3" + tag: "0.4.4" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: From 719e3917ab2a3ff4dda379c4383c6dbc0727c7a3 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 25 Aug 2025 09:03:03 -0300 Subject: [PATCH 12/12] SIENTIAPDE-1193 Refactor Redis key format in tests for last data timestamp to use colons, enhancing readability and consistency with recent changes. --- coverage.sh | 1 + tests/activities/test_redis.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100755 coverage.sh diff --git a/coverage.sh b/coverage.sh new file mode 100755 index 0000000..fb1d880 --- /dev/null +++ b/coverage.sh @@ -0,0 +1 @@ +pytest --cov=sientia --cov-report=html && xdg-open htmlcov/index.html \ No newline at end of file diff --git a/tests/activities/test_redis.py b/tests/activities/test_redis.py index f68e48f..f87d46f 100644 --- a/tests/activities/test_redis.py +++ b/tests/activities/test_redis.py @@ -83,7 +83,7 @@ async def test_get_last_data_timestamp_not_none(redis_activity): result = await redis_activity.get_last_data_timestamp(test_data) redis_activity.get.assert_called_once_with( - 'last_data_timestamp_test_pipeline_test_schedule' + 'last_data_timestamp:test_pipeline:test_schedule' ) assert result == '2023-01-01 12:00:00' @@ -163,7 +163,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity): assert result == '2023-01-01 12:00:01' redis_activity.set.assert_called_once_with( - 'last_data_timestamp_test_pipeline_test_schedule', + 'last_data_timestamp:test_pipeline:test_schedule', '2023-01-01 12:00:01', ttl=18000 )