gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1646

Browse Source 
Update project configuration and dependencies

- Added .mypy_cache and .cursor to .gitignore.
- Changed asyncio_default_fixture_loop_scope and asyncio_default_test_loop_scope to "session" in pyproject.toml.
- Updated e2e testing dependencies in requirements-dev.txt, replacing fakeredis and mongomock with pytest-httpserver.
- Updated requirements.txt to use sientia_do instead of a specific git commit.
- Modified sonar-project.properties to remove a file from coverage exclusions.
- Enhanced E2E test fixtures in e2e/conftest.py for better container management.
- Cleaned up e2e test files related to CoreScouter and PIWebAPIScouter workflows.
This commit is contained in:
vitor-aignosi
2026-05-25 12:58:03 -03:00
parent 909ad25b63
commit 34dbc886f3
65 changed files with 2591 additions and 2849 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scouter/activities/activities.py
View File

@@ -7,7 +7,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import Logger
from sientia_do.temporal.activities.postgres import Postgres
from sientia_do.temporal.activities.postgres_sync import Postgres
from scouter.activities.api import API
from scouter.activities.gates import Gates

8
scouter/activities/api.py
View File

@@ -9,7 +9,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.pi_web_api_client import PIWebAPIClient
from sientia_do.repository.pi_web_api_client_sync import PIWebAPIClient
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
@@ -78,7 +78,7 @@ class API(SientiaMonitoring):
SientiaMonitoring.shutdown(self)
@activity.defn(name='get_tag_values')
async def get_tag_values(self, input_data: dict[str, Any]) -> list[dict]:
def get_tag_values(self, input_data: dict[str, Any]) -> list[dict]:
"""
Retrieve tag values from PI Web API for specified WebIds.
@@ -123,7 +123,7 @@ class API(SientiaMonitoring):
self.info(f'Getting tag values from {endpoint}', metadata=metadata)
self.debug(f'Web IDs: {web_ids}', metadata=metadata)
try:
latest_values = await self.pi_web_api_client.get_latest_values_df(
latest_values = self.pi_web_api_client.get_latest_values_df(
endpoint=endpoint,
web_ids=web_ids,
start_time=period,
@@ -134,7 +134,7 @@ class API(SientiaMonitoring):
)
except Exception as e:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='PI_WEB_API_REQUEST_ERROR',
message=f'Error getting tag values from PI Web API: {e}',

50
scouter/activities/gates.py
View File

@@ -59,7 +59,7 @@ class Gates(SientiaMonitoring):
"""
SientiaMonitoring.shutdown(self)
async def apply_aggregation(
def apply_aggregation(
self, values: DataFrame, aggr_function: str, metadata: dict[str, Any]
) -> float | None | str:
"""
@@ -82,31 +82,16 @@ class Gates(SientiaMonitoring):
Raises:
NotificationError: If invalid aggregation function is specified
"""
# Fast path for single value
if len(values) == 1:
return values['value'].iloc[0]
if aggr_function == 'lts':
return values['value'].iloc[-1]
# Remove NaN values without inplace operation
clean_values = values['value'].dropna()
if clean_values.empty:
return None
# Use dictionary lookup for aggregation functions (faster than if-elif chain)
aggregation_map = {
'lts': lambda x: x.iloc[-1],
'avg': lambda x: x.mean(),
'mdn': lambda x: x.median(),
'max': lambda x: x.max(),
'min': lambda x: x.min(),
}
if aggr_function in aggregation_map:
return aggregation_map[aggr_function](clean_values)
else:
await self.send_notification_async(
if aggr_function not in aggregation_map:
self.send_notification(
metadata=metadata,
notification_id='AGGREGATION_ISSUES',
message=f'Invalid aggregation function: {aggr_function}',
@@ -116,8 +101,21 @@ class Gates(SientiaMonitoring):
)
return 'continue'
if len(values) == 1:
return values['value'].iloc[0]
if aggr_function == 'lts':
return aggregation_map['lts'](values['value'])
clean_values = values['value'].dropna()
if clean_values.empty:
return None
return aggregation_map[aggr_function](clean_values)
@activity.defn(name='aggregate_data')
async def aggregate_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
def aggregate_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
"""
Aggregate time-series data by tag and name using specified functions.
@@ -165,7 +163,7 @@ class Gates(SientiaMonitoring):
# Get the latest timestamp (last row since data is sorted)
latest_timestamp = group['timestamp'].iloc[-1]
aggr_value = await self.apply_aggregation(group, aggr_function, metadata)
aggr_value = self.apply_aggregation(group, aggr_function, metadata)
if aggr_value == 'continue':
continue
@@ -197,7 +195,7 @@ class Gates(SientiaMonitoring):
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='AGGREGATION_ISSUES',
message=f'Error aggregating data: {e}',
@@ -210,7 +208,7 @@ class Gates(SientiaMonitoring):
raise e
@activity.defn(name='data_quality_gate')
async def data_quality_gate(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
def data_quality_gate(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
"""
Apply data quality filters to incoming data.
@@ -255,7 +253,7 @@ class Gates(SientiaMonitoring):
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='DATA_QUALITY_GATE_ISSUES',
message=f'Error applying filter {filter_name}: {e}',
@@ -273,7 +271,7 @@ class Gates(SientiaMonitoring):
message = f'{len(filtered_data)} rows has quality issues: {filter_name}: {policy}'
attachment = filtered_data.to_string()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'DATA_QUALITY_GATE_ISSUES__{filter_name}',
message=message,
@@ -290,7 +288,7 @@ class Gates(SientiaMonitoring):
return data.to_dict()
@activity.defn(name='write_metrics')
async def write_metrics(self, input_data: dict[str, Any]) -> None:
def write_metrics(self, input_data: dict[str, Any]) -> None:
"""
Write metrics to the database.
input_data:

8
scouter/activities/mongodb.py
View File

@@ -12,7 +12,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.mongodb_repository import MongoDBRepository
from sientia_do.repository.mongodb_repository_sync import MongoDBRepository
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
@@ -84,7 +84,7 @@ class MongoDB(SientiaMonitoring):
self.close()
@activity.defn(name='load_latest_data')
async def load_latest_data(self, input_data: dict[str, Any]) -> list[dict[str, Any]]:
def load_latest_data(self, input_data: dict[str, Any]) -> list[dict[str, Any]]:
"""
Load the latest data from MongoDB collection since a specified timestamp.
@@ -123,7 +123,7 @@ class MongoDB(SientiaMonitoring):
self.debug(f'Data filter: {data_filter}', metadata=metadata)
data = await self.mongodb_repository.find(
data = self.mongodb_repository.find(
collection_name=collection_name,
filters=data_filter,
metadata=metadata,
@@ -143,7 +143,7 @@ class MongoDB(SientiaMonitoring):
return data
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='MONGO_LOAD_ERROR',
message=f'Error loading data from MongoDB: {e}',

32
scouter/activities/redis.py
View File

@@ -11,7 +11,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.redis_repository import RedisRepository
from sientia_do.repository.redis_repository_sync import RedisRepository
from sientia_do.temporal.constants import DATETIME_FORMAT, now
@@ -70,7 +70,7 @@ class Redis(SientiaMonitoring):
SientiaMonitoring.shutdown(self)
@activity.defn(name='get_last_data_timestamp')
async def get_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
def get_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
"""
Retrieve the last processed data timestamp from Redis.
@@ -97,9 +97,9 @@ class Redis(SientiaMonitoring):
self.info(f'Getting last data timestamp for {key}', metadata=metadata)
try:
data_hold = await self.redis_repository.get(key, metadata=metadata)
data_hold = self.redis_repository.get(key, metadata=metadata)
except Exception as e:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='REDIS_GET_ERROR',
message=f'Error getting last data timestamp: {e}',
@@ -117,7 +117,7 @@ class Redis(SientiaMonitoring):
return data_hold
@activity.defn(name='put_last_data_timestamp')
async def put_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
def put_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
"""
Store the last processed data timestamp in Redis.
@@ -155,11 +155,9 @@ class Redis(SientiaMonitoring):
self.info(f'Last collected timestamp to insert: {last_data_timestamp}', metadata=metadata)
try:
await self.redis_repository.set(
key, last_data_timestamp, ttl=60 * 60 * 5, metadata=metadata
)
self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5, metadata=metadata)
except Exception as e:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='REDIS_SET_ERROR',
message=f'Error setting last data timestamp: {e}',
@@ -172,7 +170,7 @@ class Redis(SientiaMonitoring):
return last_data_timestamp
@activity.defn(name='group_and_hold_data')
async def group_and_hold_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
def group_and_hold_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
"""
Group data by tags and store temporarily in Redis with TTL.
@@ -210,9 +208,9 @@ class Redis(SientiaMonitoring):
self.info(f'Getting held data for {key}', metadata=metadata)
try:
data_hold = await self.redis_repository.get(key, metadata=metadata)
data_hold = self.redis_repository.get(key, metadata=metadata)
except Exception as e:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='REDIS_GET_ERROR',
message=f'Error getting held data: {e}',
@@ -254,7 +252,7 @@ class Redis(SientiaMonitoring):
data['timestamp'].max() if not data.empty else data_hold['timestamp']
)
await self.redis_repository.set(key, data_hold, ttl=retention_time, metadata=metadata)
self.redis_repository.set(key, data_hold, ttl=retention_time, metadata=metadata)
data_hold_df = DataFrame(data_hold, index=[0])
data_hold_melted = data_hold_df.melt(
@@ -264,7 +262,7 @@ class Redis(SientiaMonitoring):
data_hold_melted.reset_index(drop=True, inplace=True)
except Exception as e:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='REDIS_SET_ERROR',
message=f'Error setting held data: {e}',
@@ -281,7 +279,7 @@ class Redis(SientiaMonitoring):
return data_hold_melted.to_dict()
@activity.defn(name='store_data_package')
async def store_data_package(self, input_data: dict[str, Any]):
def store_data_package(self, input_data: dict[str, Any]):
"""
Stores the data package in redis. It's a debug feature and must be toggled on.
input_data:
@@ -300,9 +298,9 @@ class Redis(SientiaMonitoring):
cache = {'data': data.to_dict(), 'held_data': held_data.to_dict()}
try:
await self.redis_repository.set(key, cache, ttl=120, metadata=metadata)
self.redis_repository.set(key, cache, ttl=120, metadata=metadata)
except Exception as e:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='REDIS_SET_ERROR',
message=f'Error setting data package: {e}',