Code import - branch feature/SIENTIAPDE-1646

scouter/activities/activities.py

@@ -0,0 +1,132 @@
+from sientia_do.observability.metrics_controller import MetricsController
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from os import getenv
+    from typing import Any
+
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+    from sientia_do.observability.logger import Logger
+    from sientia_do.temporal.activities.postgres_sync import Postgres
+
+    from scouter.activities.api import API
+    from scouter.activities.gates import Gates
+    from scouter.activities.mongodb import MongoDB
+    from scouter.activities.redis import Redis
+
+
+class Activities(Postgres, Redis, Gates, MongoDB, API):
+    """
+    Unified activities class that combines multiple data processing services.
+
+    This class provides a comprehensive interface for all data processing activities
+    by inheriting from specialized service classes. It handles:
+    - PostgreSQL operations for data persistence
+    - Redis operations for caching and temporary storage
+    - Data quality gates and filtering
+    - MongoDB operations for data retrieval
+    - PI Web API operations for external data ingestion
+    - Notification handling and logging
+
+    The class implements the multiple inheritance pattern to provide a unified
+    interface while maintaining separation of concerns across different data services.
+    """
+
+    def __init__(
+        self,
+        postgres_config: dict[str, Any],
+        redis_config: dict[str, Any],
+        mongodb_config: dict[str, Any],
+        api_config: dict[str, Any],
+        logger: Logger,
+        notification_handler: NotificationHandler,
+    ):
+        """
+        Initialize the Activities class with all required services.
+
+        Args:
+            postgres_config (dict[str, Any]): PostgreSQL connection configuration.
+                Required fields: host, port, user, password, dbname, min_connections, max_connections
+            redis_config (dict[str, Any]): Redis connection configuration.
+                Required fields: host, port, username, password
+            mongodb_config (dict[str, Any]): MongoDB connection configuration.
+                Required fields: connection_string, database_name
+            api_config (dict[str, Any]): PI Web API configuration.
+                Required fields: base_url, auth_type, auth_token
+            logger (Logger): Logger instance for application logging
+            notification_handler (NotificationHandler): Handler for system notifications
+        """
+        metrics_controller = MetricsController(
+            logger=logger,
+        )
+
+        # Initialize Postgres
+        Postgres.__init__(
+            self,
+            host=postgres_config['host'],
+            port=postgres_config['port'],
+            user=postgres_config['user'],
+            password=postgres_config['password'],
+            dbname=postgres_config['dbname'],
+            min_connections=postgres_config['min_connections'],
+            max_connections=postgres_config['max_connections'],
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+        # Initialize Redis
+        Redis.__init__(
+            self,
+            host=redis_config['host'],
+            port=redis_config['port'],
+            logger=logger,
+            notification_handler=notification_handler,
+            username=redis_config['username'],
+            password=redis_config['password'],
+            metrics_controller=metrics_controller,
+        )
+
+        # Initialize Gates
+        Gates.__init__(
+            self,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+        # Initialize MongoDB
+        MongoDB.__init__(
+            self,
+            connection_string=mongodb_config['connection_string'],
+            database_name=mongodb_config['database_name'],
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+        # Initialize API
+        API.__init__(
+            self,
+            base_url=api_config['base_url'],
+            auth_type=api_config['auth_type'],
+            auth_token=api_config['auth_token'],
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+        self.pod_id = getenv('HOSTNAME', 'localhost')
+
+    def shutdown(self):
+        """
+        Gracefully shutdown all service connections.
+
+        This method ensures proper cleanup of database connections and resources
+        to prevent connection leaks and ensure graceful application termination.
+        """
+        Postgres.close(self)
+        MongoDB.close(self)
+        Redis.close(self)
+        Gates.close(self)
+        API.close(self)

scouter/activities/api.py

@@ -0,0 +1,157 @@
+from temporalio import activity, workflow
+
+with workflow.unsafe.imports_passed_through():
+    import traceback
+    from typing import Any
+
+    from sientia_do.notifications.handlers import NotificationHandler
+    from sientia_do.notifications.models import NotificationLevel
+    from sientia_do.observability.logger import Logger
+    from sientia_do.observability.metrics_controller import MetricsController
+    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+    from sientia_do.repository.pi_web_api_client_sync import PIWebAPIClient
+    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
+
+
+class API(SientiaMonitoring):
+    """
+    PI Web API operations for data retrieval.
+
+    This class provides Temporal activities for interacting with the PI Web API
+    to retrieve tag values and historical data. It implements:
+    - Tag value retrieval from PI Web API endpoints
+    - Data quality filtering and validation
+    - Error handling with notifications
+    - Metrics collection for monitoring
+
+    The class wraps the PIWebAPIClient to provide Temporal-aware activity methods
+    that can be used in workflow orchestration.
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        auth_type: str,
+        auth_token: str,
+        logger: Logger,
+        notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
+    ) -> None:
+        """
+        Initialize API activity with PI Web API client.
+
+        Args:
+            base_url (str): Base URL of the PI Web API server
+            auth_type (str): Authentication type ('basic' or 'bearer')
+            auth_token (str): Authentication token
+            logger (Logger): Logger instance for operation logging
+            notification_handler (NotificationHandler): Handler for system notifications
+            metrics_controller (MetricsController): Controller for metrics collection
+        """
+        SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
+        self.pi_web_api_client = PIWebAPIClient(
+            base_url=base_url,
+            auth_config={
+                'type': auth_type,
+                'token': auth_token,
+            },
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+            headers_config={
+                'Content-Type': 'application/json',
+                'Accept': 'application/json',
+                'x-requested-with': 'piwebapistreams',
+                'User-Agent': 'Aig-Scouter-Agent/1.0',
+            },
+        )
+
+    def close(self) -> None:
+        """
+        Close the PI Web API client and shutdown monitoring services.
+
+        This method performs cleanup operations:
+        - Closes the PI Web API client connection
+        - Shuts down SientiaMonitoring services (metrics, notifications)
+        """
+        self.pi_web_api_client.close()
+        SientiaMonitoring.shutdown(self)
+
+    @activity.defn(name='get_tag_values')
+    def get_tag_values(self, input_data: dict[str, Any]) -> list[dict]:
+        """
+        Retrieve tag values from PI Web API for specified WebIds.
+
+        This activity fetches historical or real-time data from the PI Web API
+        for a set of configured tags. It returns the data as a list of dictionaries
+        suitable for further processing in the workflow.
+
+        The timestamps are normalized to ensure consistency across all records in the
+        response. After converting timestamps to string format, all timestamps are
+        set to the maximum timestamp value (lexicographically) found in the dataset.
+        This ensures all records in a single batch share the same timestamp value.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - endpoint (str): PI Web API endpoint path
+                - web_ids (dict[str, str | None]): Tag names mapped to WebIds
+                - period (dict[str, str]): Time period with 'start_time' field
+                - api_timeout (int): Request timeout in seconds
+                - max_count (int, optional): Maximum data points per tag. Defaults to 1
+
+        Returns:
+            list[dict]: List of data records, each containing:
+                - timestamp: Normalized timestamp string (all records share the same value)
+                - name: Tag name
+                - value: Numeric value
+                - tag: WebId
+
+        Raises:
+            PIMSRequestError: If API request fails
+            Exception: If data retrieval or processing fails
+        """
+        metadata = input_data['metadata']
+        endpoint = input_data['endpoint']
+        web_ids = input_data['web_ids']
+        period = input_data['period']
+        end_time = input_data.get('end_time', '*')
+        max_count = input_data.get('max_count', 1)
+        api_timeout = input_data['api_timeout']
+
+        self.info(f'Getting tag values from {endpoint}', metadata=metadata)
+        self.debug(f'Web IDs: {web_ids}', metadata=metadata)
+        try:
+            latest_values = self.pi_web_api_client.get_latest_values_df(
+                endpoint=endpoint,
+                web_ids=web_ids,
+                start_time=period,
+                end_time=end_time,
+                max_count=max_count,
+                metadata=metadata,
+                request_timeout=api_timeout,
+            )
+
+        except Exception as e:
+            self.send_notification(
+                metadata=metadata,
+                notification_id='PI_WEB_API_REQUEST_ERROR',
+                message=f'Error getting tag values from PI Web API: {e}',
+                block='get_tag_values',
+                level=NotificationLevel.ERROR,
+                attachment_content=traceback.format_exc(),
+            )
+            raise e
+
+        latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
+
+        self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata)
+
+        # Normalize the package timestamp
+        valid_timestamp_values = latest_values['timestamp'].dropna()
+        latest_values['timestamp'] = valid_timestamp_values.max()
+
+        self.info(f'Gathered {len(latest_values)} tag values', metadata=metadata)
+
+        return latest_values.to_dict(orient='records')

scouter/activities/gates.py

@@ -0,0 +1,319 @@
+from collections.abc import Hashable
+
+from sientia_do.observability.metrics_controller import MetricsController
+from temporalio import activity, workflow
+
+with workflow.unsafe.imports_passed_through():
+    import traceback
+    from typing import Any
+
+    from pandas import DataFrame
+    from sientia_do.notifications.handlers import NotificationHandler
+    from sientia_do.notifications.models import NotificationLevel
+    from sientia_do.observability.logger import Logger
+    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+
+    from scouter import metrics
+    from scouter.utils.quality.filters import null_values_filter, out_of_bounds_filter
+
+quality_gate_filters = {
+    'NULL_VALUES_FILTER': null_values_filter,
+    'OUT_OF_BOUNDS_FILTER': out_of_bounds_filter,
+}
+
+
+class Gates(SientiaMonitoring):
+    """
+    Data quality gates and filtering operations.
+
+    This class implements data quality validation and filtering for industrial
+    time-series data. It provides:
+    - Configurable data quality filters
+    - Data aggregation functions for time-series data
+    - Comprehensive error handling and notification
+    - Metrics collection for quality monitoring
+
+    The class supports multiple aggregation strategies and quality filters to
+    ensure data integrity and enable flexible data processing workflows.
+    """
+
+    def __init__(
+        self,
+        logger: Logger,
+        notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
+    ):
+        """
+        Initialize the Gates class with logging and notification services.
+
+        Args:
+            logger (Logger): Logger instance for operation logging
+            notification_handler (NotificationHandler): Handler for system notifications
+            metrics_controller (MetricsController): Metrics controller instance
+        """
+        SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
+
+    def close(self):
+        """
+        Close the Gates class.
+        """
+        SientiaMonitoring.shutdown(self)
+
+    def apply_aggregation(
+        self, values: DataFrame, aggr_function: str, metadata: dict[str, Any]
+    ) -> float | None | str:
+        """
+        Apply aggregation function to a group of time-series data.
+
+        This method applies the specified aggregation function to a group of
+        data points. It handles edge cases and provides comprehensive error
+        reporting for invalid aggregation functions.
+
+        Args:
+            values (DataFrame): Group of data points to aggregate (pre-sorted by timestamp)
+            aggr_function (str): Aggregation function to apply.
+                Supported functions: 'lts' (latest), 'avg' (average), 'mdn' (median),
+                'max' (maximum), 'min' (minimum)
+            metadata (dict[str, Any]): Workflow metadata for error reporting
+
+        Returns:
+            float | None | str: Aggregated value, None if no valid data, or 'continue' for errors
+
+        Raises:
+            NotificationError: If invalid aggregation function is specified
+        """
+        aggregation_map = {
+            'lts': lambda x: x.iloc[-1],
+            'avg': lambda x: x.mean(),
+            'mdn': lambda x: x.median(),
+            'max': lambda x: x.max(),
+            'min': lambda x: x.min(),
+        }
+
+        if aggr_function not in aggregation_map:
+            self.send_notification(
+                metadata=metadata,
+                notification_id='AGGREGATION_ISSUES',
+                message=f'Invalid aggregation function: {aggr_function}',
+                block='aggregate_data',
+                level=NotificationLevel.ERROR,
+                attachment_content=traceback.format_exc(),
+            )
+            return 'continue'
+
+        if len(values) == 1:
+            return values['value'].iloc[0]
+
+        if aggr_function == 'lts':
+            return aggregation_map['lts'](values['value'])
+
+        clean_values = values['value'].dropna()
+
+        if clean_values.empty:
+            return None
+
+        return aggregation_map[aggr_function](clean_values)
+
+    @activity.defn(name='aggregate_data')
+    def aggregate_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
+        """
+        Aggregate time-series data by tag and name using specified functions.
+
+        This activity processes time-series data by grouping it by tag and name,
+        then applying the configured aggregation functions. It handles data
+        validation and provides comprehensive error reporting.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - data (dict[str, Any]): Time-series data to aggregate
+                - model_tags (dict[str, Any]): Tag configuration with aggregation functions
+
+        Returns:
+            dict[Hashable, Any]: Aggregated data organized by tag and name
+
+        Raises:
+            Exception: If aggregation operation fails
+        """
+
+        metadata = input_data['metadata']
+
+        try:
+            # Convert input data to DataFrame
+            df = DataFrame(input_data['data'])
+
+            self.info(f'Aggregating time series data for {len(df)} rows', metadata=metadata)
+
+            # Sort once by timestamp for all data (more efficient than sorting each group)
+            df = df.sort_values(['tag', 'name', 'timestamp'])
+
+            # Group by tag and name
+            # sort=False since we already sorted
+            grouped = df.groupby(['tag', 'name'], sort=False)
+
+            # Prepare aggregation functions mapping
+            model_tags = input_data['model_tags']
+
+            # Process groups efficiently
+            results = []
+            for (tag, name), group in grouped:
+                # Get the aggregation function from model_tags
+                aggr_function = model_tags.get(name, {}).get('aggr_func', 'lts')
+
+                # Get the latest timestamp (last row since data is sorted)
+                latest_timestamp = group['timestamp'].iloc[-1]
+
+                aggr_value = self.apply_aggregation(group, aggr_function, metadata)
+
+                if aggr_value == 'continue':
+                    continue
+
+                # Store the result directly in list for better performance
+                results.append(
+                    {
+                        'tag': tag,
+                        'name': name,
+                        'value': aggr_value,
+                        'timestamp': latest_timestamp,
+                        'aggregation_function': aggr_function,
+                    }
+                )
+
+            self.info(f'Aggregated data has {len(results)} rows', metadata=metadata)
+
+            # Convert to DataFrame only once at the end if we have results
+            if results:
+                result_df = DataFrame(results)
+
+                self.debug(f'Final aggregated data:\n{result_df.to_string()}', metadata=metadata)
+
+                return result_df.to_dict()
+            else:
+                # Return empty DataFrame dict structure
+                return DataFrame().to_dict()
+
+        except Exception as e:
+            trace = traceback.format_exc()
+
+            self.send_notification(
+                metadata=metadata,
+                notification_id='AGGREGATION_ISSUES',
+                message=f'Error aggregating data: {e}',
+                block='aggregate_data',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+
+            self.error(trace, metadata=metadata)
+            raise e
+
+    @activity.defn(name='data_quality_gate')
+    def data_quality_gate(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
+        """
+        Apply data quality filters to incoming data.
+
+        This activity applies configurable quality filters to validate incoming
+        data. It supports multiple filter types and provides comprehensive
+        error reporting for quality issues.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - data (dict[str, Any]): Data to validate
+                - filters (dict[str, str]): Filter configuration
+                - model_tags (dict[str, Any]): Tag-specific validation rules
+
+        Returns:
+            dict[Hashable, Any]: Filtered data that passes quality validation
+
+        Raises:
+            Exception: If quality validation fails
+        """
+
+        metadata = input_data['metadata']
+
+        filters = input_data['filters']
+        data = DataFrame(input_data['data'])
+        model_tags = input_data['model_tags']
+
+        self.info(f'Applying quality gate to data to {len(data)} rows', metadata=metadata)
+
+        tags = list(model_tags.keys())
+
+        data = data[data['name'].isin(tags)]
+
+        for filter_name, config in filters.items():
+            policy = config['policy']
+            if filter_name not in quality_gate_filters:
+                self.warning(f'Filter {filter_name} not found', metadata=metadata)
+                continue
+
+            try:
+                filtered_data = quality_gate_filters[filter_name](data, model_tags)
+
+            except Exception as e:
+                trace = traceback.format_exc()
+                self.send_notification(
+                    metadata=metadata,
+                    notification_id='DATA_QUALITY_GATE_ISSUES',
+                    message=f'Error applying filter {filter_name}: {e}',
+                    block='data_quality_gate',
+                    level=NotificationLevel.ERROR,
+                    attachment_content=trace,
+                )
+
+                self.error(trace, metadata=metadata)
+
+            else:
+                if filtered_data.empty:
+                    continue
+
+                message = f'{len(filtered_data)} rows has quality issues: {filter_name}: {policy}'
+                attachment = filtered_data.to_string()
+
+                self.send_notification(
+                    metadata=metadata,
+                    notification_id=f'DATA_QUALITY_GATE_ISSUES__{filter_name}',
+                    message=message,
+                    block='data_quality_gate',
+                    level=NotificationLevel.WARNING,
+                    attachment_content=attachment,
+                )
+
+                if policy == 'DISCARD':
+                    data = data[~data.index.isin(filtered_data.index)]
+
+        self.info(f'Data quality gate applied, final data has {len(data)} rows', metadata=metadata)
+
+        return data.to_dict()
+
+    @activity.defn(name='write_metrics')
+    def write_metrics(self, input_data: dict[str, Any]) -> None:
+        """
+        Write metrics to the database.
+        input_data:
+            metadata: dict[str, Any]
+        """
+        metadata = input_data['metadata']
+        tag_values = DataFrame(input_data['tag_values'])
+
+        self.info(f'Writing metrics for {metadata["model_name"]}', metadata=metadata)
+
+        metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels(
+            pod_id=self.pod_id,
+            model_name=metadata['model_name'],
+            workflow_name=metadata['workflow_name'],
+        ).inc()
+
+        # Register metrics
+        for _, row in tag_values.iterrows():
+            value = row['value']
+            if value is not None:
+                metrics.TAG_CHANGES_MONITOR.labels(
+                    pod_id=self.pod_id,
+                    model_name=metadata['model_name'],
+                    workflow_name=metadata['workflow_name'],
+                    tag_name=row['variable'],
+                ).set(row['value'])
+
+        self.info(f'Metrics written for {metadata["model_name"]}', metadata=metadata)

scouter/activities/mongodb.py

@@ -0,0 +1,154 @@
+from datetime import UTC
+
+from sientia_do.observability.metrics_controller import MetricsController
+from temporalio import activity, workflow
+
+with workflow.unsafe.imports_passed_through():
+    import traceback
+    from datetime import datetime
+    from typing import Any
+
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+    from sientia_do.notifications.models import NotificationLevel
+    from sientia_do.observability.logger import Logger
+    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+    from sientia_do.repository.mongodb_repository_sync import MongoDBRepository
+    from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
+
+
+class MongoDB(SientiaMonitoring):
+    """
+    MongoDB operations for data retrieval and storage.
+
+    This class provides MongoDB connectivity and operations for the Scouter system.
+    It handles:
+    - Connection management with automatic reconnection
+    - Data retrieval with timestamp-based filtering
+    - Document cleaning and preprocessing
+    - Error handling and notification integration
+
+    The class implements Temporal activities for MongoDB operations, enabling
+    distributed data processing with fault tolerance and monitoring.
+    """
+
+    def __init__(
+        self,
+        connection_string: str,
+        database_name: str,
+        logger: Logger,
+        notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
+    ):
+        """
+        Initialize MongoDB connection and services.
+
+        Args:
+            connection_string (str): MongoDB connection URI string
+            database_name (str): Name of the target database
+            logger (Logger): Logger instance for operation logging
+            notification_handler (NotificationHandler): Handler for system notifications
+
+        Raises:
+            ConnectionError: If MongoDB connection fails
+        """
+
+        self.mongodb_repository = MongoDBRepository(
+            connection_string=connection_string,
+            database_name=database_name,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+        SientiaMonitoring.__init__(
+            self,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+    def close(self):
+        """
+        Close the MongoDB connection.
+        """
+        self.mongodb_repository.close()
+        SientiaMonitoring.shutdown(self)
+
+    def __del__(self):
+        """
+        Destructor to ensure MongoDB client is closed.
+
+        This destructor ensures that MongoDB connections are properly closed
+        when the object is garbage collected, preventing resource leaks.
+        """
+        self.close()
+
+    @activity.defn(name='load_latest_data')
+    def load_latest_data(self, input_data: dict[str, Any]) -> list[dict[str, Any]]:
+        """
+        Load the latest data from MongoDB collection since a specified timestamp.
+
+        This activity retrieves data from a MongoDB collection, optionally
+        filtering by timestamp to enable incremental data processing. It
+        handles connection management and provides comprehensive error reporting.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - collection_name (str): Name of the MongoDB collection
+                - last_data_timestamp (str | None): Last processed timestamp for filtering
+
+        Returns:
+            dict[str, Any]: Retrieved data, or empty dict if no data found
+
+        Raises:
+            Exception: If MongoDB operation fails
+        """
+        metadata = input_data['metadata']
+        collection_name = input_data['collection_name']
+        last_data_timestamp = input_data['last_data_timestamp']
+
+        self.info(f'Loading data from MongoDB: {input_data}', metadata=metadata)
+
+        try:
+            if last_data_timestamp is None:
+                data_filter = {}
+            else:
+                data_filter = {
+                    'inserted_at': {
+                        '$gt': datetime.strptime(last_data_timestamp, DATETIME_FORMAT_MS_WITH_TZ)
+                    }
+                }
+
+            self.debug(f'Data filter: {data_filter}', metadata=metadata)
+
+            data = self.mongodb_repository.find(
+                collection_name=collection_name,
+                filters=data_filter,
+                metadata=metadata,
+            )
+
+            self.debug(f'Collected: {data}', metadata=metadata)
+
+            for item in data:
+                item['inserted_at'] = (
+                    item['inserted_at'].replace(tzinfo=UTC).strftime(DATETIME_FORMAT_MS_WITH_TZ)
+                )
+
+            self.info(f'Loaded {len(data)} documents from MongoDB', metadata=metadata)
+
+            self.debug(f'Loaded data: {data}', metadata=metadata)
+
+            return data
+        except Exception as e:
+            trace = traceback.format_exc()
+            self.send_notification(
+                metadata=metadata,
+                notification_id='MONGO_LOAD_ERROR',
+                message=f'Error loading data from MongoDB: {e}',
+                block='load_latest_data',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+            raise e

scouter/activities/redis.py

@@ -0,0 +1,311 @@
+from sientia_do.observability.metrics_controller import MetricsController
+from temporalio import activity, workflow
+
+with workflow.unsafe.imports_passed_through():
+    import traceback
+    from collections.abc import Hashable
+    from typing import Any
+
+    from pandas import DataFrame
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+    from sientia_do.notifications.models import NotificationLevel
+    from sientia_do.observability.logger import Logger
+    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+    from sientia_do.repository.redis_repository_sync import RedisRepository
+    from sientia_do.temporal.constants import DATETIME_FORMAT, now
+
+
+class Redis(SientiaMonitoring):
+    """
+    Redis operations for data caching and temporary storage.
+
+    This class extends the base Redis functionality to provide specialized
+    operations for the Scouter system, including:
+    - Data timestamp management for incremental processing
+    - Temporary data storage with configurable TTL
+    - Data grouping and holding for batch processing
+    - Error handling and notification integration
+
+    The class implements Temporal activities for Redis operations, enabling
+    distributed data processing with fault tolerance and monitoring.
+    """
+
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        username: str,
+        password: str,
+        logger: Logger,
+        notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
+    ):
+        """
+        Initialize Redis connection and services.
+
+        Args:
+            host (str): Redis server hostname or IP address
+            port (int): Redis server port number
+            username (str): Redis authentication username
+            password (str): Redis authentication password
+            logger (Logger): Logger instance for operation logging
+            notification_handler (NotificationHandler): Handler for system notifications
+        """
+        SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
+        self.redis_repository = RedisRepository(
+            host=host,
+            port=port,
+            username=username,
+            password=password,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+    def close(self):
+        """
+        Close the Redis connection.
+        """
+        self.redis_repository.close()
+        SientiaMonitoring.shutdown(self)
+
+    @activity.defn(name='get_last_data_timestamp')
+    def get_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
+        """
+        Retrieve the last processed data timestamp from Redis.
+
+        This activity retrieves the timestamp of the last successfully processed
+        data point for a specific workflow and schedule combination. It's used
+        for incremental data processing to avoid reprocessing the same data.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - workflow_name (str): Name of the workflow
+                - schedule_name (str): Name of the data collection schedule
+
+        Returns:
+            str | None: Last processed timestamp string, or None if no previous data exists
+
+        Raises:
+            Exception: If Redis operation fails
+        """
+        metadata = input_data['metadata']
+        key = f'last_data_timestamp:{input_data["workflow_name"]}:{input_data["schedule_name"]}'
+
+        self.info(f'Getting last data timestamp for {key}', metadata=metadata)
+
+        try:
+            data_hold = self.redis_repository.get(key, metadata=metadata)
+        except Exception as e:
+            self.send_notification(
+                metadata=metadata,
+                notification_id='REDIS_GET_ERROR',
+                message=f'Error getting last data timestamp: {e}',
+                block='get_last_data_timestamp',
+                level=NotificationLevel.ERROR,
+                attachment_content=traceback.format_exc(),
+            )
+            raise e
+
+        self.info(f'Last collected timestamp: {data_hold}', metadata=metadata)
+
+        if not data_hold:
+            return None
+
+        return data_hold
+
+    @activity.defn(name='put_last_data_timestamp')
+    def put_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
+        """
+        Store the last processed data timestamp in Redis.
+
+        This activity stores the timestamp of the most recent data point that
+        has been successfully processed. The timestamp is used for incremental
+        data loading in subsequent workflow executions.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - data (dict[str, Any]): Processed data to extract timestamp from
+                - workflow_name (str): Name of the workflow
+                - schedule_name (str): Name of the data collection schedule
+
+        Returns:
+            str | None: The timestamp that was stored, or None if no data was processed
+
+        Raises:
+            Exception: If Redis operation fails
+        """
+        metadata = input_data['metadata']
+        key = f'last_data_timestamp:{input_data["workflow_name"]}:{input_data["schedule_name"]}'
+
+        self.info(f'Putting last data timestamp for {key}', metadata=metadata)
+
+        data = DataFrame(input_data['data'])
+
+        if data.empty:
+            self.warning('No data to insert', metadata=metadata)
+            return None
+
+        last_data_timestamp = data['inserted_at'].max()
+
+        self.info(f'Last collected timestamp to insert: {last_data_timestamp}', metadata=metadata)
+
+        try:
+            self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5, metadata=metadata)
+        except Exception as e:
+            self.send_notification(
+                metadata=metadata,
+                notification_id='REDIS_SET_ERROR',
+                message=f'Error setting last data timestamp: {e}',
+                block='put_last_data_timestamp',
+                level=NotificationLevel.ERROR,
+                attachment_content=traceback.format_exc(),
+            )
+            raise e
+
+        return last_data_timestamp
+
+    @activity.defn(name='group_and_hold_data')
+    def group_and_hold_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
+        """
+        Group data by tags and store temporarily in Redis with TTL.
+
+        This activity organizes processed data by tag names and stores it in Redis
+        with a configurable retention period. The data is grouped to enable
+        efficient batch processing and export operations.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - schedule_name (str): Name of the data collection schedule
+                - workflow_name (str): Name of the workflow
+                - data (dict[str, Any]): Data to group and store
+                - model_id (str): Unique model identifier
+                - model_tags (dict[str, Any]): Tag configuration
+                - retention_time (int): Data retention period in seconds
+
+        Returns:
+            dict[str, Any]: Grouped data organized by tag names
+
+        Raises:
+            Exception: If Redis operation fails
+        """
+        metadata = input_data['metadata']
+
+        self.debug('Grouping and holding data...', metadata=metadata)
+        data = DataFrame(input_data['data'])
+        model_tags = input_data['model_tags']
+        retention_time = input_data['retention_time']
+        fill_missing_tags = input_data['fill_missing_tags']
+
+        key = f'held_data_{input_data["workflow_name"]}_{input_data["schedule_name"]}'
+
+        self.info(f'Getting held data for {key}', metadata=metadata)
+
+        try:
+            data_hold = self.redis_repository.get(key, metadata=metadata)
+        except Exception as e:
+            self.send_notification(
+                metadata=metadata,
+                notification_id='REDIS_GET_ERROR',
+                message=f'Error getting held data: {e}',
+                block='group_and_hold_data',
+                level=NotificationLevel.ERROR,
+                attachment_content=traceback.format_exc(),
+            )
+            raise e
+
+        if not data_hold:
+            data_hold = {}
+            if data.empty:
+                self.warning('No data to export', metadata=metadata)
+                return data_hold
+
+        self.info(f'Grouping and holding data for {len(data)} rows')
+
+        try:
+            # Remove possibly removed tags
+            tags = list(model_tags.keys())
+            tags.append('timestamp')
+            self.debug(f'Tags to keep: {tags}', metadata=metadata)
+            data_hold = {tag: content for tag, content in data_hold.items() if tag in tags}
+            self.debug(f'Data hold after removing removed tags: {data_hold}', metadata=metadata)
+
+            for _, row in data.iterrows():
+                value = row['value']
+
+                data_hold[row['name']] = value
+
+            if fill_missing_tags:
+                self.debug('Filling missing tags in data package', metadata=metadata)
+                missing_tags = [tag for tag in tags if tag not in list(data_hold.keys())]
+
+                for tag in missing_tags:
+                    data_hold[tag] = None
+
+            data_hold['timestamp'] = (
+                data['timestamp'].max() if not data.empty else data_hold['timestamp']
+            )
+
+            self.redis_repository.set(key, data_hold, ttl=retention_time, metadata=metadata)
+
+            data_hold_df = DataFrame(data_hold, index=[0])
+            data_hold_melted = data_hold_df.melt(
+                id_vars='timestamp', var_name='variable', value_name='value'
+            )
+            data_hold_melted['model_id'] = input_data['model_id']
+
+            data_hold_melted.reset_index(drop=True, inplace=True)
+        except Exception as e:
+            self.send_notification(
+                metadata=metadata,
+                notification_id='REDIS_SET_ERROR',
+                message=f'Error setting held data: {e}',
+                block='group_and_hold_data',
+                level=NotificationLevel.ERROR,
+                attachment_content=traceback.format_exc(),
+            )
+            raise e
+
+        self.info(f'Data held and melted has {len(data_hold_melted)} rows')
+
+        self.debug(f'Data held and melted:\n {data_hold_melted.to_string()}', metadata=metadata)
+
+        return data_hold_melted.to_dict()
+
+    @activity.defn(name='store_data_package')
+    def store_data_package(self, input_data: dict[str, Any]):
+        """
+        Stores the data package in redis. It's a debug feature and must be toggled on.
+        input_data:
+            metadata: The metadata of the workflow.
+            workflow_name: The name of the workflow.
+            schedule_name: The name of the schedule.
+            held_data: The final scouter output.
+            data: The data used to collect the data.
+        """
+        metadata = input_data['metadata']
+        key = f'data_package_{input_data["workflow_name"]}_{input_data["schedule_name"]}_{now().strftime(DATETIME_FORMAT)}'
+
+        data = DataFrame(input_data['data'])
+        held_data = DataFrame(input_data['held_data'])
+
+        cache = {'data': data.to_dict(), 'held_data': held_data.to_dict()}
+
+        try:
+            self.redis_repository.set(key, cache, ttl=120, metadata=metadata)
+        except Exception as e:
+            self.send_notification(
+                metadata=metadata,
+                notification_id='REDIS_SET_ERROR',
+                message=f'Error setting data package: {e}',
+                block='store_data_package',
+                level=NotificationLevel.ERROR,
+                attachment_content=traceback.format_exc(),
+            )
+            raise e