SIENTIAPDE-1084

Remove deprecated files and configurations, including .env, Dockerfile, docker-compose.yml, and client-schedule.py. Update README.md to reflect new architecture and features, enhancing clarity on system capabilities and workflows. Adjust values.yaml for image tag and replica count, and improve code documentation across various modules for better maintainability.

scouter/activities/activities.py

@@ -11,8 +11,21 @@ with workflow.unsafe.imports_passed_through():
     from os import getenv
 
 
-class Activities(Postgres, Redis, Gates, MongoDB,):
-    """Activities class that combines multiple services with proper initialization."""
+class Activities(Postgres, Redis, Gates, MongoDB):
+    """
+    Unified activities class that combines multiple data processing services.
+
+    This class provides a comprehensive interface for all data processing activities
+    by inheriting from specialized service classes. It handles:
+    - PostgreSQL operations for data persistence
+    - Redis operations for caching and temporary storage
+    - Data quality gates and filtering
+    - MongoDB operations for data retrieval
+    - Notification handling and logging
+
+    The class implements the multiple inheritance pattern to provide a unified
+    interface while maintaining separation of concerns across different data services.
+    """
 
     def __init__(self,
                  postgres_config: dict[str, Any],
@@ -20,7 +33,19 @@ class Activities(Postgres, Redis, Gates, MongoDB,):
                  mongodb_config: dict[str, Any],
                  logger: Logger,
                  notification_handler: NotificationHandler):
+        """
+        Initialize the Activities class with all required services.
 
+        Args:
+            postgres_config (dict[str, Any]): PostgreSQL connection configuration.
+                Required fields: host, port, user, password, dbname, min_connections, max_connections
+            redis_config (dict[str, Any]): Redis connection configuration.
+                Required fields: host, port, username, password
+            mongodb_config (dict[str, Any]): MongoDB connection configuration.
+                Required fields: connection_string, database_name
+            logger (Logger): Logger instance for application logging
+            notification_handler (NotificationHandler): Handler for system notifications
+        """
         # Initialize Postgres
         Postgres.__init__(
             self,
@@ -65,5 +90,11 @@ class Activities(Postgres, Redis, Gates, MongoDB,):
         self.pod_id = getenv("HOSTNAME", "localhost")
 
     def shutdown(self):
+        """
+        Gracefully shutdown all service connections.
+
+        This method ensures proper cleanup of database connections and resources
+        to prevent connection leaks and ensure graceful application termination.
+        """
         Postgres.close(self)
         MongoDB.shutdown(self)

scouter/activities/faker.py

@@ -11,8 +11,30 @@ from sientia_do.observability.logger import Logger
 
 
 class Faker(BaseActivity):
+    """
+    Synthetic data generation for testing and development.
+
+    This class generates realistic industrial sensor data for testing purposes.
+    It provides:
+    - Configurable sensor tag simulation
+    - Realistic data value generation
+    - Kafka integration for data publishing
+    - Comprehensive error handling and logging
+
+    The class is designed for development, testing, and demonstration of
+    data processing pipelines without requiring real industrial data sources.
+    """
+
     def __init__(self, bootstrap_servers: str, logger: Logger,
                  notification_handler: NotificationHandler):
+        """
+        Initialize the Faker class with Kafka producer and sensor configuration.
+
+        Args:
+            bootstrap_servers (str): Kafka bootstrap servers configuration
+            logger (Logger): Logger instance for operation logging
+            notification_handler (NotificationHandler): Handler for system notifications
+        """
         self.producer = KafkaProducer(
             bootstrap_servers=bootstrap_servers,
             value_serializer=lambda v: json.dumps(v).encode('utf-8')
@@ -31,16 +53,28 @@ class Faker(BaseActivity):
         BaseActivity.__init__(self, logger, notification_handler)
 
     @activity.defn(name="generate_and_send_data")
-    async def generate_and_send_data(self, input_data: dict[str, Any]):
+    async def generate_and_send_data(self, input_data: dict[str, Any]) -> None:
         """
-        Generates random data and sends it to a Kafka topic.
+        Generate synthetic sensor data and publish to Kafka topic.
+
+        This activity creates realistic industrial sensor readings and publishes
+        them to the specified Kafka topic. The data includes sensor tags, names,
+        timestamps, and values with configurable message counts.
 
         Args:
-            input_data (dict[str, Any]): The input data containing:
-                topic (str): The Kafka topic to send data to
-                num_messages (int, optional): Number of messages to generate. 
-                Defaults to random.randint(1, len(self.tags)).
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - topic (str): Kafka topic name for data publication
+                - metadata (dict[str, Any], optional): Workflow execution metadata
+                - num_messages (int, optional): Number of messages to generate.
+                  Defaults to random count between 1 and available sensor tags
 
+        Returns:
+            None: This activity publishes data but doesn't return results
+
+        Raises:
+            ValueError: If topic is not specified
+            Exception: If data generation or Kafka publishing fails
         """
         metadata = input_data['metadata']
         topic = input_data.get('topic')

scouter/activities/gates.py

@@ -19,22 +19,52 @@ quality_gate_filters = {
 
 
 class Gates(BaseActivity):
+    """
+    Data quality gates and filtering operations.
+
+    This class implements data quality validation and filtering for industrial
+    time-series data. It provides:
+    - Configurable data quality filters
+    - Data aggregation functions for time-series data
+    - Comprehensive error handling and notification
+    - Metrics collection for quality monitoring
+
+    The class supports multiple aggregation strategies and quality filters to
+    ensure data integrity and enable flexible data processing workflows.
+    """
 
     def __init__(self, logger: Logger, notification_handler: NotificationHandler):
+        """
+        Initialize the Gates class with logging and notification services.
+
+        Args:
+            logger (Logger): Logger instance for operation logging
+            notification_handler (NotificationHandler): Handler for system notifications
+        """
         BaseActivity.__init__(
             self, logger, notification_handler, set_error_counter=True)
 
     def apply_aggregation(self, group: DataFrame, aggr_function: str,
                           metadata: dict[str, Any]) -> float | None | str:
         """
-        Apply aggregation function to a group of data.
+        Apply aggregation function to a group of time-series data.
+
+        This method applies the specified aggregation function to a group of
+        data points. It handles edge cases and provides comprehensive error
+        reporting for invalid aggregation functions.
 
         Args:
-            group (DataFrame): The group of data to apply the aggregation function to.
-            aggr_function (str): The aggregation function to apply.
+            group (DataFrame): Group of data points to aggregate
+            aggr_function (str): Aggregation function to apply.
+                Supported functions: 'lts' (latest), 'avg' (average), 'mdn' (median),
+                'max' (maximum), 'min' (minimum)
+            metadata (dict[str, Any]): Workflow metadata for error reporting
 
         Returns:
-            float | None | str: The result of the aggregation function.
+            float | None | str: Aggregated value, None if no valid data, or 'continue' for errors
+
+        Raises:
+            NotificationError: If invalid aggregation function is specified
         """
         if len(group) == 1:
             return group['value'].item()
@@ -70,17 +100,23 @@ class Gates(BaseActivity):
     @activity.defn(name="aggregate_data")
     async def aggregate_data(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
-        Aggregates time series data by tag and name, applying specified
-        aggregation functions and taking the latest timestamp.
+        Aggregate time-series data by tag and name using specified functions.
+
+        This activity processes time-series data by grouping it by tag and name,
+        then applying the configured aggregation functions. It handles data
+        validation and provides comprehensive error reporting.
 
         Args:
-            input_data (dict[str, Any]): The data to aggregate. Contains:
-                data (dict[str, Any]): The time series data.
-                model_tags (dict[str, Any]): The tags configuration
-                containing aggregation functions.
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - data (dict[str, Any]): Time-series data to aggregate
+                - model_tags (dict[str, Any]): Tag configuration with aggregation functions
 
         Returns:
-            dict[str, Any]: The aggregated data.
+            dict[str, Any]: Aggregated data organized by tag and name
+
+        Raises:
+            Exception: If aggregation operation fails
         """
 
         metadata = input_data['metadata']
@@ -177,22 +213,24 @@ class Gates(BaseActivity):
     @activity.defn(name="data_quality_gate")
     async def data_quality_gate(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
-        Data quality gate activity. for each selected filter,
-        extracts filtered data, discards or keeps filtered data
-        based on the filter.
+        Apply data quality filters to incoming data.
+
+        This activity applies configurable quality filters to validate incoming
+        data. It supports multiple filter types and provides comprehensive
+        error reporting for quality issues.
 
         Args:
-            input_data (dict[str, Any]): The data to validate. Contains:
-                filters (dict[str, str]): The filters to apply. In format:
-                    {filter_name: policy}.
-                    filter_name: The name of the filter.
-                    policy: The policy to apply. Can be "DISCARD" or "KEEP".
-                data (dict[str, Any]): The data to validate.
-                model_tags (dict[str, Any]): The tags of the model.
-                    And it's respective configuration.
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - data (dict[str, Any]): Data to validate
+                - filters (dict[str, str]): Filter configuration
+                - model_tags (dict[str, Any]): Tag-specific validation rules
 
         Returns:
-            dict[str, Any]: The data validated.
+            dict[str, Any]: Filtered data that passes quality validation
+
+        Raises:
+            Exception: If quality validation fails
         """
 
         metadata = input_data['metadata']

scouter/activities/mongodb.py

@@ -15,13 +15,20 @@ with workflow.unsafe.imports_passed_through():
 
 def clear_mongo_id(docs: list) -> list:
     """
-    Remove the MongoDB internal `_id` field from the document.
+    Remove MongoDB internal `_id` fields from documents.
+
+    This utility function recursively removes the MongoDB `_id` field from
+    documents and nested structures. It's used to clean data before
+    processing or export operations.
 
     Args:
-        docs (list): The document to clear.
+        docs (list): List of documents to clean
 
     Returns:
-        list: The documents without the `_id` field.
+        list: Documents with `_id` fields removed
+
+    Note:
+        This function modifies the input list in-place and returns the same reference
     """
     for doc in docs:
         if isinstance(doc, list):
@@ -41,9 +48,35 @@ def clear_mongo_id(docs: list) -> list:
 
 
 class MongoDB(BaseActivity):
+    """
+    MongoDB operations for data retrieval and storage.
+
+    This class provides MongoDB connectivity and operations for the Scouter system.
+    It handles:
+    - Connection management with automatic reconnection
+    - Data retrieval with timestamp-based filtering
+    - Document cleaning and preprocessing
+    - Error handling and notification integration
+
+    The class implements Temporal activities for MongoDB operations, enabling
+    distributed data processing with fault tolerance and monitoring.
+    """
+
     def __init__(self, connection_string: str, database_name: str,
                  logger: Logger,
                  notification_handler: NotificationHandler):
+        """
+        Initialize MongoDB connection and services.
+
+        Args:
+            connection_string (str): MongoDB connection URI string
+            database_name (str): Name of the target database
+            logger (Logger): Logger instance for operation logging
+            notification_handler (NotificationHandler): Handler for system notifications
+
+        Raises:
+            ConnectionError: If MongoDB connection fails
+        """
         self.connection_string = connection_string
         self.database_name = database_name
 
@@ -63,7 +96,10 @@ class MongoDB(BaseActivity):
 
     def shutdown(self):
         """
-        Close the MongoDB client connection.
+        Gracefully close MongoDB client connection.
+
+        This method ensures proper cleanup of MongoDB connections to prevent
+        connection leaks and ensure graceful application termination.
         """
         try:
             if self.client:
@@ -75,14 +111,34 @@ class MongoDB(BaseActivity):
 
     def __del__(self):
         """
-        Destructor to ensure MongoDB client is closed when the object is deleted.
+        Destructor to ensure MongoDB client is closed.
+
+        This destructor ensures that MongoDB connections are properly closed
+        when the object is garbage collected, preventing resource leaks.
         """
         self.shutdown()
 
     @activity.defn(name="load_latest_data")
     async def load_latest_data(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
-        Loads the latest data from MongoDB.
+        Load the latest data from MongoDB collection since a specified timestamp.
+
+        This activity retrieves data from a MongoDB collection, optionally
+        filtering by timestamp to enable incremental data processing. It
+        handles connection management and provides comprehensive error reporting.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - collection_name (str): Name of the MongoDB collection
+                - last_data_timestamp (str | None): Last processed timestamp for filtering
+
+        Returns:
+            dict[str, Any]: Retrieved data, or empty dict if no data found
+
+        Raises:
+            Exception: If MongoDB operation fails
         """
         metadata = input_data['metadata']
         collection_name = input_data['collection_name']

scouter/activities/redis.py

@@ -14,17 +14,58 @@ with workflow.unsafe.imports_passed_through():
 
 
 class Redis(RedisBase):
+    """
+    Redis operations for data caching and temporary storage.
+
+    This class extends the base Redis functionality to provide specialized
+    operations for the Scouter system, including:
+    - Data timestamp management for incremental processing
+    - Temporary data storage with configurable TTL
+    - Data grouping and holding for batch processing
+    - Error handling and notification integration
+
+    The class implements Temporal activities for Redis operations, enabling
+    distributed data processing with fault tolerance and monitoring.
+    """
+
     def __init__(self, host: str, port: int,
                  username: str, password: str,
                  logger: Logger, notification_handler: NotificationHandler):
+        """
+        Initialize Redis connection and services.
 
+        Args:
+            host (str): Redis server hostname or IP address
+            port (int): Redis server port number
+            username (str): Redis authentication username
+            password (str): Redis authentication password
+            logger (Logger): Logger instance for operation logging
+            notification_handler (NotificationHandler): Handler for system notifications
+        """
         RedisBase.__init__(self, host, port, username,
                            password, logger, notification_handler)
 
     @activity.defn(name="get_last_data_timestamp")
     async def get_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
         """
-        Gets the last data timestamp from redis.
+        Retrieve the last processed data timestamp from Redis.
+
+        This activity retrieves the timestamp of the last successfully processed
+        data point for a specific workflow and schedule combination. It's used
+        for incremental data processing to avoid reprocessing the same data.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - workflow_name (str): Name of the workflow
+                - schedule_name (str): Name of the data collection schedule
+
+        Returns:
+            str | None: Last processed timestamp string, or None if no previous data exists
+
+        Raises:
+            Exception: If Redis operation fails
         """
         metadata = input_data['metadata']
         key = f"last_data_timestamp:{input_data['workflow_name']}:{input_data['schedule_name']}"
@@ -55,9 +96,27 @@ class Redis(RedisBase):
         return data_hold
 
     @activity.defn(name="put_last_data_timestamp")
-    async def put_last_data_timestamp(self, input_data: dict[str, Any]):
+    async def put_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
         """
-        Puts the last data timestamp into redis.
+        Store the last processed data timestamp in Redis.
+
+        This activity stores the timestamp of the most recent data point that
+        has been successfully processed. The timestamp is used for incremental
+        data loading in subsequent workflow executions.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - data (dict[str, Any]): Processed data to extract timestamp from
+                - workflow_name (str): Name of the workflow
+                - schedule_name (str): Name of the data collection schedule
+
+        Returns:
+            str | None: The timestamp that was stored, or None if no data was processed
+
+        Raises:
+            Exception: If Redis operation fails
         """
         metadata = input_data['metadata']
         key = f"last_data_timestamp:{input_data['workflow_name']}:{input_data['schedule_name']}"
@@ -96,18 +155,30 @@ class Redis(RedisBase):
         return last_data_timestamp
 
     @activity.defn(name="group_and_hold_data")
-    async def group_and_hold_data(self, input_data: dict[str, Any]):
+    async def group_and_hold_data(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
-        Groups and holds data in redis. Keep a copy of the most recent
-        received data for a given pipeline and schedule. This activity updates
-        the data in redis and return the full keeped data.
+        Group data by tags and store temporarily in Redis with TTL.
+
+        This activity organizes processed data by tag names and stores it in Redis
+        with a configurable retention period. The data is grouped to enable
+        efficient batch processing and export operations.
 
         Args:
-            input_data (dict[str, Any]): The data to group and hold.
-                workflow_name (str): The name of the workflow.
-                schedule_name (str): The name of the schedule.
-                data (dict[str, Any]): The data to group and hold.
-                retention_time (int): The retention time for data in redis in seconds.
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - schedule_name (str): Name of the data collection schedule
+                - workflow_name (str): Name of the workflow
+                - data (dict[str, Any]): Data to group and store
+                - model_id (str): Unique model identifier
+                - model_tags (dict[str, Any]): Tag configuration
+                - retention_time (int): Data retention period in seconds
+
+        Returns:
+            dict[str, Any]: Grouped data organized by tag names
+
+        Raises:
+            Exception: If Redis operation fails
         """
         metadata = input_data['metadata']
 

scouter/metrics.py

@@ -1,19 +1,23 @@
 from prometheus_client import Gauge, Counter
 
+# Application health and status metrics
 APP_UP = Gauge(
     "app_up",
     "Indicates if the application is running (1) or shutting down (0)",
     ["pod_id"],
 )
 
+# Core labels for consistent metric labeling
 CORE_LABELS = ["pod_id", "model_name", "pipeline_name"]
 
+# Data processing metrics
 LABORIOUS_DATA_WRITTEN_COUNT = Counter(
     "scouter_laborious_data_written_count",
     "Number of writings to the database table laborious_data",
     CORE_LABELS,
 )
 
+# Tag monitoring metrics
 TAG_CHANGES_MONITOR = Gauge(
     "scouter_tag_changes_monitor",
     "Current value change of each tag",

scouter/utils/connectors_config.py

@@ -1,7 +1,21 @@
 from os import getenv
+from typing import Any
 
 
-def build_postgres_config():
+def build_postgres_config() -> dict[str, Any]:
+    """
+    Build PostgreSQL connection configuration from environment variables.
+
+    Returns:
+        dict[str, Any]: PostgreSQL configuration dictionary with keys:
+            - host: Database hostname (default: localhost)
+            - port: Database port (default: 5432)
+            - user: Database username (default: sientia)
+            - password: Database password (default: sientia)
+            - dbname: Database name (default: sientia)
+            - min_connections: Minimum connection pool size (default: 5)
+            - max_connections: Maximum connection pool size (default: 20)
+    """
     return {
         'host': getenv('POSTGRES_HOST', 'localhost'),
         'port': int(getenv('POSTGRES_PORT', '5432')),
@@ -13,7 +27,16 @@ def build_postgres_config():
     }
 
 
-def build_kafka_config():
+def build_kafka_config() -> dict[str, Any]:
+    """
+    Build Kafka configuration from environment variables.
+
+    Returns:
+        dict[str, Any]: Kafka configuration dictionary with keys:
+            - bootstrap_servers: Kafka broker addresses (default: localhost:9092)
+            - polling_time: Consumer polling interval in milliseconds (default: 1000)
+            - group_id: Consumer group identifier (default: scouter-group)
+    """
     return {
         'bootstrap_servers': getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092'),
         'polling_time': int(getenv('KAFKA_POLLING_TIME', '1000')),
@@ -21,7 +44,17 @@ def build_kafka_config():
     }
 
 
-def build_redis_config():
+def build_redis_config() -> dict[str, Any]:
+    """
+    Build Redis connection configuration from environment variables.
+
+    Returns:
+        dict[str, Any]: Redis configuration dictionary with keys:
+            - host: Redis server hostname (default: localhost)
+            - port: Redis server port (default: 6379)
+            - username: Redis authentication username (default: None)
+            - password: Redis authentication password (default: None)
+    """
     return {
         'host': getenv('REDIS_HOST', 'localhost'),
         'port': int(getenv('REDIS_PORT', '6379')),
@@ -30,7 +63,15 @@ def build_redis_config():
     }
 
 
-def build_mongodb_config():
+def build_mongodb_config() -> dict[str, Any]:
+    """
+    Build MongoDB connection configuration from environment variables.
+
+    Returns:
+        dict[str, Any]: MongoDB configuration dictionary with keys:
+            - connection_string: Complete MongoDB connection URI
+            - database_name: Target database name (default: sientia)
+    """
     username = getenv('MONGODB_USERNAME', 'sientia')
     password = getenv('MONGODB_PASSWORD', 'sientia')
     uri = getenv('MONGODB_URL', 'localhost:27017')
@@ -42,7 +83,15 @@ def build_mongodb_config():
     }
 
 
-def build_druid_config():
+def build_druid_config() -> dict[str, Any]:
+    """
+    Build Apache Druid connection configuration from environment variables.
+
+    Returns:
+        dict[str, Any]: Druid configuration dictionary with keys:
+            - host: Druid server hostname (default: localhost)
+            - port: Druid server port (default: 8082)
+    """
     return {
         'host': getenv('DRUID_HOST', 'localhost'),
         'port': int(getenv('DRUID_PORT', '8082')),

scouter/utils/quality/filters.py

@@ -5,14 +5,21 @@ from typing import Any
 
 def check_data_range(value: float | int | None, val_range: list) -> bool:
     """
-    Check if a value is out of a given range.
+    Check if a value falls outside the specified range.
+
+    This function validates if a numeric value is within the acceptable range
+    defined by the minimum and maximum bounds. It handles edge cases including
+    None values and NaN values.
 
     Args:
-        value (float | int | None): The value to check.
-        val_range (list): The range to check against.
+        value (float | int | None): The numeric value to validate
+        val_range (list): List containing [min_value, max_value] bounds
 
     Returns:
-        bool: True if the value is out of the range, False otherwise.
+        bool: True if value is outside the range, False if within range
+
+    Note:
+        None and NaN values are considered out of range (return True)
     """
     if value is None or np.isnan(value):
         return True
@@ -23,32 +30,46 @@ def check_data_range(value: float | int | None, val_range: list) -> bool:
     return value < bottom or value > up
 
 
-def out_of_bounds_filter(df: DataFrame, model_tags: dict[str, Any]):
+def out_of_bounds_filter(df: DataFrame, model_tags: dict[str, Any]) -> DataFrame:
     """
-    Filter out rows where the value is out of the range.
+    Filter DataFrame rows where values are outside configured ranges.
+
+    This function applies range validation to each row in the DataFrame based
+    on tag-specific configuration. Rows with values outside the configured
+    ranges are filtered out.
 
     Args:
-        df (DataFrame): The DataFrame to filter.
-        model_tags (dict[str, Any]): The model tags. Contains
-        the data_range for each tag. If the tag does not have a data_range,
-        it will be considered as (-inf, inf).
+        df (DataFrame): DataFrame containing sensor data with 'name' and 'value' columns
+        model_tags (dict[str, Any]): Tag configuration containing data_range for each tag.
+            If a tag doesn't have data_range, it's considered to have infinite bounds.
 
     Returns:
-        DataFrame: The filtered DataFrame.
+        DataFrame: Filtered DataFrame with out-of-bounds values removed
+
+    Note:
+        Tags without data_range configuration are treated as having infinite bounds
     """
     return df[df.apply(lambda x: check_data_range(
         x['value'], model_tags[x['name']].get('data_range', (-np.inf, np.inf))),
         axis=1)]
 
 
-def null_values_filter(df: DataFrame, _model_tags: dict[str, Any]):
+def null_values_filter(df: DataFrame, _model_tags: dict[str, Any]) -> DataFrame:
     """
-    Filter out rows where the value is null.
+    Filter DataFrame rows containing null values.
+
+    This function removes rows where the 'value' column contains null values.
+    It's used for data quality filtering to ensure only complete data records
+    are processed.
 
     Args:
-        df (DataFrame): The DataFrame to filter.
+        df (DataFrame): DataFrame containing sensor data with 'value' column
+        _model_tags (dict[str, Any]): Tag configuration (unused in this filter)
 
     Returns:
-        DataFrame: The filtered DataFrame.
+        DataFrame: Filtered DataFrame with null values removed
+
+    Note:
+        The _model_tags parameter is included for interface consistency but not used
     """
     return df[df['value'].isnull()]

scouter/worker/worker.py

@@ -21,11 +21,32 @@ with workflow.unsafe.imports_passed_through():
         build_mongodb_config
     )
 
+# Environment configuration
 POD_ID = os.getenv("HOSTNAME", "localhost")
 SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', "9091"))
 
 
 async def main():
+    """
+    Main entry point for the Scouter Temporal worker.
+
+    This function initializes and starts all required services:
+    - Prometheus metrics server
+    - Notification handler for MongoDB
+    - Activity implementations for data processing
+    - Temporal client and workers
+    - Multiple task queues for different workflow types
+
+    The worker supports two main task queues:
+    - scouter-queue: Main data processing workflows
+    - fake_data-queue: Test data generation workflows
+
+    Returns:
+        None
+
+    Raises:
+        SystemExit: If worker initialization or execution fails
+    """
     host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
 
     logger = get_logger(__name__)
@@ -148,6 +169,19 @@ async def main():
 
 
 def start_prometheus_server():
+    """
+    Start the Prometheus metrics HTTP server.
+
+    This function initializes the Prometheus metrics server on the configured
+    port and sets the application health status. It's essential for
+    monitoring and observability of the Scouter system.
+
+    Returns:
+        None
+
+    Raises:
+        SystemExit: If metrics server fails to start
+    """
     try:
         port = int(os.getenv("HTTP_METRICS_PORT", 9090))
         start_http_server(port)

scouter/workflow/fake_data.py

@@ -9,14 +9,43 @@ with workflow.unsafe.imports_passed_through():
 
 @workflow.defn(name="fake_data")
 class FakeData:
+    """
+    Test data generation workflow for development and testing purposes.
+
+    This workflow generates synthetic industrial sensor data and publishes it to
+    Kafka topics. It's designed for:
+    - Development and testing of data processing pipelines
+    - Load testing of downstream systems
+    - Demonstration of data flow patterns
+    - Validation of data quality filters and aggregation functions
+
+    The generated data simulates realistic industrial sensor readings with
+    configurable message counts and topic routing.
+    """
+
     @workflow.run
     async def run(self, workflow_input: Dict[str, Any]) -> str:
         """
-        Generates random data and sends it to a Kafka topic.
+        Execute the fake data generation workflow.
+
+        This method generates synthetic sensor data and publishes it to the specified
+        Kafka topic. The data includes realistic industrial sensor readings with
+        configurable parameters for testing and development purposes.
 
         Args:
-            workflow_input (dict[str, Any]): The input data containing:
-                topic (str): The Kafka topic to send data to
+            workflow_input (dict[str, Any]): Workflow configuration parameters.
+                Required fields:
+                - topic (str): Kafka topic name for data publication
+                - metadata (dict[str, Any], optional): Workflow execution metadata
+                - num_messages (int, optional): Number of messages to generate.
+                  Defaults to random count between 1 and available sensor tags.
+
+        Returns:
+            str: Success confirmation message
+
+        Raises:
+            WorkflowExecutionError: If workflow execution fails
+            ActivityExecutionError: If data generation or Kafka publishing fails
         """
         await workflow.execute_activity_method(
             Faker.generate_and_send_data,

scouter/workflow/scouter.py

@@ -9,25 +9,55 @@ with workflow.unsafe.imports_passed_through():
 
 @workflow.defn(name="scouter")
 class Scouter:
+    """
+    Main Scouter workflow that orchestrates data ingestion and processing.
+
+    This workflow serves as the entry point for data processing pipelines. It loads
+    data from MongoDB collections, manages data timestamps for incremental processing,
+    and delegates the actual data processing to the CoreScouter workflow.
+
+    The workflow implements a robust data ingestion pattern with:
+    - Incremental data loading based on last processed timestamp
+    - Automatic timestamp management for data continuity
+    - Error handling and retry policies
+    - Child workflow orchestration for data processing
+    """
+
     @workflow.run
-    async def run(self, input_data: dict[str, Any]):
+    async def run(self, input_data: dict[str, Any]) -> None:
         """
-        Scouter workflow. Loads data from kafka and sends it to the core_scouter
-        workflow.
+        Execute the main Scouter workflow.
+
+        This method orchestrates the complete data ingestion process:
+        1. Retrieves the last processed timestamp from Redis
+        2. Loads new data from MongoDB since the last timestamp
+        3. Updates the last processed timestamp
+        4. Delegates data processing to the CoreScouter workflow
 
         Args:
-            input_data (dict[str, Any]): The data to process. Contains:
-                topic (str): The topic to load data from.
-                schedule_name (str): The name of the schedule.
-                model_name (str): The name of the model.
-                model_id (str): The id of the model.
-                trigger_laborious (bool): Whether to trigger laborious.
-                filters (dict[str, str]): The filters to apply.
-                schema (str): The schema of the table to export data to.
-                table_name (str): The name of the table to export data to.
-                retention_time (int): The retention time for data in redis in seconds.
-                model_tags (dict[str, Any]): The tags of the model.
-                    And it's respective configuration.
+            input_data (dict[str, Any]): Configuration and parameters for the workflow execution.
+                Required fields:
+                - topic (str): The Kafka topic name for data source identification
+                - schedule_name (str): Unique identifier for the data collection schedule
+                - model_name (str): Name of the data model being processed
+                - model_id (str): Unique identifier for the data model
+                - trigger_laborious (bool): Flag to enable intensive data processing
+                - filters (dict[str, str]): Data quality filters configuration
+                - schema (str): Target database schema for data export
+                - table_name (str): Target table name for data export
+                - retention_time (int): Data retention period in Redis (seconds)
+                - model_tags (dict[str, Any]): Tag-specific configuration including:
+                    - data_range: [min, max] values for data validation
+                    - aggr_function: Aggregation method (avg, mdn, max, min, lts)
+                    - frequency: Data collection frequency in milliseconds
+                    - topics: List of Kafka topics for data routing
+
+        Returns:
+            None: This workflow doesn't return data, it orchestrates data processing
+
+        Raises:
+            WorkflowExecutionError: If workflow execution fails
+            ActivityExecutionError: If any activity fails after retry attempts
         """
 
         input_data['workflow_name'] = 'scouter'

scouter/workflow/sub_workflows/core_scouter.py

@@ -10,27 +10,54 @@ with workflow.unsafe.imports_passed_through():
 
 @workflow.defn(name="core_scouter")
 class CoreScouter:
+    """
+    Core data processing workflow that handles data quality, aggregation, and export.
+
+    This workflow implements the core data processing pipeline for industrial data:
+    - Data quality validation and filtering
+    - Time-series data aggregation using configurable functions
+    - Data grouping and temporary storage in Redis
+    - Asynchronous export to PostgreSQL for persistent storage
+    - Metrics collection and monitoring
+
+    The workflow is designed for high-throughput data processing with configurable
+    quality gates and aggregation strategies.
+    """
+
     @workflow.run
-    async def run(self, input_data: dict[str, Any]):
+    async def run(self, input_data: dict[str, Any]) -> None:
         """
-        Core scouter workflow. Passes data through data_quality_gate,
-        group_and_hold_data, and then asynchronously exports data to postgres
-        using export_data_to_postgres and in the future will trigger_laborious
-        if needed.
+        Execute the core data processing workflow.
+
+        This method processes industrial time-series data through a series of stages:
+        1. Data Quality Gate: Applies configurable filters for data validation
+        2. Data Aggregation: Groups and aggregates data using specified functions
+        3. Data Grouping: Organizes data by tags and applies retention policies
+        4. Data Export: Persists processed data to PostgreSQL
+        5. Metrics Collection: Records processing metrics for monitoring
 
         Args:
-            input_data (dict[str, Any]): The data to process. Contains:
-                metadata (dict[str, Any]): The metadata of the workflow.
-                workflow_name (str): The name of the workflow.
-                schedule_name (str): The name of the schedule.
-                model_name (str): The name of the model.
-                model_id (str): The id of the model.
-                data (dict[str, Any]): The data to process.
-                trigger_laborious (bool): Whether to trigger laborious.
-                filters (dict[str, str]): The filters to apply.
-                schema (str): The schema of the table to export data to.
-                table_name (str): The name of the table to export data to.
-                retention_time (int): The retention time for data in redis in seconds.
+            input_data (dict[str, Any]): Complete workflow configuration and data.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - workflow_name (str): Name of the parent workflow
+                - schedule_name (str): Data collection schedule identifier
+                - model_name (str): Data model name
+                - model_id (str): Unique model identifier
+                - data (dict[str, Any]): Raw time-series data to process
+                - trigger_laborious (bool): Enable intensive processing mode
+                - filters (dict[str, str]): Data quality filter configurations
+                - schema (str): Target database schema
+                - table_name (str): Target database table
+                - retention_time (int): Redis data retention period (seconds)
+                - model_tags (dict[str, Any]): Tag-specific processing rules
+
+        Returns:
+            None: This workflow processes data but doesn't return results
+
+        Raises:
+            WorkflowExecutionError: If workflow execution fails
+            ActivityExecutionError: If any activity fails after retry attempts
         """
 
         metadata = input_data['metadata']