SIENTIAPDE-1084

Remove deprecated files and configurations, including .env, Dockerfile, docker-compose.yml, and client-schedule.py. Update README.md to reflect new architecture and features, enhancing clarity on system capabilities and workflows. Adjust values.yaml for image tag and replica count, and improve code documentation across various modules for better maintainability.

scouter/workflow/fake_data.py

@@ -9,14 +9,43 @@ with workflow.unsafe.imports_passed_through():
 
 @workflow.defn(name="fake_data")
 class FakeData:
+    """
+    Test data generation workflow for development and testing purposes.
+
+    This workflow generates synthetic industrial sensor data and publishes it to
+    Kafka topics. It's designed for:
+    - Development and testing of data processing pipelines
+    - Load testing of downstream systems
+    - Demonstration of data flow patterns
+    - Validation of data quality filters and aggregation functions
+
+    The generated data simulates realistic industrial sensor readings with
+    configurable message counts and topic routing.
+    """
+
     @workflow.run
     async def run(self, workflow_input: Dict[str, Any]) -> str:
         """
-        Generates random data and sends it to a Kafka topic.
+        Execute the fake data generation workflow.
+
+        This method generates synthetic sensor data and publishes it to the specified
+        Kafka topic. The data includes realistic industrial sensor readings with
+        configurable parameters for testing and development purposes.
 
         Args:
-            workflow_input (dict[str, Any]): The input data containing:
-                topic (str): The Kafka topic to send data to
+            workflow_input (dict[str, Any]): Workflow configuration parameters.
+                Required fields:
+                - topic (str): Kafka topic name for data publication
+                - metadata (dict[str, Any], optional): Workflow execution metadata
+                - num_messages (int, optional): Number of messages to generate.
+                  Defaults to random count between 1 and available sensor tags.
+
+        Returns:
+            str: Success confirmation message
+
+        Raises:
+            WorkflowExecutionError: If workflow execution fails
+            ActivityExecutionError: If data generation or Kafka publishing fails
         """
         await workflow.execute_activity_method(
             Faker.generate_and_send_data,

scouter/workflow/scouter.py

@@ -9,25 +9,55 @@ with workflow.unsafe.imports_passed_through():
 
 @workflow.defn(name="scouter")
 class Scouter:
+    """
+    Main Scouter workflow that orchestrates data ingestion and processing.
+
+    This workflow serves as the entry point for data processing pipelines. It loads
+    data from MongoDB collections, manages data timestamps for incremental processing,
+    and delegates the actual data processing to the CoreScouter workflow.
+
+    The workflow implements a robust data ingestion pattern with:
+    - Incremental data loading based on last processed timestamp
+    - Automatic timestamp management for data continuity
+    - Error handling and retry policies
+    - Child workflow orchestration for data processing
+    """
+
     @workflow.run
-    async def run(self, input_data: dict[str, Any]):
+    async def run(self, input_data: dict[str, Any]) -> None:
         """
-        Scouter workflow. Loads data from kafka and sends it to the core_scouter
-        workflow.
+        Execute the main Scouter workflow.
+
+        This method orchestrates the complete data ingestion process:
+        1. Retrieves the last processed timestamp from Redis
+        2. Loads new data from MongoDB since the last timestamp
+        3. Updates the last processed timestamp
+        4. Delegates data processing to the CoreScouter workflow
 
         Args:
-            input_data (dict[str, Any]): The data to process. Contains:
-                topic (str): The topic to load data from.
-                schedule_name (str): The name of the schedule.
-                model_name (str): The name of the model.
-                model_id (str): The id of the model.
-                trigger_laborious (bool): Whether to trigger laborious.
-                filters (dict[str, str]): The filters to apply.
-                schema (str): The schema of the table to export data to.
-                table_name (str): The name of the table to export data to.
-                retention_time (int): The retention time for data in redis in seconds.
-                model_tags (dict[str, Any]): The tags of the model.
-                    And it's respective configuration.
+            input_data (dict[str, Any]): Configuration and parameters for the workflow execution.
+                Required fields:
+                - topic (str): The Kafka topic name for data source identification
+                - schedule_name (str): Unique identifier for the data collection schedule
+                - model_name (str): Name of the data model being processed
+                - model_id (str): Unique identifier for the data model
+                - trigger_laborious (bool): Flag to enable intensive data processing
+                - filters (dict[str, str]): Data quality filters configuration
+                - schema (str): Target database schema for data export
+                - table_name (str): Target table name for data export
+                - retention_time (int): Data retention period in Redis (seconds)
+                - model_tags (dict[str, Any]): Tag-specific configuration including:
+                    - data_range: [min, max] values for data validation
+                    - aggr_function: Aggregation method (avg, mdn, max, min, lts)
+                    - frequency: Data collection frequency in milliseconds
+                    - topics: List of Kafka topics for data routing
+
+        Returns:
+            None: This workflow doesn't return data, it orchestrates data processing
+
+        Raises:
+            WorkflowExecutionError: If workflow execution fails
+            ActivityExecutionError: If any activity fails after retry attempts
         """
 
         input_data['workflow_name'] = 'scouter'

scouter/workflow/sub_workflows/core_scouter.py

@@ -10,27 +10,54 @@ with workflow.unsafe.imports_passed_through():
 
 @workflow.defn(name="core_scouter")
 class CoreScouter:
+    """
+    Core data processing workflow that handles data quality, aggregation, and export.
+
+    This workflow implements the core data processing pipeline for industrial data:
+    - Data quality validation and filtering
+    - Time-series data aggregation using configurable functions
+    - Data grouping and temporary storage in Redis
+    - Asynchronous export to PostgreSQL for persistent storage
+    - Metrics collection and monitoring
+
+    The workflow is designed for high-throughput data processing with configurable
+    quality gates and aggregation strategies.
+    """
+
     @workflow.run
-    async def run(self, input_data: dict[str, Any]):
+    async def run(self, input_data: dict[str, Any]) -> None:
         """
-        Core scouter workflow. Passes data through data_quality_gate,
-        group_and_hold_data, and then asynchronously exports data to postgres
-        using export_data_to_postgres and in the future will trigger_laborious
-        if needed.
+        Execute the core data processing workflow.
+
+        This method processes industrial time-series data through a series of stages:
+        1. Data Quality Gate: Applies configurable filters for data validation
+        2. Data Aggregation: Groups and aggregates data using specified functions
+        3. Data Grouping: Organizes data by tags and applies retention policies
+        4. Data Export: Persists processed data to PostgreSQL
+        5. Metrics Collection: Records processing metrics for monitoring
 
         Args:
-            input_data (dict[str, Any]): The data to process. Contains:
-                metadata (dict[str, Any]): The metadata of the workflow.
-                workflow_name (str): The name of the workflow.
-                schedule_name (str): The name of the schedule.
-                model_name (str): The name of the model.
-                model_id (str): The id of the model.
-                data (dict[str, Any]): The data to process.
-                trigger_laborious (bool): Whether to trigger laborious.
-                filters (dict[str, str]): The filters to apply.
-                schema (str): The schema of the table to export data to.
-                table_name (str): The name of the table to export data to.
-                retention_time (int): The retention time for data in redis in seconds.
+            input_data (dict[str, Any]): Complete workflow configuration and data.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - workflow_name (str): Name of the parent workflow
+                - schedule_name (str): Data collection schedule identifier
+                - model_name (str): Data model name
+                - model_id (str): Unique model identifier
+                - data (dict[str, Any]): Raw time-series data to process
+                - trigger_laborious (bool): Enable intensive processing mode
+                - filters (dict[str, str]): Data quality filter configurations
+                - schema (str): Target database schema
+                - table_name (str): Target database table
+                - retention_time (int): Redis data retention period (seconds)
+                - model_tags (dict[str, Any]): Tag-specific processing rules
+
+        Returns:
+            None: This workflow processes data but doesn't return results
+
+        Raises:
+            WorkflowExecutionError: If workflow execution fails
+            ActivityExecutionError: If any activity fails after retry attempts
         """
 
         metadata = input_data['metadata']