diff --git a/laborious/__init__.py b/laborious/__init__.py index 1fe9f95..e69de29 100644 --- a/laborious/__init__.py +++ b/laborious/__init__.py @@ -1,82 +0,0 @@ -""" -Sientia DataOps Laborious Package - -A high-performance, scalable machine learning prediction system built on Temporal.io -for industrial data processing and ML model inference. The Laborious system provides -enterprise-grade ML model management, batch prediction processing, and real-time -data export capabilities. - -Package Overview: - The Laborious package implements a comprehensive ML workflow orchestration - system that integrates with MLFlow for model management, PostgreSQL for data - storage, and OPC servers for real-time industrial data export. - -Key Components: - - activities: Temporal activity implementations for ML operations - - workflows: Temporal workflow definitions for prediction orchestration - - worker: Main worker implementation for workflow execution - - utils: Utility functions and configuration management - - metrics: Prometheus metrics for monitoring and observability - -Main Features: - - Batch prediction processing using MLFlow models - - Data quality validation and filtering - - Real-time data export to OPC servers - - PostgreSQL data persistence - - Comprehensive monitoring and metrics - - Automatic retry policies and error handling - -Architecture: - The system uses Temporal.io for workflow orchestration with clear separation - of concerns between data loading, ML operations, quality validation, and - data export. It supports multiple OPC servers and implements configurable - data quality gates throughout the prediction pipeline. - -Example Usage: - >>> from laborious.worker.worker import main - >>> import asyncio - >>> - >>> # Start the Laborious worker - >>> asyncio.run(main()) - - >>> # Or use specific components - >>> from laborious.activities.activities import Activities - >>> from laborious.workflows.predictions_batch import PredictionsBatch - -Dependencies: - - temporalio: Temporal workflow orchestration - - psycopg2-binary: PostgreSQL database adapter - - sqlalchemy: Database ORM and connection management - - asyncua: OPC UA client implementation - - redis: Caching and session management - - prometheus-client: Metrics collection and export - -Environment Configuration: - The system is configured through environment variables for database - connections, MLFlow servers, OPC servers, and other external services. - See the README.md for complete configuration documentation. - -License: - This project is licensed under the terms specified in the LICENSE file. - -For more information, see the project README.md and documentation. -""" - -__version__ = "0.4.4" -__author__ = "Sientia DataOps Team" -__description__ = "ML prediction system built on Temporal.io for industrial data processing" -__keywords__ = ["machine-learning", "temporal", "mlflow", "opc", "postgresql", "industrial"] -__url__ = "https://github.com/Aignosi/sientia-dataops-laborious" - -# Import key components for easy access -from . import metrics -from . import activities -from . import workflows -from . import worker - -__all__ = [ - "metrics", - "activities", - "workflows", - "worker" -] diff --git a/laborious/activities/__init__.py b/laborious/activities/__init__.py index 9ef1102..e69de29 100644 --- a/laborious/activities/__init__.py +++ b/laborious/activities/__init__.py @@ -1,10 +0,0 @@ -""" -Laborious Activities Package - -This package contains all Temporal activity implementations for the Laborious system, -including data quality gates, MLFlow operations, OPC server integration, and -database operations. - -Activities are the building blocks of workflows and implement the actual business -logic for data processing, ML model inference, and data export operations. -""" diff --git a/laborious/utils/__init__.py b/laborious/utils/__init__.py index b3056cc..e69de29 100644 --- a/laborious/utils/__init__.py +++ b/laborious/utils/__init__.py @@ -1,9 +0,0 @@ -""" -Laborious Utilities Package - -This package contains utility functions and configuration management for the Laborious system, -including database connectors, data quality filters, and repository implementations. - -Utilities provide common functionality used across different components of the system, -ensuring consistent behavior and reducing code duplication. -""" diff --git a/laborious/utils/filters/__init__.py b/laborious/utils/filters/__init__.py index c98035c..e69de29 100644 --- a/laborious/utils/filters/__init__.py +++ b/laborious/utils/filters/__init__.py @@ -1,10 +0,0 @@ -""" -Laborious Data Quality Filters Package - -This package contains data quality validation and filtering functions for the Laborious system, -including conditional filters for input data validation and MLFlow-specific filters for -response quality assessment. - -Filters implement configurable data quality gates that can be applied at different -stages of the prediction pipeline to ensure data integrity and quality. -""" diff --git a/laborious/workflows/__init__.py b/laborious/workflows/__init__.py index 1b87103..e69de29 100644 --- a/laborious/workflows/__init__.py +++ b/laborious/workflows/__init__.py @@ -1,10 +0,0 @@ -""" -Laborious Workflows Package - -This package contains all Temporal workflow definitions for the Laborious system, -including batch prediction workflows, model retraining workflows, and specialized -sub-workflows for data processing and export operations. - -Workflows orchestrate the execution of activities and implement the business -process logic for ML model inference and data processing pipelines. -""" diff --git a/laborious/workflows/sub_workflows/__init__.py b/laborious/workflows/sub_workflows/__init__.py new file mode 100644 index 0000000..e69de29