995ba7900a
Remove Docker configuration files and refactor project structure - Deleted docker-compose.yml and Dockerfile as part of the project restructuring. - Updated README.md to reflect changes in project setup and configuration. - Introduced a new __init__.py file in the laborious package to provide an overview of the system. - Enhanced documentation across various modules, including metrics, activities, and workflows, to improve clarity and usability. - Added comprehensive docstrings and comments to key classes and methods for better maintainability.
83 lines
3.0 KiB
Python
83 lines
3.0 KiB
Python
"""
|
|
Sientia DataOps Laborious Package
|
|
|
|
A high-performance, scalable machine learning prediction system built on Temporal.io
|
|
for industrial data processing and ML model inference. The Laborious system provides
|
|
enterprise-grade ML model management, batch prediction processing, and real-time
|
|
data export capabilities.
|
|
|
|
Package Overview:
|
|
The Laborious package implements a comprehensive ML workflow orchestration
|
|
system that integrates with MLFlow for model management, PostgreSQL for data
|
|
storage, and OPC servers for real-time industrial data export.
|
|
|
|
Key Components:
|
|
- activities: Temporal activity implementations for ML operations
|
|
- workflows: Temporal workflow definitions for prediction orchestration
|
|
- worker: Main worker implementation for workflow execution
|
|
- utils: Utility functions and configuration management
|
|
- metrics: Prometheus metrics for monitoring and observability
|
|
|
|
Main Features:
|
|
- Batch prediction processing using MLFlow models
|
|
- Data quality validation and filtering
|
|
- Real-time data export to OPC servers
|
|
- PostgreSQL data persistence
|
|
- Comprehensive monitoring and metrics
|
|
- Automatic retry policies and error handling
|
|
|
|
Architecture:
|
|
The system uses Temporal.io for workflow orchestration with clear separation
|
|
of concerns between data loading, ML operations, quality validation, and
|
|
data export. It supports multiple OPC servers and implements configurable
|
|
data quality gates throughout the prediction pipeline.
|
|
|
|
Example Usage:
|
|
>>> from laborious.worker.worker import main
|
|
>>> import asyncio
|
|
>>>
|
|
>>> # Start the Laborious worker
|
|
>>> asyncio.run(main())
|
|
|
|
>>> # Or use specific components
|
|
>>> from laborious.activities.activities import Activities
|
|
>>> from laborious.workflows.predictions_batch import PredictionsBatch
|
|
|
|
Dependencies:
|
|
- temporalio: Temporal workflow orchestration
|
|
- psycopg2-binary: PostgreSQL database adapter
|
|
- sqlalchemy: Database ORM and connection management
|
|
- asyncua: OPC UA client implementation
|
|
- redis: Caching and session management
|
|
- prometheus-client: Metrics collection and export
|
|
|
|
Environment Configuration:
|
|
The system is configured through environment variables for database
|
|
connections, MLFlow servers, OPC servers, and other external services.
|
|
See the README.md for complete configuration documentation.
|
|
|
|
License:
|
|
This project is licensed under the terms specified in the LICENSE file.
|
|
|
|
For more information, see the project README.md and documentation.
|
|
"""
|
|
|
|
__version__ = "0.4.4"
|
|
__author__ = "Sientia DataOps Team"
|
|
__description__ = "ML prediction system built on Temporal.io for industrial data processing"
|
|
__keywords__ = ["machine-learning", "temporal", "mlflow", "opc", "postgresql", "industrial"]
|
|
__url__ = "https://github.com/Aignosi/sientia-dataops-laborious"
|
|
|
|
# Import key components for easy access
|
|
from . import metrics
|
|
from . import activities
|
|
from . import workflows
|
|
from . import worker
|
|
|
|
__all__ = [
|
|
"metrics",
|
|
"activities",
|
|
"workflows",
|
|
"worker"
|
|
]
|