a973da9d60
Remove deprecated files and configurations, including .env, Dockerfile, docker-compose.yml, and client-schedule.py. Update README.md to reflect new architecture and features, enhancing clarity on system capabilities and workflows. Adjust values.yaml for image tag and replica count, and improve code documentation across various modules for better maintainability.
101 lines
3.8 KiB
Python
101 lines
3.8 KiB
Python
from temporalio import workflow
|
|
|
|
with workflow.unsafe.imports_passed_through():
|
|
from sientia_do.temporal.activities.postgres import Postgres
|
|
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
|
from sientia_do.observability.logger import Logger
|
|
from scouter.activities.redis import Redis
|
|
from scouter.activities.gates import Gates
|
|
from scouter.activities.mongodb import MongoDB
|
|
from typing import Any
|
|
from os import getenv
|
|
|
|
|
|
class Activities(Postgres, Redis, Gates, MongoDB):
|
|
"""
|
|
Unified activities class that combines multiple data processing services.
|
|
|
|
This class provides a comprehensive interface for all data processing activities
|
|
by inheriting from specialized service classes. It handles:
|
|
- PostgreSQL operations for data persistence
|
|
- Redis operations for caching and temporary storage
|
|
- Data quality gates and filtering
|
|
- MongoDB operations for data retrieval
|
|
- Notification handling and logging
|
|
|
|
The class implements the multiple inheritance pattern to provide a unified
|
|
interface while maintaining separation of concerns across different data services.
|
|
"""
|
|
|
|
def __init__(self,
|
|
postgres_config: dict[str, Any],
|
|
redis_config: dict[str, Any],
|
|
mongodb_config: dict[str, Any],
|
|
logger: Logger,
|
|
notification_handler: NotificationHandler):
|
|
"""
|
|
Initialize the Activities class with all required services.
|
|
|
|
Args:
|
|
postgres_config (dict[str, Any]): PostgreSQL connection configuration.
|
|
Required fields: host, port, user, password, dbname, min_connections, max_connections
|
|
redis_config (dict[str, Any]): Redis connection configuration.
|
|
Required fields: host, port, username, password
|
|
mongodb_config (dict[str, Any]): MongoDB connection configuration.
|
|
Required fields: connection_string, database_name
|
|
logger (Logger): Logger instance for application logging
|
|
notification_handler (NotificationHandler): Handler for system notifications
|
|
"""
|
|
# Initialize Postgres
|
|
Postgres.__init__(
|
|
self,
|
|
host=postgres_config['host'],
|
|
port=postgres_config['port'],
|
|
user=postgres_config['user'],
|
|
password=postgres_config['password'],
|
|
dbname=postgres_config['dbname'],
|
|
min_connections=postgres_config['min_connections'],
|
|
max_connections=postgres_config['max_connections'],
|
|
logger=logger,
|
|
notification_handler=notification_handler
|
|
)
|
|
|
|
# Initialize Redis
|
|
Redis.__init__(
|
|
self,
|
|
host=redis_config['host'],
|
|
port=redis_config['port'],
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
username=redis_config['username'],
|
|
password=redis_config['password']
|
|
)
|
|
|
|
# Initialize Gates
|
|
Gates.__init__(
|
|
self,
|
|
logger=logger,
|
|
notification_handler=notification_handler
|
|
)
|
|
|
|
# Initialize MongoDB
|
|
MongoDB.__init__(
|
|
self,
|
|
connection_string=mongodb_config['connection_string'],
|
|
database_name=mongodb_config['database_name'],
|
|
logger=logger,
|
|
notification_handler=notification_handler
|
|
)
|
|
|
|
self.pod_id = getenv("HOSTNAME", "localhost")
|
|
|
|
def shutdown(self):
|
|
"""
|
|
Gracefully shutdown all service connections.
|
|
|
|
This method ensures proper cleanup of database connections and resources
|
|
to prevent connection leaks and ensure graceful application termination.
|
|
"""
|
|
Postgres.close(self)
|
|
MongoDB.shutdown(self)
|