97eb5bc904
Update .gitignore and refactor metrics.py, activities.py, and gates.py for improved clarity and consistency. Added coverage.xml and cache directories to .gitignore. Standardized string formatting and parameter handling in metrics and activities classes, enhancing code readability. Removed the deprecated faker.py file and adjusted related tests accordingly.
101 lines
3.7 KiB
Python
101 lines
3.7 KiB
Python
from temporalio import workflow
|
|
|
|
with workflow.unsafe.imports_passed_through():
|
|
from os import getenv
|
|
from typing import Any
|
|
|
|
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
|
from sientia_do.observability.logger import Logger
|
|
from sientia_do.temporal.activities.postgres import Postgres
|
|
|
|
from scouter.activities.gates import Gates
|
|
from scouter.activities.mongodb import MongoDB
|
|
from scouter.activities.redis import Redis
|
|
|
|
|
|
class Activities(Postgres, Redis, Gates, MongoDB):
|
|
"""
|
|
Unified activities class that combines multiple data processing services.
|
|
|
|
This class provides a comprehensive interface for all data processing activities
|
|
by inheriting from specialized service classes. It handles:
|
|
- PostgreSQL operations for data persistence
|
|
- Redis operations for caching and temporary storage
|
|
- Data quality gates and filtering
|
|
- MongoDB operations for data retrieval
|
|
- Notification handling and logging
|
|
|
|
The class implements the multiple inheritance pattern to provide a unified
|
|
interface while maintaining separation of concerns across different data services.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
postgres_config: dict[str, Any],
|
|
redis_config: dict[str, Any],
|
|
mongodb_config: dict[str, Any],
|
|
logger: Logger,
|
|
notification_handler: NotificationHandler,
|
|
):
|
|
"""
|
|
Initialize the Activities class with all required services.
|
|
|
|
Args:
|
|
postgres_config (dict[str, Any]): PostgreSQL connection configuration.
|
|
Required fields: host, port, user, password, dbname, min_connections, max_connections
|
|
redis_config (dict[str, Any]): Redis connection configuration.
|
|
Required fields: host, port, username, password
|
|
mongodb_config (dict[str, Any]): MongoDB connection configuration.
|
|
Required fields: connection_string, database_name
|
|
logger (Logger): Logger instance for application logging
|
|
notification_handler (NotificationHandler): Handler for system notifications
|
|
"""
|
|
# Initialize Postgres
|
|
Postgres.__init__(
|
|
self,
|
|
host=postgres_config['host'],
|
|
port=postgres_config['port'],
|
|
user=postgres_config['user'],
|
|
password=postgres_config['password'],
|
|
dbname=postgres_config['dbname'],
|
|
min_connections=postgres_config['min_connections'],
|
|
max_connections=postgres_config['max_connections'],
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
)
|
|
|
|
# Initialize Redis
|
|
Redis.__init__(
|
|
self,
|
|
host=redis_config['host'],
|
|
port=redis_config['port'],
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
username=redis_config['username'],
|
|
password=redis_config['password'],
|
|
)
|
|
|
|
# Initialize Gates
|
|
Gates.__init__(self, logger=logger, notification_handler=notification_handler)
|
|
|
|
# Initialize MongoDB
|
|
MongoDB.__init__(
|
|
self,
|
|
connection_string=mongodb_config['connection_string'],
|
|
database_name=mongodb_config['database_name'],
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
)
|
|
|
|
self.pod_id = getenv('HOSTNAME', 'localhost')
|
|
|
|
def shutdown(self):
|
|
"""
|
|
Gracefully shutdown all service connections.
|
|
|
|
This method ensures proper cleanup of database connections and resources
|
|
to prevent connection leaks and ensure graceful application termination.
|
|
"""
|
|
Postgres.close(self)
|
|
MongoDB.shutdown(self)
|