SIENTIAPDE-1094
Update environment configuration and refactor activity imports - Changed Kafka, Redis, and Temporal host configurations to use localhost. - Updated the version reference for the sientia-dataops-library in requirements.txt. - Refactored import paths for activities to align with new module structure. - Removed unused base.py and postgres.py files. - Updated logger and policies imports to reflect new module locations. - Adjusted values.yaml for branch and log level settings.
This commit is contained in:
vitor-aignosi
@@ -1,12 +1,12 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from scouter.activities.postgres import Postgres
|
||||
from sientia_do.temporal.activities.postgres import Postgres
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from scouter.activities.redis import Redis
|
||||
from scouter.activities.kafka import Kafka
|
||||
from scouter.activities.gates import Gates
|
||||
from logging import Logger
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from typing import Any
|
||||
|
||||
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
from typing import Any
|
||||
from logging import Logger
|
||||
from temporalio import activity
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
|
||||
|
||||
class BaseActivity:
|
||||
def __init__(self, logger: Logger, notification_handler: NotificationHandler):
|
||||
self.logger = logger
|
||||
self.notification_handler = notification_handler
|
||||
|
||||
@activity.defn(name="prepare_activity")
|
||||
async def prepare_activity(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Prepare the activity for the notification handler.
|
||||
|
||||
Args:
|
||||
workflow_name (str): The name of the workflow.
|
||||
schedule_name (str): The name of the schedule.
|
||||
model_name (str): The name of the model.
|
||||
model_id (str): The id of the model.
|
||||
"""
|
||||
self.notification_handler.base_notification.pipeline_name = input_data['workflow_name']
|
||||
self.notification_handler.base_notification.schedule_name = input_data['schedule_name']
|
||||
self.notification_handler.base_notification.model_name = input_data['model_name']
|
||||
self.notification_handler.base_notification.model_id = input_data['model_id']
|
||||
@@ -7,7 +7,7 @@ from kafka import KafkaProducer
|
||||
from temporalio import activity
|
||||
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from scouter.activities.base import BaseActivity
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
|
||||
|
||||
class Faker(BaseActivity):
|
||||
|
||||
@@ -2,7 +2,7 @@ from temporalio import workflow, activity
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from scouter.activities.base import BaseActivity
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
from scouter.utils.quality.filters import null_values_filter, out_of_bounds_filter
|
||||
from typing import Any
|
||||
import traceback
|
||||
|
||||
@@ -3,7 +3,7 @@ from temporalio import workflow, activity
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from logging import Logger
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from scouter.activities.base import BaseActivity
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
from typing import Any
|
||||
from kafka import KafkaConsumer
|
||||
from pandas import DataFrame
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
import traceback
|
||||
from temporalio import workflow, activity
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.pool import QueuePool
|
||||
from pandas import DataFrame
|
||||
from logging import Logger
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from scouter.activities.base import BaseActivity
|
||||
from typing import Any
|
||||
|
||||
|
||||
class Postgres(BaseActivity):
|
||||
def __init__(self, host: str, port: int,
|
||||
user: str, password: str, dbname: str,
|
||||
min_connections: int, max_connections: int,
|
||||
logger: Logger, notification_handler: NotificationHandler):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.user = user
|
||||
self.password = password
|
||||
self.dbname = dbname
|
||||
|
||||
# Create SQLAlchemy engine with connection pooling
|
||||
self.engine = create_engine(
|
||||
f'postgresql://{user}:{password}@{host}:{port}/{dbname}',
|
||||
poolclass=QueuePool,
|
||||
pool_size=min_connections,
|
||||
max_overflow=max_connections - min_connections,
|
||||
pool_pre_ping=True
|
||||
)
|
||||
self.session_factory = sessionmaker(bind=self.engine)
|
||||
|
||||
BaseActivity.__init__(self, logger, notification_handler)
|
||||
|
||||
def close(self):
|
||||
self.engine.dispose()
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
@activity.defn(name="export_data_to_postgres")
|
||||
async def export_data_to_postgres(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Exports data to a postgres table.
|
||||
|
||||
Args:
|
||||
input_data (dict[str, Any]): The data to export. Contains:
|
||||
schema (str): The schema of the table.
|
||||
table_name (str): The name of the table.
|
||||
data (DataFrame): The data to export.
|
||||
"""
|
||||
|
||||
self.logger.debug(
|
||||
f"Exporting data to postgres: {input_data['data']}")
|
||||
|
||||
schema = input_data["schema"]
|
||||
table_name = input_data["table_name"]
|
||||
data = DataFrame(input_data["data"])
|
||||
|
||||
with self.session_factory() as session:
|
||||
try:
|
||||
data.to_sql(table_name, self.engine, schema=schema,
|
||||
if_exists="append", index=False)
|
||||
session.commit()
|
||||
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id="ERROR_EXPORTING_DATA_TO_POSTGRES",
|
||||
message=f"Error exporting data to postgres: {e}",
|
||||
block="export_data_to_postgres",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
|
||||
self.logger.error(trace)
|
||||
|
||||
else:
|
||||
self.logger.debug("Data exported to postgres")
|
||||
finally:
|
||||
session.close()
|
||||
@@ -3,39 +3,19 @@ from temporalio import workflow, activity
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from logging import Logger
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from scouter.activities.base import BaseActivity
|
||||
import redis
|
||||
import json
|
||||
from sientia_do.temporal.activities.redis_base import Redis as RedisBase
|
||||
from typing import Any
|
||||
from pandas import DataFrame
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class Redis(BaseActivity):
|
||||
class Redis(RedisBase):
|
||||
def __init__(self, host: str, port: int,
|
||||
username: str, password: str,
|
||||
logger: Logger, notification_handler: NotificationHandler):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.username = username
|
||||
self.password = password
|
||||
|
||||
self.redis_client = redis.Redis(
|
||||
host=self.host,
|
||||
port=self.port,
|
||||
decode_responses=True,
|
||||
username=self.username,
|
||||
password=self.password
|
||||
)
|
||||
|
||||
BaseActivity.__init__(self, logger, notification_handler)
|
||||
|
||||
def get(self, key: str):
|
||||
history = self.redis_client.get(key)
|
||||
return json.loads(history) if history else None
|
||||
|
||||
def set(self, key: str, data: dict, ttl=600):
|
||||
self.redis_client.set(key, json.dumps(data), ex=ttl)
|
||||
RedisBase.__init__(self, host, port, username,
|
||||
password, logger, notification_handler)
|
||||
|
||||
@activity.defn(name="group_and_hold_data")
|
||||
async def group_and_hold_data(self, input_data: dict[str, Any]):
|
||||
|
||||
Reference in New Issue
Block a user