gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1646

Browse Source 
Enhance orchestration configuration and documentation. Added `RUNTIME` variable to `.env.example`, updated `.gitignore` to exclude `openspec/` and `.cursor/`, and modified `README.md` to clarify queue naming conventions and runtime handling. Refactored activities to use synchronous database and email handling, improving performance and consistency. Updated test cases to reflect these changes and ensure compatibility with new activity definitions.
This commit is contained in:
vitor-aignosi
2026-05-22 15:02:16 -03:00
parent 4d5231ca16
commit 474c2ef42c
57 changed files with 2744 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
orchestrator/activities/activities.py
View File

@@ -6,7 +6,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.temporal.activities.postgres import Postgres
from sientia_do.temporal.activities.postgres_sync import Postgres
from orchestrator.activities.email import Email
from orchestrator.activities.formatters import Formatters

4
orchestrator/activities/email.py
View File

@@ -84,7 +84,7 @@ class Email(SientiaMonitoring):
self.close()
@activity.defn(name='build_email_html')
async def build_email_html(self, input_data: dict[str, Any]) -> dict[str, Any]:
def build_email_html(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Build HTML email content for configured receiver groups.
@@ -193,7 +193,7 @@ class Email(SientiaMonitoring):
self.server.sendmail(self.sender_email, receivers, msg.as_string())
@activity.defn(name='send_email')
async def send_email(self, input_data: dict[str, Any]) -> dict[str, Any]:
def send_email(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Send email notifications to configured receiver groups.

56
orchestrator/activities/formatters.py
View File

@@ -2,10 +2,10 @@ from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
import json
from collections.abc import Callable, Hashable
from collections.abc import Callable
from logging import Logger
from math import ceil
from typing import Any, TypedDict
from typing import Any, TypedDict, cast
from pandas import DataFrame
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
@@ -152,7 +152,7 @@ class Formatters(SientiaMonitoring):
self.close()
@activity.defn(name='process_schedules')
async def process_schedules(self, input_data: dict[str, Any]) -> dict[str, Any]:
def process_schedules(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Process pipeline configurations into Temporal-compatible schedule configurations.
@@ -209,7 +209,7 @@ class Formatters(SientiaMonitoring):
return schedule_config
@activity.defn(name='process_slots')
async def process_slots(self, input_data: dict[str, Any]) -> dict[str, Any]:
def process_slots(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Extracts all read tags from input pipelines, divides them into slots and
returns a slot config dictionary. If no ingestor is available, only one slot
@@ -254,7 +254,7 @@ class Formatters(SientiaMonitoring):
slot_config[f'{i}'], notifications = build_tag_config(slot_tags, opc_servers)
if notifications:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='ORCHESTRATOR_SERVER_NOT_FOUND_DURING_SLOT_CONFIGURATION',
message=f'Servers {", ".join(notifications)} not found in opc_servers',
@@ -267,7 +267,7 @@ class Formatters(SientiaMonitoring):
slot_tags = tags[last_index:]
slot_config[f'{number_of_slots}'], notifications = build_tag_config(slot_tags, opc_servers)
if notifications:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='ORCHESTRATOR_SERVER_NOT_FOUND_DURING_SLOT_CONFIGURATION',
message=f'Servers {", ".join(notifications)} not found in opc_servers',
@@ -281,7 +281,7 @@ class Formatters(SientiaMonitoring):
return slot_config
@activity.defn(name='format_schedule_config')
async def format_schedule_config(self, input_data: dict[str, Any]) -> dict[str, Any]:
def format_schedule_config(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Format the schedule config to a dictionary with the schedule name as the key.
@@ -358,7 +358,7 @@ class Formatters(SientiaMonitoring):
to_create[namespace][schedule_name] = schedule
@activity.defn(name='create_schedule_config')
async def create_schedule_config(self, input_data: dict[str, Any]) -> dict[str, Any]:
def create_schedule_config(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Create a schedule config dictionary based on the input data.
@@ -418,7 +418,7 @@ class Formatters(SientiaMonitoring):
return output
@activity.defn(name='create_slot_config')
async def create_slot_config(self, input_data: dict[str, Any]) -> dict[str, Any]:
def create_slot_config(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Create a slot config dictionary based on the input data.
@@ -459,7 +459,7 @@ class Formatters(SientiaMonitoring):
return output
async def send_success_report(
def send_success_report(
self,
metadata: dict[str, Any],
message: str,
@@ -478,7 +478,7 @@ class Formatters(SientiaMonitoring):
notification_id (str): The ID of the notification to send
attachment (Any | None, optional): Optional attachment content to include
"""
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=notification_id,
message=message,
@@ -487,7 +487,7 @@ class Formatters(SientiaMonitoring):
attachment_content=json.dumps(attachment, indent=4, sort_keys=True),
)
async def send_error_report(
def send_error_report(
self, metadata: dict[str, Any], message: str, notification_id: str, attachment: str
) -> None:
"""
@@ -502,7 +502,7 @@ class Formatters(SientiaMonitoring):
notification_id (str): The ID of the notification
attachment (str): The attachment content for the notification
"""
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=notification_id,
message=message,
@@ -569,7 +569,7 @@ class Formatters(SientiaMonitoring):
return success_keys, error_keys
async def manage_and_send_report(
def manage_and_send_report(
self,
metadata: dict[str, Any],
success_keys: list[str],
@@ -591,7 +591,7 @@ class Formatters(SientiaMonitoring):
schedule_data (dict[str, Any]): The schedule data containing items and notification ID
"""
if len(success_keys) > 0:
await self.send_success_report(
self.send_success_report(
metadata=metadata,
message=f'Successfully {schedule_type}: \n {", ".join(success_keys)}',
notification_id=schedule_data['id'],
@@ -606,7 +606,7 @@ class Formatters(SientiaMonitoring):
else:
attachment.append(f'{key}:\n{value["message"]}')
await self.send_error_report(
self.send_error_report(
metadata=metadata,
message=f'Fails on {schedule_type}: \n {", ".join(error_keys)}',
notification_id=f'{schedule_data["id"]}_ERROR',
@@ -614,7 +614,7 @@ class Formatters(SientiaMonitoring):
)
@activity.defn(name='report_schedule_orchestration')
async def report_schedule_orchestration(self, input_data: dict[str, Any]) -> None:
def report_schedule_orchestration(self, input_data: dict[str, Any]) -> None:
"""
Report the orchestration result to the notification handler.
@@ -655,7 +655,7 @@ class Formatters(SientiaMonitoring):
if len(schedule_data['items']) > 0:
success_keys, error_keys = self.parse_report_schedule(schedule_data['items'])
await self.manage_and_send_report(
self.manage_and_send_report(
metadata=metadata,
success_keys=success_keys,
error_keys=error_keys,
@@ -664,7 +664,7 @@ class Formatters(SientiaMonitoring):
)
@activity.defn(name='report_slot_orchestration')
async def report_slot_orchestration(self, input_data: dict[str, Any]) -> None:
def report_slot_orchestration(self, input_data: dict[str, Any]) -> None:
"""
Report the slot orchestration result to the notification handler.
@@ -689,14 +689,14 @@ class Formatters(SientiaMonitoring):
success_keys, error_keys = self.parse_report(inserted_slots)
if len(success_keys) > 0:
await self.send_success_report(
self.send_success_report(
metadata=metadata,
message=f'Inserted slots: \n {", ".join(success_keys)}',
notification_id='REPORT_ORCHESTRATION_INSERTED_SLOTS',
)
if len(error_keys) > 0:
await self.send_error_report(
self.send_error_report(
metadata=metadata,
message=f'Failed to insert slots: \n {", ".join(error_keys)}',
notification_id='REPORT_ORCHESTRATION_INSERTED_SLOTS',
@@ -707,14 +707,14 @@ class Formatters(SientiaMonitoring):
success_keys, error_keys = self.parse_report(deleted_slots)
if len(success_keys) > 0:
await self.send_success_report(
self.send_success_report(
metadata=metadata,
message=f'Deleted slots: \n {", ".join(success_keys)}',
notification_id='REPORT_ORCHESTRATION_DELETED_SLOTS',
)
if len(error_keys) > 0:
await self.send_error_report(
self.send_error_report(
metadata=metadata,
message=f'Failed to delete slots: \n {", ".join(error_keys)}',
notification_id='REPORT_ORCHESTRATION_DELETED_SLOTS',
@@ -722,7 +722,7 @@ class Formatters(SientiaMonitoring):
)
@activity.defn(name='format_log_report')
async def format_log_report(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
def format_log_report(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Format the receiver_groups status to a dataframe to be stored in the database.
@@ -736,7 +736,7 @@ class Formatters(SientiaMonitoring):
- metadata (dict): Metadata for logging purposes
Returns:
dict[Hashable, Any]: The formatted log report as a dictionary representation of a DataFrame
dict[str, Any]: The formatted log report as a dictionary representation of a DataFrame
"""
metadata = input_data['metadata']
mail_type = input_data['mail_type']
@@ -776,10 +776,12 @@ class Formatters(SientiaMonitoring):
data_values: DataFrame = DataFrame(list(data.values()))
return data_values.to_dict()
# ``DataFrame.to_dict()`` is typed as ``dict[Hashable, Any]`` in pandas
# stubs, but default orientation uses column names (str keys).
return cast(dict[str, Any], data_values.to_dict())
@activity.defn(name='filter_notification_reports')
async def filter_notification_reports(self, input_data: dict[str, Any]) -> dict[str, Any]:
def filter_notification_reports(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Filter notifications for comprehensive scheduled reports.

57
orchestrator/activities/mongo_db.py
View File

@@ -9,10 +9,9 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.sientia_monitoring import MetricsController, SientiaMonitoring
from sientia_do.repository.mongodb_repository import MongoDBRepository
from sientia_do.repository.mongodb_repository_sync import MongoDBRepository
from sientia_do.temporal.constants import (
DATETIME_FORMAT_MS_WITH_TZ,
DATETIME_FORMAT_WITH_TZ,
now,
)
@@ -84,7 +83,7 @@ class MongoDB(SientiaMonitoring):
@activity.defn(
name='find_documents_in_mongodb',
)
async def find_documents_in_mongodb(self, input_data: dict[str, Any]) -> list[dict[str, Any]]:
def find_documents_in_mongodb(self, input_data: dict[str, Any]) -> list[dict[str, Any]]:
"""
Find documents in a MongoDB collection based on the provided query parameters.
@@ -113,7 +112,7 @@ class MongoDB(SientiaMonitoring):
)
try:
documents = await self.mongo_db_repository.find(collection_name, filters, metadata)
documents = self.mongo_db_repository.find(collection_name, filters, metadata)
self.info(
f"Loaded {len(documents)} documents from collection '{collection_name}'",
@@ -135,7 +134,7 @@ class MongoDB(SientiaMonitoring):
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='MONGODB_QUERY_ERROR',
message=f'Failed to execute MongoDB query: {e}',
@@ -148,7 +147,7 @@ class MongoDB(SientiaMonitoring):
raise e
@activity.defn(name='aggregate_documents_in_mongodb')
async def aggregate_documents_in_mongodb(
def aggregate_documents_in_mongodb(
self, input_data: dict[str, Any]
) -> list[dict[str, Any]]:
"""
@@ -181,7 +180,7 @@ class MongoDB(SientiaMonitoring):
)
try:
aggregated_documents = await self.mongo_db_repository.aggregate(
aggregated_documents = self.mongo_db_repository.aggregate(
collection_name, aggregation, metadata
)
@@ -205,7 +204,7 @@ class MongoDB(SientiaMonitoring):
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='MONGODB_AGGREGATION_ERROR',
message=f'Failed to execute MongoDB aggregation: {e}',
@@ -218,7 +217,7 @@ class MongoDB(SientiaMonitoring):
raise e
@activity.defn(name='update_pipelines_timestamps')
async def update_pipelines_timestamps(self, input_data: dict[str, Any]) -> None:
def update_pipelines_timestamps(self, input_data: dict[str, Any]) -> None:
"""
Update `updated_at` timestamps for successfully updated pipelines.
@@ -244,12 +243,12 @@ class MongoDB(SientiaMonitoring):
data_filter = {'$or': argument} if argument else {}
try:
await self.mongo_db_repository.update_many(
self.mongo_db_repository.update_many(
'orchestrated_schedules', data_filter, {'$set': {'updated_at': date_now}}, metadata
)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='MONGODB_UPDATE_PIPELINES_ERROR',
message=f'Failed to update pipelines timestamps: {e}',
@@ -266,7 +265,7 @@ class MongoDB(SientiaMonitoring):
)
@activity.defn(name='create_pipelines_timestamps')
async def create_pipelines_timestamps(self, input_data: dict[str, Any]) -> None:
def create_pipelines_timestamps(self, input_data: dict[str, Any]) -> None:
"""
Insert `updated_at` timestamps for newly created pipelines.
@@ -298,12 +297,12 @@ class MongoDB(SientiaMonitoring):
try:
if data_filter:
await self.mongo_db_repository.insert_many(
self.mongo_db_repository.insert_many(
'orchestrated_schedules', data_filter, metadata
)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='MONGODB_CREATE_PIPELINES_ERROR',
message=f'Failed to create pipelines timestamps: {e}',
@@ -320,7 +319,7 @@ class MongoDB(SientiaMonitoring):
)
@activity.defn(name='delete_pipelines_timestamps')
async def delete_pipelines_timestamps(self, input_data: dict[str, Any]) -> None:
def delete_pipelines_timestamps(self, input_data: dict[str, Any]) -> None:
"""
Delete timestamp rows for successfully deleted pipelines.
@@ -345,12 +344,12 @@ class MongoDB(SientiaMonitoring):
data_filter = {'$or': argument} if argument else {}
try:
await self.mongo_db_repository.delete_many(
self.mongo_db_repository.delete_many(
'orchestrated_schedules', data_filter, metadata
)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='MONGODB_DELETE_PIPELINES_ERROR',
message=f'Failed to delete pipelines timestamps: {e}',
@@ -367,7 +366,7 @@ class MongoDB(SientiaMonitoring):
)
@activity.defn(name='create_collection_with_ttl_index')
async def create_collection_with_ttl_index(self, input_data: dict[str, Any]) -> None:
def create_collection_with_ttl_index(self, input_data: dict[str, Any]) -> None:
"""
Create collections with TTL indexes for pipeline topics.
@@ -424,7 +423,7 @@ class MongoDB(SientiaMonitoring):
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='MONGODB_CREATE_COLLECTION_ERROR',
message=f'Failed to create collection {collection} with TTL index: {e}',
@@ -444,7 +443,7 @@ class MongoDB(SientiaMonitoring):
self.debug(f'Created indexes: {created_indexes}', metadata=metadata)
@activity.defn(name='load_latest_data')
async def load_latest_data(self, input_data: dict[str, Any]) -> list[dict[str, Any]]:
def load_latest_data(self, input_data: dict[str, Any]) -> list[dict[str, Any]]:
"""
Load the latest data from MongoDB collection since a specified timestamp.
@@ -477,16 +476,24 @@ class MongoDB(SientiaMonitoring):
if last_data_timestamp is None:
data_filter = base_data_filter
else:
# ``notification_queue.timestamp`` is stored as a string in
# ``DATETIME_FORMAT_WITH_TZ`` (``Notification`` writes it as
# ``now().strftime(DATETIME_FORMAT_WITH_TZ)``). Coercing
# ``last_data_timestamp`` to ``datetime`` here would force a
# BSON ``String`` vs ``Date`` comparison, which always yields
# ``False`` (``String < Date`` in BSON sort order) and breaks
# incremental loading entirely. Comparing strings preserves the
# intended chronological filter because the format is
# lexicographically ordered when the timezone is fixed
# (``Notification.timestamp`` always uses UTC).
data_filter = {
**base_data_filter,
'timestamp': {
'$gt': datetime.strptime(last_data_timestamp, DATETIME_FORMAT_WITH_TZ)
},
'timestamp': {'$gt': last_data_timestamp},
}
self.debug(f'Data filter: {data_filter}', metadata=metadata)
data = await self.mongo_db_repository.find(collection_name, data_filter, metadata)
data = self.mongo_db_repository.find(collection_name, data_filter, metadata)
self.debug(f'Collected: {data}', metadata=metadata)
@@ -497,7 +504,7 @@ class MongoDB(SientiaMonitoring):
return data
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='MONGO_LOAD_ERROR',
message=f'Error loading data from MongoDB: {e}',

44
orchestrator/activities/slot_manager.py
View File

@@ -11,7 +11,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.sientia_monitoring import MetricsController, SientiaMonitoring
from sientia_do.repository.redis_repository import RedisRepository
from sientia_do.repository.redis_repository_sync import RedisRepository
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ, now
@@ -84,7 +84,7 @@ class SlotManager(SientiaMonitoring):
self.close()
@activity.defn(name='load_opc_slots')
async def load_opc_slots(self, input_data: dict[str, Any]) -> dict[str, Any]:
def load_opc_slots(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Load all OPC slots from Redis for current system state assessment.
@@ -112,7 +112,7 @@ class SlotManager(SientiaMonitoring):
opc_slots = {}
try:
slot_keys = await self.redis_repository.keys('slot:opc_tags:*')
slot_keys = self.redis_repository.keys('slot:opc_tags:*')
self.debug(f'Slot keys: {slot_keys}', metadata=metadata)
@@ -123,10 +123,10 @@ class SlotManager(SientiaMonitoring):
decoded_keys = slot_keys
for key in decoded_keys:
opc_slots[key] = await self.redis_repository.get(key)
opc_slots[key] = self.redis_repository.get(key)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='REDIS_GET_ERROR',
message=f'Failed to load OPC slots: {e}',
@@ -142,7 +142,7 @@ class SlotManager(SientiaMonitoring):
return opc_slots
@activity.defn(name='load_active_ingestors')
async def load_active_ingestors(self, input_data: dict[str, Any]) -> list[str]:
def load_active_ingestors(self, input_data: dict[str, Any]) -> list[str]:
"""
Load all active ingestors from Redis.
@@ -161,7 +161,7 @@ class SlotManager(SientiaMonitoring):
self.info('Loading active ingestors...', metadata=metadata)
try:
active_ingestors = await self.redis_repository.keys('heartbeat:ingestor:*')
active_ingestors = self.redis_repository.keys('heartbeat:ingestor:*')
self.info(f'Loaded {len(active_ingestors)} active ingestors', metadata=metadata)
@@ -178,7 +178,7 @@ class SlotManager(SientiaMonitoring):
return ingestors
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='REDIS_GET_ERROR',
message=f'Failed to load active ingestors: {e}',
@@ -190,7 +190,7 @@ class SlotManager(SientiaMonitoring):
raise e
@activity.defn(name='update_slots')
async def update_slots(self, input_data: dict[str, Any]) -> dict[str, Any]:
def update_slots(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Update OPC slots in Redis
@@ -213,7 +213,7 @@ class SlotManager(SientiaMonitoring):
for slot in to_insert:
try:
await self.redis_repository.set(f'slot:opc_tags:{slot}', to_insert[slot], ttl=None)
self.redis_repository.set(f'slot:opc_tags:{slot}', to_insert[slot], ttl=None)
report[slot] = {'success': True, 'message': 'Slot updated successfully'}
success_count += 1
except Exception as e:
@@ -227,7 +227,7 @@ class SlotManager(SientiaMonitoring):
return report
@activity.defn(name='delete_slots')
async def delete_slots(self, input_data: dict[str, Any]) -> dict[str, Any]:
def delete_slots(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Delete OPC slots from Redis
@@ -251,7 +251,7 @@ class SlotManager(SientiaMonitoring):
for slot in to_delete:
try:
await self.redis_repository.delete(f'slot:opc_tags:{slot}')
self.redis_repository.delete(f'slot:opc_tags:{slot}')
report[slot] = {'success': True, 'message': 'Slot deleted successfully'}
success_count += 1
except Exception as e:
@@ -265,7 +265,7 @@ class SlotManager(SientiaMonitoring):
return report
@activity.defn(name='get_last_data_timestamp')
async def get_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
def get_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
"""
Get the last data timestamp from Redis.
@@ -285,9 +285,9 @@ class SlotManager(SientiaMonitoring):
key = f'notification_last_timestamp:{input_data["mail_type"]}'
try:
data_hold = await self.redis_repository.get(key)
data_hold = self.redis_repository.get(key)
except Exception as e:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='REDIS_GET_ERROR',
message=f'Error getting last data timestamp: {e}',
@@ -305,7 +305,7 @@ class SlotManager(SientiaMonitoring):
return data_hold
@activity.defn(name='put_last_data_timestamp')
async def put_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
def put_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
"""
Store the last data timestamp in Redis.
@@ -336,9 +336,9 @@ class SlotManager(SientiaMonitoring):
self.debug(f'Last collected timestamp to insert: {last_data_timestamp}', metadata=metadata)
try:
await self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5)
self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5)
except Exception as e:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='REDIS_SET_ERROR',
message=f'Error setting last data timestamp: {e}',
@@ -351,7 +351,7 @@ class SlotManager(SientiaMonitoring):
return last_data_timestamp
@activity.defn(name='filter_notification_alerts')
async def filter_notification_alerts(self, input_data: dict[str, Any]) -> dict[str, Any]:
def filter_notification_alerts(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Filter notification alerts with intelligent TTL-based duplicate prevention.
@@ -407,7 +407,7 @@ class SlotManager(SientiaMonitoring):
# Check if notification was recently sent
key = f'{notification["trigger"]}:{notification_id}'
last_sent = await self.redis_repository.get(key)
last_sent = self.redis_repository.get(key)
if last_sent is None:
alert_type = 'core_alerts'
@@ -438,7 +438,7 @@ class SlotManager(SientiaMonitoring):
return receiver_groups
@activity.defn(name='store_notification_cache')
async def store_notification_cache(self, input_data: dict[str, Any]) -> None:
def store_notification_cache(self, input_data: dict[str, Any]) -> None:
"""
Store notification cache in Redis to track recently sent notifications.
@@ -463,6 +463,6 @@ class SlotManager(SientiaMonitoring):
status = row['status']
if status == 'sent':
key = f'{row["schedule"]}:{row["notification_id"]}'
await self.redis_repository.set(key, date_now, ttl=sent_ttl)
self.redis_repository.set(key, date_now, ttl=sent_ttl)
self.info('Notification cache stored...', metadata=metadata)

9
orchestrator/activities/temporal_manager.py
View File

@@ -20,6 +20,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.sientia_monitoring import MetricsController, SientiaMonitoring
from sientia_do.temporal.worker.prepare_worker import build_queue_name
from orchestrator.utils.converters import parse_frequency
@@ -220,7 +221,9 @@ class TemporalManager(SientiaMonitoring):
workflow_type,
schedule,
id=schedule_name,
task_queue=f'{workflow_type}-queue',
task_queue=build_queue_name(
workflow_type, schedule.get('runtime', 'legacy')
),
execution_timeout=timedelta(seconds=execution_timeout_seconds),
run_timeout=timedelta(seconds=execution_timeout_seconds),
task_timeout=timedelta(seconds=task_timeout_seconds),
@@ -278,6 +281,10 @@ class TemporalManager(SientiaMonitoring):
"""
Update schedules in Temporal.
Does not modify ``task_queue``. The ``ScheduleUpdate`` callback only patches
workflow ``args`` and schedule ``intervals``. A ``runtime`` change requires
delete-then-create on the next orchestrator tick.
Args:
- input_data (dict[str, Any]): The input data containing
the schedules to update.

1
orchestrator/utils/orchestrator_functions.py
View File

@@ -39,6 +39,7 @@ def common_config(config: dict[str, Any]):
'execution_timeout_seconds': config.get('execution_timeout_seconds', 300),
'task_timeout_seconds': config.get('task_timeout_seconds', 300),
'on_conflict': config.get('on_conflict', 'error'),
'runtime': config.get('runtime', 'legacy'),
}

72
orchestrator/worker/prepare_worker.py
View File

@@ -1,72 +0,0 @@
import os
import re
from collections.abc import Sequence
from typing import Any
from sientia_do.observability.logger import Logger
from temporalio.client import Client
from temporalio.worker import PollerBehaviorAutoscaling, Worker
parameters = [
('MAX_CONCURRENT_WORKFLOW_TASKS', '200'),
('MAX_CONCURRENT_ACTIVITIES', '200'),
('MAX_CONCURRENT_LOCAL_ACTIVITIES', '200'),
('MAX_CACHED_WORKFLOWS', '200'),
('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10'),
('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100'),
('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200'),
('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10'),
('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100'),
('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200'),
]
def camel_to_snake(text: str) -> str:
"""Convert camelCase or PascalCase to snake_case."""
text = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', text)
text = re.sub('([a-z0-9])([A-Z])', r'\1_\2', text)
return text.lower()
def prepare_worker(
main_workflow: type,
other_workflows: Sequence[type],
activities: Sequence[Any],
temporal_client: Client,
logger: Logger,
) -> Worker:
main_workflow_name = main_workflow.__name__.upper()
queue_name = f'{camel_to_snake(main_workflow.__name__)}-queue'
local_workflow_parameters = {}
for parameter in parameters:
local_workflow_parameters[parameter[0]] = int(
os.getenv(main_workflow_name + '_' + parameter[0], parameter[1])
)
logger.info(f'Preparing worker for {main_workflow_name} with queue {queue_name}')
return Worker(
temporal_client,
task_queue=queue_name,
workflows=[main_workflow, *other_workflows],
activities=[*activities],
max_concurrent_workflow_tasks=local_workflow_parameters['MAX_CONCURRENT_WORKFLOW_TASKS'],
max_concurrent_activities=local_workflow_parameters['MAX_CONCURRENT_ACTIVITIES'],
max_concurrent_local_activities=local_workflow_parameters[
'MAX_CONCURRENT_LOCAL_ACTIVITIES'
],
max_cached_workflows=local_workflow_parameters['MAX_CACHED_WORKFLOWS'],
workflow_task_poller_behavior=PollerBehaviorAutoscaling(
minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MINIMUM'],
initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_INITIAL'],
maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM'],
),
activity_task_poller_behavior=PollerBehaviorAutoscaling(
minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MINIMUM'],
initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_INITIAL'],
maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM'],
),
)

17
orchestrator/worker/worker.py
View File

@@ -9,6 +9,7 @@ with workflow.unsafe.imports_passed_through():
from prometheus_client import start_http_server
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import get_logger
from sientia_do.temporal.worker.prepare_worker import prepare_worker
from orchestrator import metrics
from orchestrator.activities.activities import Activities
@@ -19,7 +20,6 @@ with workflow.unsafe.imports_passed_through():
build_redis_config,
build_temporal_config,
)
from orchestrator.worker.prepare_worker import prepare_worker
from orchestrator.workflows.alerts import Alerts
from orchestrator.workflows.orchestrator import Orchestrator
from orchestrator.workflows.reports import Reports
@@ -136,7 +136,7 @@ async def main():
activities.report_slot_orchestration,
activities.format_schedule_config,
],
logger=logger,
logger=logger
),
prepare_worker(
temporal_client=temporal_client,
@@ -158,7 +158,7 @@ async def main():
# Store notification cache
activities.store_notification_cache,
],
logger=logger,
logger=logger
),
prepare_worker(
temporal_client=temporal_client,
@@ -178,7 +178,7 @@ async def main():
activities.format_log_report,
activities.export_data_to_postgres,
],
logger=logger,
logger=logger
),
]
@@ -188,20 +188,19 @@ async def main():
logger.custom_info('Workers started successfully', metadata=metadata)
exit_code = 0
try:
# This will run the workers and wait for them to complete.
# If an exception occurs in any of the worker handlers, it will be propagated here.
await asyncio.gather(*handlers)
except BaseException as e:
logger.error(f'An unhandled exception occurred: {e}', exc_info=True)
exit_code = 1
finally:
if notification_handler:
notification_handler.shutdown()
if activities:
activities.shutdown()
# Exit with a non-zero status code to indicate failure to Kubernetes
metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN
sys.exit(1)
metrics.APP_UP.labels(pod_id=POD_ID).set(0)
sys.exit(exit_code)
def start_prometheus_server():

4
orchestrator/workflows/subworkflows/process_notifications.py
View File

@@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through():
from datetime import timedelta
from typing import Any
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from sientia_do.temporal.policies import retry_policy
from orchestrator.activities.activities import Activities
@@ -88,7 +88,7 @@ class ProcessNotifications:
'data': log_report,
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_MS_WITH_TZ,
'format': DATETIME_FORMAT_WITH_TZ,
},
},
schedule_to_close_timeout=timedelta(seconds=60),