gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update dependencies, improve CI workflow, and enhance code formatting

Browse Source 
- Updated the `sientia-dataops-library` dependency version from 1.4.3 to 1.4.6 in `requirements.txt`.
- Modified the GitHub Actions workflow to install development and runtime dependencies separately, improving clarity and organization.
- Added code formatting and linting checks using Ruff, along with type checking using mypy, to ensure code quality.
- Updated `.gitignore` to include additional cache directories and log files.
- Refactored code in various files for consistency in string formatting and improved logging messages.
This commit is contained in:
vitor-aignosi
2025-10-17 12:58:24 -03:00
parent 464c9aae9b
commit e2462af31c
21 changed files with 1536 additions and 1368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
ingestor/managers/resource_manager.py
View File

@@ -1,13 +1,14 @@
import json
from typing import List
from redis import Redis
from time import time
import ingestor.metrics as metrics
from redis import Redis
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.temporal.activities.base import BaseActivity
import ingestor.metrics as metrics
class ResourceManager(BaseActivity):
"""
@@ -80,9 +81,9 @@ class ResourceManager(BaseActivity):
Metrics:
- REDIS_CONNECTION_STATUS: Set to 1 on successful connection, 0 on failure
"""
BaseActivity.__init__(self, logger=logger,
notification_handler=notification_handler,
set_error_counter=True)
BaseActivity.__init__(
self, logger=logger, notification_handler=notification_handler, set_error_counter=True
)
try:
self.redis = Redis(
host=host,
@@ -94,7 +95,7 @@ class ResourceManager(BaseActivity):
self.redis.ping()
metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1)
except Exception as e:
self.logger.error(f"Failed to connect to Redis: {e}")
self.logger.error(f'Failed to connect to Redis: {e}')
metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
raise
@@ -146,9 +147,9 @@ class ResourceManager(BaseActivity):
).inc()
self.send_notification(
metadata=self.metadata,
notification_id=f"REDIS_OPERATION_ERROR_{operation_name}",
notification_id=f'REDIS_OPERATION_ERROR_{operation_name}',
message=f"Error in Redis operation '{operation_name}': {e}",
block="redis_manager",
block='redis_manager',
level=NotificationLevel.ERROR,
)
raise
@@ -172,7 +173,7 @@ class ResourceManager(BaseActivity):
- REDIS_OPERATIONS_DURATION: Records timing for get operations
"""
history = self._execute_redis_op("get", self.redis.get, key)
history = self._execute_redis_op('get', self.redis.get, key)
return json.loads(history) if history else None
def get_tag_slot(self, id: str) -> dict:
@@ -193,7 +194,7 @@ class ResourceManager(BaseActivity):
and delegates to the get() method for the actual Redis operation.
"""
return self.get(f"slot:opc_tags:{id}")
return self.get(f'slot:opc_tags:{id}')
def ingestor_heartbeat(self) -> None:
"""
@@ -215,9 +216,9 @@ class ResourceManager(BaseActivity):
"""
self._execute_redis_op(
"set",
'set',
self.redis.set,
f"heartbeat:ingestor:{self.pod_id}",
f'heartbeat:ingestor:{self.pod_id}',
1,
ex=self.heartbeat_ttl,
)
@@ -249,9 +250,9 @@ class ResourceManager(BaseActivity):
"""
return self._execute_redis_op(
"set_nx",
'set_nx',
self.redis.set,
f"lease:opc_tags:{tag_id}",
f'lease:opc_tags:{tag_id}',
self.pod_id,
nx=True,
ex=self.lease_ttl,
@@ -282,12 +283,10 @@ class ResourceManager(BaseActivity):
- REDIS_OPERATIONS_DURATION: Records timing for renewal operations
"""
current = self._execute_redis_op(
"get", self.redis.get, f"lease:opc_tags:{tag_id}"
)
current = self._execute_redis_op('get', self.redis.get, f'lease:opc_tags:{tag_id}')
if current == self.pod_id:
self._execute_redis_op(
"expire", self.redis.expire, f"lease:opc_tags:{tag_id}", self.lease_ttl
'expire', self.redis.expire, f'lease:opc_tags:{tag_id}', self.lease_ttl
)
return True
return False
@@ -313,10 +312,9 @@ class ResourceManager(BaseActivity):
- REDIS_OPERATIONS_DURATION: Records timing for lease dropping operations
"""
self._execute_redis_op("delete", self.redis.delete,
f"lease:opc_tags:{tag_id}")
self._execute_redis_op('delete', self.redis.delete, f'lease:opc_tags:{tag_id}')
def get_all_ingestors(self) -> List[str]:
def get_all_ingestors(self) -> list[str]:
"""
Retrieves all active ingestors from Redis.
@@ -338,9 +336,9 @@ class ResourceManager(BaseActivity):
- REDIS_OPERATIONS_DURATION: Records timing for ingestor discovery
"""
return self._execute_redis_op("keys", self.redis.keys, "heartbeat:ingestor:*")
return self._execute_redis_op('keys', self.redis.keys, 'heartbeat:ingestor:*')
def get_all_slots(self) -> List[str]:
def get_all_slots(self) -> list[str]:
"""
Retrieves all available slots from Redis.
@@ -362,9 +360,9 @@ class ResourceManager(BaseActivity):
- REDIS_OPERATIONS_DURATION: Records timing for slot discovery
"""
return self._execute_redis_op("keys", self.redis.keys, "slot:opc_tags:*")
return self._execute_redis_op('keys', self.redis.keys, 'slot:opc_tags:*')
def get_all_leases(self) -> List[str]:
def get_all_leases(self) -> list[str]:
"""
Retrieves all active leases from Redis.
@@ -386,4 +384,4 @@ class ResourceManager(BaseActivity):
- REDIS_OPERATIONS_DURATION: Records timing for lease discovery
"""
return self._execute_redis_op("keys", self.redis.keys, "lease:opc_tags:*")
return self._execute_redis_op('keys', self.redis.keys, 'lease:opc_tags:*')