gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1081

Browse Source 
Enhance documentation across multiple modules with detailed parameter descriptions and usage examples
This commit is contained in:
vitor-aignosi
2025-05-26 16:45:50 -03:00
parent 5326051714
commit 10081b71d2
12 changed files with 294 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
laborious/utils/connectors_config.py
View File

@@ -1,3 +1,7 @@
"""
Builds the configuration for the connectors.
"""
from os import getenv
import json

14
laborious/utils/filters/conditional_filters.py
View File

@@ -4,6 +4,13 @@ from pandas import DataFrame
def filter_specific_variables_null_values(data: DataFrame, config: dict) -> bool:
"""
Returns True if the specific columns have null values, False otherwise.
Args:
- data (DataFrame): The data to filter.
- config (dict): The configuration.
Returns:
bool: True if the specific columns have null values, False otherwise.
"""
return not data[
data['variable'].isin(config['VARIABLES']) & data['value'].isna()].empty
@@ -12,5 +19,12 @@ def filter_specific_variables_null_values(data: DataFrame, config: dict) -> bool
def filter_empty_data(data: DataFrame, _config: dict) -> bool:
"""
Returns True if the data is empty, False otherwise.
Args:
- data (DataFrame): The data to filter.
- _config (dict): The configuration.
Returns:
bool: True if the data is empty, False otherwise.
"""
return data.empty

20
laborious/utils/filters/mlflow_filters.py
View File

@@ -3,6 +3,16 @@ from pandas import DataFrame
def api_error_filter(response: dict, _config: dict):
"""
Returns True if the API response is empty or the 'success' key is False, False otherwise.
Args:
- response (dict): The API response.
- _config (dict): The configuration.
Returns:
bool: True if the API response is empty or the 'success' key is False, False otherwise.
"""
if not response:
return True
@@ -13,6 +23,16 @@ def api_error_filter(response: dict, _config: dict):
def nan_values_filter(predictions: DataFrame, _config: dict):
"""
Returns True if the predictions DataFrame contains only NaN values, False otherwise.
Args:
- predictions (DataFrame): The predictions DataFrame.
- _config (dict): The configuration.
Returns:
bool: True if the predictions DataFrame contains only NaN values, False otherwise.
"""
data = predictions.replace({None: np.nan}).drop(
columns=['timestamp'], errors='ignore').infer_objects(copy=False)

31
laborious/utils/repository/model_repository.py
View File

@@ -1,11 +1,11 @@
"""
Model Monitoring Repository
This module contains the ModelMonitoringRepository class, which is responsible
for handling the communication with the Model Monitoring API.
This module contains the ModelMonitoringRepository class,
which is responsible for handling the communication with the Model Monitoring API.
It includes the methods that are used to answer ModelMonitoringService requests using
the Model Monitoring API functions.
It includes the methods that are used to answer ModelMonitoringService
requests using the Model Monitoring API functions.
By Monitoring we mean the evaluation of the performance of models, the generation of reports.
@@ -23,6 +23,18 @@ class MLFlowRepository():
username=username, password=password)
def transform(self, model_name: str, data: pd.DataFrame, model_retention: int):
"""
Transform data using a model.
Parameters:
- model_name (str): The name of the model to use for transformation.
- data (pandas.DataFrame): The data to transform.
- model_retention (int): The number of minutes to keep the model.
Returns:
- dict: A dictionary containing the transformed data.
"""
try:
return {
'success': True,
@@ -40,6 +52,17 @@ class MLFlowRepository():
}
def predict(self, model_name: str, data: pd.DataFrame, model_retention: int):
"""
Predict data using a model.
Parameters:
- model_name (str): The name of the model to use for prediction.
- data (pandas.DataFrame): The data to predict.
- model_retention (int): The number of minutes to keep the model.
Returns:
- dict: A dictionary containing the predicted data.
"""
try:
start_time = datetime.now()
data = self.model_serving.get_cached_predict(

53
laborious/utils/repository/opc_repository.py
View File

@@ -1,12 +1,12 @@
import traceback
from logging import Logger
from datetime import datetime
from pathlib import Path
from asyncua.sync import Client
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from asyncua.ua import DataValue, Variant, VariantType
from logging import Logger
from datetime import datetime
from sientia_do.notifications.handlers import NotificationHandler
from sientia_do.notifications.models import NotificationLevel
import traceback
data_type_map = {
'float': {
@@ -33,7 +33,8 @@ data_type_map = {
class OpcRepository():
def __init__(self, name: str, url: str, logger: Logger, notification_handler: NotificationHandler,
def __init__(self, name: str, url: str, logger: Logger,
notification_handler: NotificationHandler,
reconnection_interval: int = 60, server_uri: str = None, cert_path: str = None,
private_key_path: str = None, server_cert_path: str = None):
self.url = url
@@ -57,12 +58,12 @@ class OpcRepository():
Raises:
ValueError: If either the certificate path or private key path is not provided.
Attributes:
cert_path (str): Path to the client's certificate file.
private_key_path (str): Path to the client's private key file.
server_cert_path (str, optional): Path to the server's certificate file.
server_uri (str): The URI of the server to be used as the application URI.
client (opcua.Client): The OPC UA client instance.
logger (logging.Logger): Logger instance for logging information.
- cert_path (str): Path to the client's certificate file.
- private_key_path (str): Path to the client's private key file.
- server_cert_path (str, optional): Path to the server's certificate file.
- server_uri (str): The URI of the server to be used as the application URI.
- client (opcua.Client): The OPC UA client instance.
- logger (logging.Logger): Logger instance for logging information.
Security Settings:
- Security Policy: Basic256
- Secure Channel Timeout: 10,000,000 ms
@@ -105,6 +106,12 @@ class OpcRepository():
return self.try_connect()
def try_connect(self):
"""
Tries to connect to the OPC server.
Returns:
bool: True if the connection was successful, False otherwise.
"""
try:
self.last_reconnection_time = datetime.now()
self.client.connect()
@@ -122,6 +129,9 @@ class OpcRepository():
return False
def disconnect(self):
"""
Disconnects from the OPC server.
"""
if self.client is None:
return
self.client.disconnect()
@@ -129,12 +139,25 @@ class OpcRepository():
self.logger.info('Disconnected from OPC server')
def __del__(self):
"""
Disconnects from the OPC server when the object is destroyed.
"""
try:
self.disconnect()
except Exception as e:
self.logger.error(f"Error in destructor: {e}")
def validate_connection(self):
"""
Validates the connection to the OPC server.
If the connection is not established, it attempts to reconnect.
If the connection is established but the client is not connected,
it attempts to reconnect.
If the connection is established but the client is connected,
it checks if the client is connected to the OPC server.
If the client is not connected, it attempts to reconnect.
If the client is connected, it returns True.
"""
if self.client is None:
return self.connect()
@@ -168,6 +191,16 @@ class OpcRepository():
return True
def write_data(self, node, value, data_type):
"""
Writes data to the OPC server.
If the connection is not established, it attempts to reconnect.
If the connection is established but the client is not connected,
it attempts to reconnect.
If the connection is established but the client is connected,
it checks if the client is connected to the OPC server.
If the client is not connected, it attempts to reconnect.
If the client is connected, it returns True.
"""
if not self.validate_connection():
return
try: