SIENTIAPDE-1182
Remove Docker configuration files and refactor project structure - Deleted docker-compose.yml and Dockerfile as part of the project restructuring. - Updated README.md to reflect changes in project setup and configuration. - Introduced a new __init__.py file in the laborious package to provide an overview of the system. - Enhanced documentation across various modules, including metrics, activities, and workflows, to improve clarity and usability. - Added comprehensive docstrings and comments to key classes and methods for better maintainability.
This commit is contained in:
vitor-aignosi
@@ -103,21 +103,44 @@ class MLFlowRepository():
|
||||
|
||||
def get_next_run_name(self, model_name: str) -> str:
|
||||
"""
|
||||
Function to get the next run number of a specific model
|
||||
Generate the next run name for a specific MLFlow model.
|
||||
|
||||
Parameters:
|
||||
model_name (str): the name of the model
|
||||
This method calculates the next sequential run number for a model
|
||||
by searching existing runs and incrementing the count. It ensures
|
||||
unique run names for model training and retraining operations.
|
||||
|
||||
Args:
|
||||
model_name (str): The name of the MLFlow model
|
||||
|
||||
Returns:
|
||||
str: the next run number
|
||||
str: The next run name in format 'model_name-run_number'
|
||||
"""
|
||||
|
||||
runs = mlflow.search_runs(
|
||||
experiment_names=[model_name], order_by=["start_time desc"])
|
||||
next_run_number = len(runs) + 1
|
||||
return f"{model_name}-{next_run_number}"
|
||||
|
||||
def create_model_experiment(self, model_name: str, data: pd.DataFrame) -> tuple:
|
||||
"""
|
||||
Create a new MLFlow experiment for model retraining.
|
||||
|
||||
This method sets up the complete environment for model retraining by:
|
||||
1. Loading the current production prediction model
|
||||
2. Loading the current production transformation model
|
||||
3. Fitting the transformation model with new data
|
||||
4. Preparing data for prediction model retraining
|
||||
5. Setting up the MLFlow experiment context
|
||||
|
||||
Args:
|
||||
model_name (str): Name of the MLFlow model to retrain
|
||||
data (pd.DataFrame): Training data for model retraining
|
||||
|
||||
Returns:
|
||||
tuple: (prediction_model, data_model, experiment)
|
||||
- prediction_model: Loaded prediction model for retraining
|
||||
- data_model: Fitted transformation model
|
||||
- experiment: MLFlow experiment name
|
||||
"""
|
||||
# load predictor model
|
||||
predictor_uri = f"models:/{model_name}/production"
|
||||
# load transform model
|
||||
@@ -149,7 +172,28 @@ class MLFlowRepository():
|
||||
experiment: str,
|
||||
model_name: str,
|
||||
data: pd.DataFrame):
|
||||
"""
|
||||
Execute the complete model retraining process in MLFlow.
|
||||
|
||||
This method performs the actual model retraining by:
|
||||
1. Starting a new MLFlow run with descriptive metadata
|
||||
2. Logging model parameters and hyperparameters
|
||||
3. Retraining both prediction and transformation models
|
||||
4. Logging training data as artifacts
|
||||
5. Saving retrained models to MLFlow registry
|
||||
|
||||
Args:
|
||||
prediction_model: MLFlow prediction model to retrain
|
||||
data_model: MLFlow transformation model to retrain
|
||||
experiment (str): MLFlow experiment name for the retraining
|
||||
model_name (str): Name of the model being retrained
|
||||
data (pd.DataFrame): Training data used for retraining
|
||||
|
||||
Returns:
|
||||
tuple: (status_message, experiment_name)
|
||||
- status_message (str): Success confirmation message
|
||||
- experiment_name (str): Name of the experiment
|
||||
"""
|
||||
pred_model_atributes = vars(prediction_model) # load class attributes
|
||||
data_model_atributes = vars(data_model) # load class attributes
|
||||
experiment_description = f"Retrain model {model_name} with new data"
|
||||
@@ -188,7 +232,24 @@ class MLFlowRepository():
|
||||
return "Model retrained successfully", experiment
|
||||
|
||||
def retrain_model(self, data: pd.DataFrame, model_name: str) -> tuple:
|
||||
"""
|
||||
Orchestrate the complete model retraining workflow.
|
||||
|
||||
This method coordinates the entire model retraining process by:
|
||||
1. Creating the MLFlow experiment environment
|
||||
2. Loading existing production models
|
||||
3. Executing the retraining process
|
||||
4. Returning comprehensive retraining results
|
||||
|
||||
Args:
|
||||
data (pd.DataFrame): Training data for model retraining
|
||||
model_name (str): Name of the MLFlow model to retrain
|
||||
|
||||
Returns:
|
||||
tuple: (status_message, experiment_name)
|
||||
- status_message (str): Retraining operation status
|
||||
- experiment_name (str): MLFlow experiment identifier
|
||||
"""
|
||||
prediction_model, data_model, experiment = self.create_model_experiment(
|
||||
model_name, data)
|
||||
retrain_result = self.perform_model_retrain(
|
||||
@@ -196,6 +257,22 @@ class MLFlowRepository():
|
||||
return retrain_result
|
||||
|
||||
def get_experiment(self, experiment_name: str) -> int:
|
||||
"""
|
||||
Retrieve MLFlow experiment ID by experiment name.
|
||||
|
||||
This method searches for an MLFlow experiment by name and
|
||||
returns its unique identifier. It provides error handling
|
||||
for non-existent experiments.
|
||||
|
||||
Args:
|
||||
experiment_name (str): Name of the MLFlow experiment
|
||||
|
||||
Returns:
|
||||
int: MLFlow experiment ID
|
||||
|
||||
Raises:
|
||||
ValueError: If the experiment name is not found
|
||||
"""
|
||||
experiment = mlflow.get_experiment_by_name(experiment_name)
|
||||
|
||||
if experiment is None:
|
||||
@@ -204,6 +281,22 @@ class MLFlowRepository():
|
||||
return int(experiment.experiment_id)
|
||||
|
||||
def get_experiment_last_run(self, experiment_id: int) -> str:
|
||||
"""
|
||||
Retrieve the most recent retraining run ID for an experiment.
|
||||
|
||||
This method searches for the latest run in an MLFlow experiment
|
||||
that has been marked as a retraining run. It filters runs by
|
||||
the 'retrain' parameter and orders them by completion time.
|
||||
|
||||
Args:
|
||||
experiment_id (int): MLFlow experiment ID
|
||||
|
||||
Returns:
|
||||
str: MLFlow run ID of the most recent retraining run
|
||||
|
||||
Raises:
|
||||
ValueError: If runs data is not in expected DataFrame format
|
||||
"""
|
||||
runs = mlflow.search_runs(
|
||||
experiment_ids=[experiment_id],
|
||||
filter_string="", # Sem filtro no MLflow ainda
|
||||
@@ -229,6 +322,29 @@ class MLFlowRepository():
|
||||
return latest_run_id
|
||||
|
||||
def update_production_model_by_run_id(self, run_id: str, model_name: str) -> dict:
|
||||
"""
|
||||
Update production model with a specific MLFlow run.
|
||||
|
||||
This method promotes a model from a specific MLFlow run to
|
||||
production stage. It handles model registration, versioning,
|
||||
and stage transitions with proper error handling.
|
||||
|
||||
Args:
|
||||
run_id (str): MLFlow run ID containing the model to promote
|
||||
model_name (str): Name of the MLFlow model
|
||||
|
||||
Returns:
|
||||
dict: Model update metadata containing:
|
||||
- model_name (str): Name of the updated model
|
||||
- version (str): New model version number
|
||||
- mlflow_run_id (str): Source run ID
|
||||
|
||||
Update Process:
|
||||
1. Registers the model from the specified run
|
||||
2. Retrieves the latest model version
|
||||
3. Transitions the model to 'Production' stage
|
||||
4. Archives existing production versions
|
||||
"""
|
||||
# Registrar o modelo
|
||||
# Aqui estamos assumindo que você já tem um modelo salvo, caso contrário você precisará treiná-lo e salvá-lo primeiro.
|
||||
# Se o modelo já está registrado, você pode usar o método register_model() ou pyfunc.load_model() para isso.
|
||||
@@ -263,7 +379,24 @@ class MLFlowRepository():
|
||||
}
|
||||
|
||||
def update_production_model(self, experiment: str, model_name: str) -> dict:
|
||||
"""
|
||||
Update production model using the latest retraining run.
|
||||
|
||||
This method orchestrates the complete production model update
|
||||
process by identifying the most recent retraining run and
|
||||
promoting it to production stage.
|
||||
|
||||
Args:
|
||||
experiment (str): MLFlow experiment name
|
||||
model_name (str): Name of the MLFlow model
|
||||
|
||||
Returns:
|
||||
dict: Complete model update metadata containing:
|
||||
- model_name (str): Name of the updated model
|
||||
- version (str): New model version number
|
||||
- mlflow_run_id (str): Source run ID
|
||||
- mlflow_experiment_id (int): Experiment ID
|
||||
"""
|
||||
experiment_id = self.get_experiment(experiment)
|
||||
run_id = self.get_experiment_last_run(experiment_id)
|
||||
metadata = self.update_production_model_by_run_id(run_id, model_name)
|
||||
|
||||
@@ -121,10 +121,17 @@ class OpcRepository():
|
||||
|
||||
async def try_connect(self) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Tries to connect to the OPC server.
|
||||
Attempt to establish connection to the OPC server.
|
||||
|
||||
This method performs the actual connection attempt to the OPC server
|
||||
and handles connection failures with comprehensive error reporting.
|
||||
It updates reconnection timing and provides detailed error information
|
||||
for operational monitoring and debugging.
|
||||
|
||||
Returns:
|
||||
bool: True if the connection was successful, False otherwise.
|
||||
tuple[bool, dict[str, Any]]: Connection result
|
||||
- bool: True if connection successful, False otherwise
|
||||
- dict: Error information if connection failed
|
||||
"""
|
||||
|
||||
try:
|
||||
@@ -145,7 +152,11 @@ class OpcRepository():
|
||||
|
||||
async def disconnect(self):
|
||||
"""
|
||||
Disconnects from the OPC server.
|
||||
Gracefully disconnect from the OPC server.
|
||||
|
||||
This method safely terminates the connection to the OPC server
|
||||
and cleans up client resources. It handles disconnection errors
|
||||
gracefully and ensures proper resource cleanup.
|
||||
"""
|
||||
if self.client is None:
|
||||
return
|
||||
@@ -160,15 +171,32 @@ class OpcRepository():
|
||||
|
||||
async def validate_connection(self) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Validates the connection to the OPC server using protocol state checking.
|
||||
Validate and maintain OPC server connection health.
|
||||
|
||||
If the connection is not established, it attempts to reconnect.
|
||||
If the connection is established but the client is not connected,
|
||||
it attempts to reconnect.
|
||||
If the connection is established but the client is connected,
|
||||
it checks if the client is connected to the OPC server.
|
||||
If the client is not connected, it attempts to reconnect.
|
||||
If the client is connected, it returns True.
|
||||
This method performs comprehensive connection validation and
|
||||
implements automatic reconnection logic for production reliability.
|
||||
It handles various connection states and implements intelligent
|
||||
reconnection strategies with error counting and timing controls.
|
||||
|
||||
Connection Validation:
|
||||
1. Checks client existence and connection state
|
||||
2. Implements error counting with automatic disconnection
|
||||
3. Enforces reconnection timing windows
|
||||
4. Provides detailed error reporting and notifications
|
||||
|
||||
Reconnection Strategy:
|
||||
- Error Count Threshold: Disconnects after 5 consecutive errors
|
||||
- Reconnection Window: Enforces minimum intervals between attempts
|
||||
- Automatic Recovery: Attempts reconnection when conditions allow
|
||||
- State Monitoring: Continuously monitors connection health
|
||||
|
||||
Args:
|
||||
None
|
||||
|
||||
Returns:
|
||||
tuple[bool, dict[str, Any]]: Connection validation result
|
||||
- bool: True if connection is healthy, False otherwise
|
||||
- dict: Error information if validation fails
|
||||
"""
|
||||
if self.client is None:
|
||||
return await self.connect()
|
||||
@@ -222,14 +250,32 @@ class OpcRepository():
|
||||
async def write_data(self, node: str, value: Any, data_type: str,
|
||||
logger: Logger, metadata: dict[str, Any]) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Writes data to the OPC server.
|
||||
If the connection is not established, it attempts to reconnect.
|
||||
If the connection is established but the client is not connected,
|
||||
it attempts to reconnect.
|
||||
If the connection is established but the client is connected,
|
||||
it checks if the client is connected to the OPC server.
|
||||
If the client is not connected, it attempts to reconnect.
|
||||
If the client is connected, it returns True.
|
||||
Write data to OPC server with comprehensive validation and monitoring.
|
||||
|
||||
This method provides secure and reliable data writing to OPC servers
|
||||
with automatic connection validation, data type conversion, and
|
||||
comprehensive error handling. It implements performance monitoring
|
||||
and metrics collection for operational visibility.
|
||||
|
||||
Data Writing Process:
|
||||
1. Connection validation and automatic reconnection
|
||||
2. Node validation and error handling
|
||||
3. Data type conversion and validation
|
||||
4. OPC data writing with timestamp
|
||||
5. Performance metrics collection
|
||||
6. Error handling and notification
|
||||
|
||||
Args:
|
||||
node (str): OPC node identifier to write data to
|
||||
value (Any): Data value to write to the OPC node
|
||||
data_type (str): Data type for OPC conversion
|
||||
logger (Logger): Logger instance for operation logging
|
||||
metadata (dict[str, Any]): Context metadata for logging and metrics
|
||||
|
||||
Returns:
|
||||
tuple[bool, dict[str, Any]]: Write operation result
|
||||
- bool: True if write successful, False otherwise
|
||||
- dict: Error information if write failed
|
||||
"""
|
||||
|
||||
is_connected, error = await self.validate_connection()
|
||||
|
||||
Reference in New Issue
Block a user