Merge branch 'main' into release/SIENTIAPDE-1645

This commit is contained in:
vitor-aignosi
2026-04-06 08:34:15 -03:00
18 changed files with 85 additions and 676 deletions

View File

@@ -1,5 +1,5 @@
"""
Cleanup workflow for removing stale files from MinIO and local filesystem.
Cleanup workflow for removing local filesystem.
This module provides a Temporal cron workflow that runs daily to clean up
temporary files and directories older than the configured retention period.
@@ -13,11 +13,9 @@ with workflow.unsafe.imports_passed_through():
from typing import Any
from model_manager.activities.activities import Activities
from model_manager.workflows.train_model import POD_ID, network_retry_policy, no_retry_policy
from model_manager.workflows.train_model import POD_ID, no_retry_policy
TIMEOUT_CLEANUP_MINIO = int(os.getenv('TIMEOUT_CLEANUP_MINIO', '300'))
TIMEOUT_CLEANUP_LOCAL = int(os.getenv('TIMEOUT_CLEANUP_LOCAL', '120'))
DEFAULT_CLEANUP_BUCKET = os.getenv('DEFAULT_CLEANUP_BUCKET', 'model-training')
@workflow.defn(name='cleanup_files')
@@ -26,7 +24,6 @@ class CleanupFiles:
Cleanup workflow for removing stale files.
This workflow cleans up:
- MinIO files with timestamp prefixes
- Local temporary directories with timestamp suffixes
The workflow is designed to be simple and robust, with error handling
@@ -38,17 +35,10 @@ class CleanupFiles:
"""
Execute the cleanup workflow.
This method orchestrates the cleanup of MinIO files and local directories
This method orchestrates the cleanup of local directories
in sequence. No exception handling is needed as activities handle their
own errors and notifications.
Args:
input_data: Workflow configuration containing optional:
- bucket_name (str): Bucket to clean (defaults to environment variable)
"""
# Get bucket name from input or environment
bucket_name = input_data.get('bucket_name', DEFAULT_CLEANUP_BUCKET)
# Default temp path for local cleanup
temp_path = 'model_manager/reports/temp'
@@ -60,17 +50,6 @@ class CleanupFiles:
}
}
# Execute MinIO cleanup
await workflow.execute_activity_method(
Activities.cleanup_minio_files,
{
**metadata,
'bucket_name': bucket_name,
},
retry_policy=network_retry_policy,
start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_MINIO),
)
# Execute local directory cleanup
await workflow.execute_activity_method(
Activities.cleanup_temp_directories,

View File

@@ -20,7 +20,6 @@ with workflow.unsafe.imports_passed_through():
from model_manager.activities.activities import Activities
from model_manager.activities.experiment_tracking import UpdateType
from model_manager.utils.exceptions import ModelTrainingError
from model_manager.utils.models.experiment_status import ExperimentStatus
from model_manager.utils.models.train_model_params import TrainModelParams
@@ -273,7 +272,7 @@ class TrainModel:
metadata=metadata,
experiment_run_id=experiment_run_id,
update_type=UpdateType.MODEL_SAVED,
status=ExperimentStatus.TRACKING_SENT,
status=ExperimentStatus.TRAINING_SUCCESS,
run_name=train_result.get('run_name'),
)
@@ -310,7 +309,7 @@ class TrainModel:
metadata: dict[str, Any],
) -> None:
"""
Cleanup resources and delete file from MinIO.
Cleanup resources.
This method deletes the training file from MinIO. On success, updates
DB status to FILE_DELETED.
@@ -319,9 +318,6 @@ class TrainModel:
Args:
experiment_run_id: Validated experiment run ID
metadata: Workflow execution metadata
Raises:
Exception: If cleanup fails (after updating DB status)
"""
try:
await workflow.execute_activity_method(