gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into release/SIENTIAPDE-1645

Browse Source
This commit is contained in:
vitor-aignosi
2026-04-06 08:34:15 -03:00
parent bf2b6b3888 dfe05a063a
commit d2bb588739
18 changed files with 85 additions and 676 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
model_manager/activities/cleanup.py
View File

@@ -1,5 +1,5 @@
"""
Cleanup activities for removing stale files from MinIO and local filesystem.
Cleanup activities for removing stale files from local filesystem.
This module provides activities for cleaning up temporary files and directories
that are older than the configured retention period. It operates independently
@@ -13,7 +13,7 @@ with workflow.unsafe.imports_passed_through():
import re
import shutil
import traceback
from datetime import UTC, datetime, timedelta
from datetime import datetime, timedelta
from typing import Any
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
@@ -27,7 +27,6 @@ with workflow.unsafe.imports_passed_through():
RETENTION_HOURS = int(os.getenv('CLEANUP_RETENTION_HOURS', '24'))
DRY_RUN = os.getenv('CLEANUP_DRY_RUN', 'false').lower() == 'true'
MAX_KEYS_CLEANUP = int(os.getenv('MAX_KEYS_CLEANUP', '1000'))
class Cleanup(SientiaMonitoring):
@@ -35,13 +34,11 @@ class Cleanup(SientiaMonitoring):
Activity for cleaning up stale files and directories.
This activity extends SientiaMonitoring and handles cleanup of:
- MinIO files with timestamp prefixes (timestamp-filename pattern)
- Local temporary directories with timestamp suffixes
"""
def __init__(
self,
minio_repository: MinioRepository,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
@@ -50,145 +47,21 @@ class Cleanup(SientiaMonitoring):
Initialize Cleanup activity.
Args:
minio_repository: Repository for MinIO operations
logger: Logger instance for observability
notification_handler: Handler for sending notifications
metrics_controller: Controller for metrics emission
"""
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
self.minio_repository = minio_repository
# Configuration from environment variables
self.retention_hours = RETENTION_HOURS
self.dry_run = DRY_RUN
# MinIO list operation page size
self.max_keys_cleanup = MAX_KEYS_CLEANUP
# Regex patterns for timestamp extraction
self.minio_timestamp_pattern = re.compile(r'^(\d{13})-(.+)') # timestamp-filename
self.dir_timestamp_pattern = re.compile(
r'^(.+)_(\d{8}_\d{6}_\d{6})$'
) # name_YYYYMMDD_HHMMSS_microseconds
@activity.defn(name='cleanup_minio_files')
async def cleanup_minio_files(self, input_data: dict[str, Any]) -> None:
"""
Clean up stale files from MinIO based on timestamp in filename.
This activity scans a single MinIO bucket for files following the pattern
'{timestamp}-{filename}' where timestamp is milliseconds since epoch.
Files older than the retention period are deleted.
Args:
input_data: Cleanup configuration containing:
- metadata (dict): Workflow execution metadata
- bucket_name (str): Name of the bucket to scan
Returns:
None: Results are logged and tracked via metrics
Raises:
Exception: If cleanup fails (after sending notification)
"""
metadata = input_data.get('metadata', {})
bucket_name = input_data.get('bucket_name')
metrics_status = 'success'
if not bucket_name:
raise ValueError('bucket_name must be provided')
cutoff_time = datetime.now(UTC) - timedelta(hours=self.retention_hours)
cutoff_timestamp_ms = int(cutoff_time.timestamp() * 1000)
try:
self.info(
f'Starting MinIO cleanup - Bucket: {bucket_name}, '
f'Retention: {self.retention_hours}h, Dry run: {self.dry_run}, '
f'Cutoff: {cutoff_time.isoformat()}',
metadata,
)
files_scanned = 0
files_deleted = 0
errors = []
# List objects in the specified bucket. MinioRepository applies BASE_PREFIX
# internally; we request all objects under that prefix for this bucket.
objects = await self.minio_repository.list_objects(
prefix='',
bucket=bucket_name,
recursive=True,
metadata=metadata,
)
for obj_key in objects:
files_scanned += 1
# Extract timestamp from the filename portion of the object key
filename = obj_key.split('/')[-1]
match = self.minio_timestamp_pattern.match(filename)
if not match:
self.debug(f'Skipping file without timestamp pattern: {obj_key}', metadata)
continue
file_timestamp_ms = int(match.group(1))
if file_timestamp_ms < cutoff_timestamp_ms:
if self.dry_run:
self.info(
f'[DRY RUN] Would delete: {obj_key} (age: {(cutoff_time.timestamp() - file_timestamp_ms / 1000) / 3600:.1f}h)',
metadata,
)
files_deleted += 1
else:
try:
await self.minio_repository.delete_file(
object_name=obj_key,
bucket=None,
metadata=metadata,
)
self.info(f'Deleted stale file: {obj_key}', metadata)
files_deleted += 1
except Exception as e: # noqa: BLE001
error_msg = f'Failed to delete {obj_key}: {str(e)}'
errors.append(error_msg)
self.error(error_msg, metadata)
else:
self.debug(
f'Keeping recent file: {obj_key} (age: {(cutoff_time.timestamp() - file_timestamp_ms / 1000) / 3600:.1f}h)',
metadata,
)
self.info(
f'MinIO cleanup completed - Bucket: {bucket_name}, '
f'Scanned: {files_scanned}, Deleted: {files_deleted}, Errors: {len(errors)}',
metadata,
)
except Exception as e:
metrics_status = 'error'
error_msg = f'Error in MinIO cleanup: {str(e)}'
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id='CLEANUP_MINIO_ERROR',
message=error_msg,
block='cleanup_minio_files',
level=NotificationLevel.ERROR,
attachment_content=trace,
)
raise
finally:
await self._emit_metrics(
metadata=metadata,
metrics_status=metrics_status,
activity_name='cleanup_minio_files',
emit_workflow_metric=(metrics_status == 'error'),
)
@activity.defn(name='cleanup_temp_directories')
async def cleanup_temp_directories(self, input_data: dict[str, Any]) -> None:
"""