SIENTIAPDE-1350: Implement file cleanup workflow and activities, including MinIO and local directory cleanup, configuration, and metrics.

.env.example

@@ -1,44 +1,57 @@
-POSTGRES_HOST="paradedb-rw.paradedb.svc.cluster.local"
-POSTGRES_PORT="5432"
-POSTGRES_USER="sientia"
-POSTGRES_PASSWORD="password"
-POSTGRES_DBNAME="sientia"
-POSTGRES_MIN_CONNECTIONS="10"
-POSTGRES_MAX_CONNECTIONS="30"
+POSTGRES_HOST=paradedb-rw.paradedb.svc.cluster.local
+POSTGRES_PORT=5432
+POSTGRES_USER=sientia
+POSTGRES_PASSWORD=password
+POSTGRES_DBNAME=sientia
+POSTGRES_MIN_CONNECTIONS=10
+POSTGRES_MAX_CONNECTIONS=30
 
-MLFLOW_URL="http://sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local:80"
-MLFLOW_USERNAME="aignosi"
-MLFLOW_PASSWORD="mlflow_password"
+MLFLOW_URL=http://sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local:80
+MLFLOW_USERNAME=aignosi
+MLFLOW_PASSWORD=mlflow_password
 
-LOG_LEVEL="DEBUG"
-HTTP_METRICS_PORT="9090"
-HTTP_SDK_METRICS_PORT="9091"
-PROJECT_NAME="sientia-model-manager"
+LOG_LEVEL=DEBUG
+HTTP_METRICS_PORT=9090
+HTTP_SDK_METRICS_PORT=9091
+PROJECT_NAME=sientia-model-manager
 
-TEMPORAL_HOST="temporal-frontend.temporal.svc.cluster.local:7233"
-TEMPORAL_NAMESPACE="model-manager"
+TEMPORAL_HOST=temporal-frontend.temporal.svc.cluster.local:7233
+TEMPORAL_NAMESPACE=model-manager
 
-MONGODB_USERNAME="mongo_user"
-MONGODB_PASSWORD="mongo_db_password"
-MONGODB_URL="my-release-mongodb.mongodb.svc.cluster.local:27017"
-MONGODB_DATABASE="sientia"
-MONGODB_TTL_INDEX_HOURS="1"
+MONGODB_USERNAME=mongo_user
+MONGODB_PASSWORD=mongo_db_password
+MONGODB_URL=my-release-mongodb.mongodb.svc.cluster.local:27017
+MONGODB_DATABASE=sientia
+MONGODB_TTL_INDEX_HOURS=1
 
-MINIO_ENDPOINT_URL="http://minio.minio.svc.cluster.local:9000"
-MINIO_ACCESS_KEY="minioadmin"
-MINIO_SECRET_KEY="minioadmin"
-MINIO_REGION="us-east-1"
-MINIO_USE_SSL="false"
-MINIO_MAX_RETRY_ATTEMPTS="3"
-MINIO_RETRY_MODE="adaptive"
-MINIO_CONNECT_TIMEOUT="10"
-MINIO_READ_TIMEOUT="60"
+MINIO_ENDPOINT_URL=http://minio.minio.svc.cluster.local:9000
+MINIO_ACCESS_KEY=minioadmin
+MINIO_SECRET_KEY=minioadmin
+MINIO_REGION=us-east-1
+MINIO_USE_SSL=false
+MINIO_MAX_RETRY_ATTEMPTS=3
+MINIO_RETRY_MODE=adaptive
+MINIO_CONNECT_TIMEOUT=10
+MINIO_READ_TIMEOUT=60
 
 # Workflow Activity Timeouts (in seconds)
 # These timeouts are designed to handle large files (up to 200MB)
-TIMEOUT_VALIDATE_PARAMS="30"           # Parameter validation (fast operation)
-TIMEOUT_TRAIN_MODEL="2700"             # Model training (30 min for large datasets)
-TIMEOUT_DELETE_FILE="120"               # Delete file from MinIO (1 min)
-TIMEOUT_UPDATE_DATABASE="30"           # Database update operations (30 sec)
+TIMEOUT_VALIDATE_PARAMS=30 # Parameter validation (fast operation)
+TIMEOUT_TRAIN_MODEL=2700 # Model training (30 min for large datasets)
+TIMEOUT_DELETE_FILE=120 # Delete file from MinIO (1 min)
+TIMEOUT_UPDATE_DATABASE=30 # Database update operations (30 sec)
 
-EXTRA_PIP_REQUIREMENTS="git+https://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@github.com/Aignosi/sientia-mlops-library.git"
+# Cleanup Configuration
+# Cleanup retention period in hours (files older than this will be deleted)
+CLEANUP_RETENTION_HOURS=24 # Default: 24 hours
+# Enable dry run mode to test without actually deleting files
+CLEANUP_DRY_RUN=false # Set to true for testing without deletion
+# Cleanup operation timeouts
+TIMEOUT_CLEANUP_MINIO=300 # MinIO cleanup timeout (5 minutes)
+TIMEOUT_CLEANUP_LOCAL=120 # Local directory cleanup timeout (2 minutes)
+# MinIO list operation page size for cleanup
+MAX_KEYS_CLEANUP=1000 # Maximum keys per page when listing objects
+# Default bucket for cleanup operations
+DEFAULT_CLEANUP_BUCKET=model-training # Default bucket to clean
+
+EXTRA_PIP_REQUIREMENTS=git+https://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@github.com/Aignosi/sientia-mlops-library.git