Merge pull request #15 from Aignosi/feature/SIENTIAPDE-1350
SIENTIAPDE-1350: Implement Automated File Cleanup Workflow with Temporal Schedules
This commit is contained in:
Bruno Domingues
290
.dockerignore
290
.dockerignore
@@ -1,225 +1,93 @@
|
|||||||
# Git
|
# ============================================================================
|
||||||
.git
|
# WHITELIST APPROACH: Block everything by default, then allow only what's needed
|
||||||
.gitignore
|
# ============================================================================
|
||||||
.gitattributes
|
|
||||||
|
|
||||||
# Documentation
|
# Block everything first
|
||||||
*.md
|
*
|
||||||
docs/
|
|
||||||
README*
|
|
||||||
|
|
||||||
# Tests and development
|
# ============================================================================
|
||||||
tests/
|
# ALLOW: Application source code (model_manager package)
|
||||||
.pytest_cache/
|
# ============================================================================
|
||||||
.coverage
|
|
||||||
htmlcov/
|
# Allow the main package directory and all Python files
|
||||||
.tox/
|
!model_manager/
|
||||||
.nox/
|
!model_manager/**/*.py
|
||||||
.mypy_cache/
|
!model_manager/**/__init__.py
|
||||||
.pyre/
|
|
||||||
coverage.xml
|
# Allow subdirectories structure
|
||||||
*.cover
|
!model_manager/activities/
|
||||||
.hypothesis/
|
!model_manager/activities/**
|
||||||
|
!model_manager/schedules/
|
||||||
|
!model_manager/schedules/**
|
||||||
|
!model_manager/sientia/
|
||||||
|
!model_manager/sientia/**
|
||||||
|
!model_manager/utils/
|
||||||
|
!model_manager/utils/**
|
||||||
|
!model_manager/utils/models/
|
||||||
|
!model_manager/utils/models/**
|
||||||
|
!model_manager/utils/repository/
|
||||||
|
!model_manager/utils/repository/**
|
||||||
|
!model_manager/worker/
|
||||||
|
!model_manager/worker/**
|
||||||
|
!model_manager/workflows/
|
||||||
|
!model_manager/workflows/**
|
||||||
|
|
||||||
|
# Allow reports directory with header.html
|
||||||
|
!model_manager/reports/
|
||||||
|
!model_manager/reports/header.html
|
||||||
|
|
||||||
|
# Allow temp directory structure (but not its contents)
|
||||||
|
!model_manager/reports/temp/
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# ALLOW: Dependencies file (needed for pip install in Dockerfile)
|
||||||
|
# ============================================================================
|
||||||
|
!requirements.txt
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# BLOCK: Explicitly block unwanted files even if they match above patterns
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
# Python cache and compiled files
|
# Python cache and compiled files
|
||||||
__pycache__/
|
**/__pycache__/
|
||||||
*.py[cod]
|
**/*.pyc
|
||||||
*$py.class
|
**/*.pyo
|
||||||
*.so
|
**/*.pyd
|
||||||
.Python
|
**/.Python
|
||||||
build/
|
**/*.so
|
||||||
develop-eggs/
|
**/*.egg
|
||||||
dist/
|
**/*.egg-info/
|
||||||
downloads/
|
|
||||||
eggs/
|
|
||||||
.eggs/
|
|
||||||
lib/
|
|
||||||
lib64/
|
|
||||||
parts/
|
|
||||||
sdist/
|
|
||||||
var/
|
|
||||||
wheels/
|
|
||||||
*.egg-info/
|
|
||||||
.installed.cfg
|
|
||||||
*.egg
|
|
||||||
MANIFEST
|
|
||||||
|
|
||||||
# Virtual environments
|
# Tests (not needed in production)
|
||||||
venv/
|
model_manager/**/test_*.py
|
||||||
env/
|
model_manager/**/*_test.py
|
||||||
ENV/
|
|
||||||
.venv/
|
|
||||||
.env/
|
|
||||||
|
|
||||||
# IDE and editors
|
# IDE and editor files
|
||||||
.vscode/
|
**/.vscode/
|
||||||
.idea/
|
**/.idea/
|
||||||
*.swp
|
**/*.swp
|
||||||
*.swo
|
**/*.swo
|
||||||
*~
|
**/*~
|
||||||
.DS_Store
|
|
||||||
Thumbs.db
|
|
||||||
|
|
||||||
# OS files
|
# OS files
|
||||||
.dockerignore
|
**/.DS_Store
|
||||||
.dockerignore.dockerignore
|
**/Thumbs.db
|
||||||
|
|
||||||
# CI/CD
|
# Logs and temporary files
|
||||||
.github/
|
**/*.log
|
||||||
.gitlab-ci.yml
|
**/*.tmp
|
||||||
.travis.yml
|
**/*.temp
|
||||||
.circleci/
|
|
||||||
Jenkinsfile
|
|
||||||
|
|
||||||
# Local configuration
|
# Local configuration
|
||||||
.env
|
**/.env
|
||||||
.env.local
|
**/.env.local
|
||||||
.env.*.local
|
**/*.local
|
||||||
config/local/
|
|
||||||
*.local
|
|
||||||
|
|
||||||
# Logs
|
# Documentation inside code
|
||||||
*.log
|
**/*.md
|
||||||
logs/
|
**/README*
|
||||||
npm-debug.log*
|
|
||||||
yarn-debug.log*
|
|
||||||
yarn-error.log*
|
|
||||||
|
|
||||||
# Runtime data
|
|
||||||
pids/
|
|
||||||
*.pid
|
|
||||||
*.seed
|
|
||||||
*.pid.lock
|
|
||||||
|
|
||||||
# Temporary files
|
|
||||||
tmp/
|
|
||||||
temp/
|
|
||||||
*.tmp
|
|
||||||
*.temp
|
|
||||||
|
|
||||||
# Node.js (if any frontend tools)
|
|
||||||
node_modules/
|
|
||||||
npm-debug.log*
|
|
||||||
|
|
||||||
# Database
|
|
||||||
*.db
|
|
||||||
*.sqlite
|
|
||||||
*.sqlite3
|
|
||||||
|
|
||||||
# Jupyter Notebook
|
|
||||||
.ipynb_checkpoints
|
|
||||||
|
|
||||||
# pyenv
|
|
||||||
.python-version
|
|
||||||
|
|
||||||
# pipenv
|
|
||||||
Pipfile.lock
|
|
||||||
|
|
||||||
# PEP 582
|
|
||||||
__pypackages__/
|
|
||||||
|
|
||||||
# Celery
|
|
||||||
celerybeat-schedule
|
|
||||||
celerybeat.pid
|
|
||||||
|
|
||||||
# SageMath parsed files
|
|
||||||
*.sage.py
|
|
||||||
|
|
||||||
# Spyder project settings
|
|
||||||
.spyderproject
|
|
||||||
.spyproject
|
|
||||||
|
|
||||||
# Rope project settings
|
|
||||||
.ropeproject
|
|
||||||
|
|
||||||
# mkdocs documentation
|
|
||||||
/site
|
|
||||||
|
|
||||||
# Backup files
|
# Backup files
|
||||||
*.bak
|
**/*.bak
|
||||||
*.backup
|
**/*.backup
|
||||||
*.old
|
**/*.old
|
||||||
|
|
||||||
# Local development scripts
|
|
||||||
scripts/local/
|
|
||||||
dev-*
|
|
||||||
|
|
||||||
# Docker files (excluding the main ones)
|
|
||||||
docker-compose*.yml
|
|
||||||
docker-compose*.yaml
|
|
||||||
Dockerfile.*
|
|
||||||
!Dockerfile
|
|
||||||
|
|
||||||
# Helm charts (already in .gitignore but reinforcing)
|
|
||||||
charts/
|
|
||||||
|
|
||||||
# Kubernetes manifests
|
|
||||||
k8s/
|
|
||||||
kube-*
|
|
||||||
|
|
||||||
# Terraform
|
|
||||||
*.tfstate
|
|
||||||
*.tfstate.*
|
|
||||||
.terraform/
|
|
||||||
|
|
||||||
# Monitoring and profiling
|
|
||||||
*.prof
|
|
||||||
*.profile
|
|
||||||
.perf
|
|
||||||
|
|
||||||
# Security
|
|
||||||
*.pem
|
|
||||||
*.key
|
|
||||||
*.crt
|
|
||||||
*.p12
|
|
||||||
secrets/
|
|
||||||
*.secret
|
|
||||||
|
|
||||||
# Large binaries and datasets
|
|
||||||
*.bin
|
|
||||||
*.pkl
|
|
||||||
*.pickle
|
|
||||||
*.joblib
|
|
||||||
data/
|
|
||||||
datasets/
|
|
||||||
models/pre-trained/
|
|
||||||
|
|
||||||
# Build artifacts
|
|
||||||
build/
|
|
||||||
dist/
|
|
||||||
target/
|
|
||||||
out/
|
|
||||||
|
|
||||||
# Package manager lock files (keeping requirements.txt)
|
|
||||||
package-lock.json
|
|
||||||
yarn.lock
|
|
||||||
Pipfile.lock
|
|
||||||
|
|
||||||
# Local tools
|
|
||||||
tools/local/
|
|
||||||
bin/local/
|
|
||||||
|
|
||||||
# Cache directories
|
|
||||||
.cache/
|
|
||||||
cache/
|
|
||||||
|
|
||||||
# Development and configuration files
|
|
||||||
.env.example
|
|
||||||
requirements-dev.txt
|
|
||||||
pyproject.toml
|
|
||||||
sonar-project.properties
|
|
||||||
todo-list.txt
|
|
||||||
validate.sh
|
|
||||||
run_local.sh
|
|
||||||
LICENSE
|
|
||||||
|
|
||||||
# Helm charts (development only)
|
|
||||||
sientia-module/
|
|
||||||
*.yaml
|
|
||||||
|
|
||||||
# Local directories
|
|
||||||
data/
|
|
||||||
logs/
|
|
||||||
models/
|
|
||||||
temp/
|
|
||||||
scripts/
|
|
||||||
|
|||||||
87
.env.example
87
.env.example
@@ -1,44 +1,57 @@
|
|||||||
POSTGRES_HOST="paradedb-rw.paradedb.svc.cluster.local"
|
POSTGRES_HOST=paradedb-rw.paradedb.svc.cluster.local
|
||||||
POSTGRES_PORT="5432"
|
POSTGRES_PORT=5432
|
||||||
POSTGRES_USER="sientia"
|
POSTGRES_USER=sientia
|
||||||
POSTGRES_PASSWORD="password"
|
POSTGRES_PASSWORD=password
|
||||||
POSTGRES_DBNAME="sientia"
|
POSTGRES_DBNAME=sientia
|
||||||
POSTGRES_MIN_CONNECTIONS="10"
|
POSTGRES_MIN_CONNECTIONS=10
|
||||||
POSTGRES_MAX_CONNECTIONS="30"
|
POSTGRES_MAX_CONNECTIONS=30
|
||||||
|
|
||||||
MLFLOW_URL="http://sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local:80"
|
MLFLOW_URL=http://sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local:80
|
||||||
MLFLOW_USERNAME="aignosi"
|
MLFLOW_USERNAME=aignosi
|
||||||
MLFLOW_PASSWORD="mlflow_password"
|
MLFLOW_PASSWORD=mlflow_password
|
||||||
|
|
||||||
LOG_LEVEL="DEBUG"
|
LOG_LEVEL=DEBUG
|
||||||
HTTP_METRICS_PORT="9090"
|
HTTP_METRICS_PORT=9090
|
||||||
HTTP_SDK_METRICS_PORT="9091"
|
HTTP_SDK_METRICS_PORT=9091
|
||||||
PROJECT_NAME="sientia-model-manager"
|
PROJECT_NAME=sientia-model-manager
|
||||||
|
|
||||||
TEMPORAL_HOST="temporal-frontend.temporal.svc.cluster.local:7233"
|
TEMPORAL_HOST=temporal-frontend.temporal.svc.cluster.local:7233
|
||||||
TEMPORAL_NAMESPACE="model-manager"
|
TEMPORAL_NAMESPACE=model-manager
|
||||||
|
TRAIN_TASK_QUEUE=train_model-queue
|
||||||
|
CLEANUP_TASK_QUEUE=cleanup-queue
|
||||||
|
TEMPORAL_USE_TLS=false
|
||||||
|
|
||||||
MONGODB_USERNAME="mongo_user"
|
MONGODB_USERNAME=mongo_user
|
||||||
MONGODB_PASSWORD="mongo_db_password"
|
MONGODB_PASSWORD=mongo_db_password
|
||||||
MONGODB_URL="my-release-mongodb.mongodb.svc.cluster.local:27017"
|
MONGODB_URL=my-release-mongodb.mongodb.svc.cluster.local:27017
|
||||||
MONGODB_DATABASE="sientia"
|
MONGODB_DATABASE=sientia
|
||||||
MONGODB_TTL_INDEX_HOURS="1"
|
MONGODB_TTL_INDEX_HOURS=1
|
||||||
|
|
||||||
MINIO_ENDPOINT_URL="http://minio.minio.svc.cluster.local:9000"
|
MINIO_ENDPOINT_URL=http://minio.minio.svc.cluster.local:9000
|
||||||
MINIO_ACCESS_KEY="minioadmin"
|
MINIO_ACCESS_KEY=minioadmin
|
||||||
MINIO_SECRET_KEY="minioadmin"
|
MINIO_SECRET_KEY=minioadmin
|
||||||
MINIO_REGION="us-east-1"
|
MINIO_REGION=us-east-1
|
||||||
MINIO_USE_SSL="false"
|
MINIO_USE_SSL=false
|
||||||
MINIO_MAX_RETRY_ATTEMPTS="3"
|
MINIO_MAX_RETRY_ATTEMPTS=3
|
||||||
MINIO_RETRY_MODE="adaptive"
|
MINIO_RETRY_MODE=adaptive
|
||||||
MINIO_CONNECT_TIMEOUT="10"
|
MINIO_CONNECT_TIMEOUT=10
|
||||||
MINIO_READ_TIMEOUT="60"
|
MINIO_READ_TIMEOUT=60
|
||||||
|
|
||||||
# Workflow Activity Timeouts (in seconds)
|
TIMEOUT_VALIDATE_PARAMS=30
|
||||||
# These timeouts are designed to handle large files (up to 200MB)
|
TIMEOUT_TRAIN_MODEL=2700
|
||||||
TIMEOUT_VALIDATE_PARAMS="30" # Parameter validation (fast operation)
|
TIMEOUT_DELETE_FILE=120
|
||||||
TIMEOUT_TRAIN_MODEL="2700" # Model training (30 min for large datasets)
|
TIMEOUT_UPDATE_DATABASE=30
|
||||||
TIMEOUT_DELETE_FILE="120" # Delete file from MinIO (1 min)
|
|
||||||
TIMEOUT_UPDATE_DATABASE="30" # Database update operations (30 sec)
|
|
||||||
|
|
||||||
EXTRA_PIP_REQUIREMENTS="git+https://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@github.com/Aignosi/sientia-mlops-library.git"
|
CLEANUP_RETENTION_HOURS=24
|
||||||
|
CLEANUP_DRY_RUN=false
|
||||||
|
TIMEOUT_CLEANUP_MINIO=300
|
||||||
|
TIMEOUT_CLEANUP_LOCAL=120
|
||||||
|
MAX_KEYS_CLEANUP=1000
|
||||||
|
DEFAULT_CLEANUP_BUCKET=model-training
|
||||||
|
|
||||||
|
CLEANUP_SCHEDULE_ID=cleanup-files-daily
|
||||||
|
CLEANUP_CRON="0 0 * * *"
|
||||||
|
CLEANUP_TIMEZONE=UTC
|
||||||
|
CLEANUP_EXECUTION_TIMEOUT_HOURS=1
|
||||||
|
|
||||||
|
EXTRA_PIP_REQUIREMENTS=git+https://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@github.com/Aignosi/sientia-mlops-library.git
|
||||||
|
|||||||
175
README.md
175
README.md
@@ -15,6 +15,7 @@ An enterprise-grade ML model training orchestration platform built on Temporal.
|
|||||||
- [Security Architecture](#security-architecture)
|
- [Security Architecture](#security-architecture)
|
||||||
- [Workflows](#workflows)
|
- [Workflows](#workflows)
|
||||||
- [Train Model Workflow](#train-model-workflow-train_modelpy)
|
- [Train Model Workflow](#train-model-workflow-train_modelpy)
|
||||||
|
- [Cleanup Files Workflow](#cleanup-files-workflow-cleanup_filespy)
|
||||||
- [Installation & Setup](#installation--setup)
|
- [Installation & Setup](#installation--setup)
|
||||||
- [Prerequisites](#prerequisites)
|
- [Prerequisites](#prerequisites)
|
||||||
- [Environment Setup](#environment-setup)
|
- [Environment Setup](#environment-setup)
|
||||||
@@ -66,11 +67,13 @@ An enterprise-grade ML model training orchestration platform built on Temporal.
|
|||||||
|
|
||||||
### Core Functionality
|
### Core Functionality
|
||||||
- **ML Model Training Pipeline**: Complete training workflow from validation to deployment using MLFlow
|
- **ML Model Training Pipeline**: Complete training workflow from validation to deployment using MLFlow
|
||||||
|
- **Automated File Cleanup**: Scheduled cleanup of stale files from MinIO and local filesystem
|
||||||
- **Temporal Workflow Orchestration**: Robust workflow management with granular retry policies and fault tolerance
|
- **Temporal Workflow Orchestration**: Robust workflow management with granular retry policies and fault tolerance
|
||||||
- **Parameter Validation**: Defense-in-depth validation with business rules and type checking
|
- **Parameter Validation**: Defense-in-depth validation with business rules and type checking
|
||||||
- **Experiment Tracking**: Comprehensive status tracking in PostgreSQL database
|
- **Experiment Tracking**: Comprehensive status tracking in PostgreSQL database
|
||||||
- **Resource Management**: Automatic cleanup of temporary files and storage
|
- **Resource Management**: Automatic cleanup of temporary files and storage
|
||||||
- **Comprehensive Monitoring**: Prometheus metrics and detailed logging for operational visibility
|
- **Comprehensive Monitoring**: Prometheus metrics and detailed logging for operational visibility
|
||||||
|
- **Scheduled Jobs**: Automated daily cleanup with configurable cron schedules
|
||||||
|
|
||||||
### Advanced Capabilities
|
### Advanced Capabilities
|
||||||
- **Granular Retry Policies**: Different strategies for network, training, MLFlow, database, and filesystem operations
|
- **Granular Retry Policies**: Different strategies for network, training, MLFlow, database, and filesystem operations
|
||||||
@@ -133,38 +136,49 @@ The Model Manager system uses a Temporal-based workflow architecture with clear
|
|||||||
- Health check endpoints for Kubernetes liveness/readiness probes
|
- Health check endpoints for Kubernetes liveness/readiness probes
|
||||||
- Graceful shutdown with cleanup procedures
|
- Graceful shutdown with cleanup procedures
|
||||||
- Multi-instance deployment support
|
- Multi-instance deployment support
|
||||||
- Dedicated task queue: `train_model-queue` for ML model training workflows
|
- Two dedicated task queues:
|
||||||
|
- `train_model-queue`: ML model training workflows
|
||||||
|
- `cleanup-queue`: File cleanup workflows
|
||||||
|
- Automated cleanup schedule management
|
||||||
|
|
||||||
#### **Workflows (`model_manager/workflows/`)**
|
#### **Workflows (`model_manager/workflows/`)**
|
||||||
- **TrainModel**: Complete ML model training pipeline from validation to deployment
|
- **TrainModel**: Complete ML model training pipeline from validation to deployment
|
||||||
|
- **CleanupFiles**: Automated cleanup of stale files from MinIO and local filesystem
|
||||||
- **Key Features**:
|
- **Key Features**:
|
||||||
- Temporal workflow definitions with granular retry policies
|
- Temporal workflow definitions with granular retry policies
|
||||||
- Parameter validation with business rules
|
- Parameter validation with business rules
|
||||||
- Comprehensive error handling and status tracking
|
- Comprehensive error handling and status tracking
|
||||||
- Configurable timeouts for different operation types
|
- Configurable timeouts for different operation types
|
||||||
- Automatic resource cleanup and management
|
- Automatic resource cleanup and management
|
||||||
|
- Scheduled cleanup jobs with cron expressions
|
||||||
|
|
||||||
#### **Activities (`model_manager/activities/`)**
|
#### **Activities (`model_manager/activities/`)**
|
||||||
- **Activities**: Main activity orchestrator combining all functionality through multiple inheritance
|
- **Activities**: Main activity orchestrator combining all functionality through multiple inheritance
|
||||||
- **ExperimentTracking**: ML experiment lifecycle tracking and database operations (extends Postgres)
|
- **ExperimentTracking**: ML experiment lifecycle tracking and database operations
|
||||||
- Unified `update_experiment_run()` method for all experiment status updates
|
- Unified `update_experiment_run()` method for all experiment status updates
|
||||||
- Support for three update types: STATUS, STATUS_WITH_ERROR, MODEL_SAVED
|
- Support for three update types: STATUS, STATUS_WITH_ERROR, MODEL_SAVED
|
||||||
- Automatic error message truncation (1024 chars)
|
- Automatic error message truncation (1024 chars)
|
||||||
- Connection pooling and retry logic via Postgres base class
|
- Connection pooling and retry logic
|
||||||
- **Training**: ML model training operations (standalone activity, composition pattern)
|
- **Training**: ML model training operations with MLFlow and MinIO integration
|
||||||
- Unified `train_model()` method for complete training pipeline
|
- Unified `train_model()` method for complete training pipeline
|
||||||
- Receives pre-downloaded files (BytesIO) to avoid memory leaks
|
- Receives pre-downloaded files (BytesIO) to avoid memory leaks
|
||||||
- Returns success/failure status with TrainModelResult or error message
|
- Returns success/failure status with TrainModelResult or error message
|
||||||
- No exception raising on failure - allows workflow to handle errors gracefully
|
- No exception raising on failure - allows workflow to handle errors gracefully
|
||||||
- Integration with TrainingRepository for business logic separation
|
- Integration with TrainingRepository for business logic separation
|
||||||
- **MLFlow**: Model saving and artifact management operations
|
- MLFlow model saving and artifact management
|
||||||
- **MinIO**: Object storage operations for training data management
|
- MinIO object storage operations
|
||||||
|
- **Cleanup**: File and directory cleanup operations
|
||||||
|
- `cleanup_minio_files()`: Removes stale files from MinIO based on timestamp prefixes
|
||||||
|
- `cleanup_temp_directories()`: Cleans local temporary directories
|
||||||
|
- Configurable retention period (default: 24 hours)
|
||||||
|
- Dry-run mode for testing
|
||||||
- **Key Features**:
|
- **Key Features**:
|
||||||
- Multiple inheritance pattern for unified activity interface
|
- Multiple inheritance pattern for unified activity interface
|
||||||
- Parameter validation with business rules
|
- Parameter validation with business rules
|
||||||
- MLFlow integration for model persistence
|
- MLFlow integration for model persistence
|
||||||
- Comprehensive error handling and notification integration
|
- Comprehensive error handling and notification integration
|
||||||
- Experiment tracking with automatic status management
|
- Experiment tracking with automatic status management
|
||||||
|
- Timestamp-based file cleanup with regex pattern matching
|
||||||
|
|
||||||
#### **Data Services (`model_manager/utils/`)**
|
#### **Data Services (`model_manager/utils/`)**
|
||||||
- **Connectors Config**: Environment variable-based configuration management
|
- **Connectors Config**: Environment variable-based configuration management
|
||||||
@@ -311,6 +325,79 @@ The workflow validates 10 business rules beyond type checking:
|
|||||||
5. **target_variable**: Must be in variable_columns
|
5. **target_variable**: Must be in variable_columns
|
||||||
6. **bucket_name, file_name, experiment_name**: Cannot be empty or whitespace
|
6. **bucket_name, file_name, experiment_name**: Cannot be empty or whitespace
|
||||||
|
|
||||||
|
### Cleanup Files Workflow (`cleanup_files.py`)
|
||||||
|
|
||||||
|
The **CleanupFiles** workflow provides automated cleanup of stale files from MinIO storage and local temporary directories. It runs on a scheduled basis (default: daily at midnight UTC) to maintain storage hygiene.
|
||||||
|
|
||||||
|
#### Purpose
|
||||||
|
- **Storage Management**: Automatic removal of old files from MinIO and local filesystem
|
||||||
|
- **Retention Policy**: Configurable retention period (default: 24 hours)
|
||||||
|
- **Scheduled Execution**: Cron-based scheduling for automated cleanup
|
||||||
|
- **Resource Optimization**: Prevents storage bloat and reduces costs
|
||||||
|
|
||||||
|
#### Execution Flow
|
||||||
|
1. **Cleanup MinIO Files**: Scan and delete files older than retention period from MinIO bucket
|
||||||
|
2. **Cleanup Local Directories**: Remove temporary directories older than retention period
|
||||||
|
|
||||||
|
#### Key Features
|
||||||
|
- **Timestamp-Based Cleanup**: Uses filename/directory timestamps for age determination
|
||||||
|
- **Pattern Matching**: Regex patterns for MinIO (`timestamp-filename`) and directories (`name_YYYYMMDD_HHMMSS_microseconds`)
|
||||||
|
- **Configurable Retention**: Environment variable-based retention period
|
||||||
|
- **Dry-Run Mode**: Test cleanup operations without actual deletion
|
||||||
|
- **Idempotent**: Safe to run multiple times
|
||||||
|
- **Error Handling**: Continues cleanup even if individual operations fail
|
||||||
|
|
||||||
|
#### Input Parameters
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"bucket_name": "model-training" // Optional, defaults to DEFAULT_CLEANUP_BUCKET env var
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Schedule Configuration
|
||||||
|
|
||||||
|
The cleanup schedule is automatically created when the worker starts:
|
||||||
|
|
||||||
|
| Configuration | Environment Variable | Default | Description |
|
||||||
|
|--------------|---------------------|---------|-------------|
|
||||||
|
| **Schedule ID** | `CLEANUP_SCHEDULE_ID` | `cleanup-files-daily` | Unique identifier for the schedule |
|
||||||
|
| **Cron Expression** | `CLEANUP_CRON` | `0 0 * * *` | Daily at midnight UTC |
|
||||||
|
| **Timezone** | `CLEANUP_TIMEZONE` | `UTC` | Timezone for cron execution |
|
||||||
|
| **Task Queue** | `CLEANUP_TASK_QUEUE` | `cleanup-queue` | Dedicated task queue |
|
||||||
|
| **Execution Timeout** | `CLEANUP_EXECUTION_TIMEOUT_HOURS` | `1` | Maximum execution time (hours) |
|
||||||
|
| **Retention Period** | `CLEANUP_RETENTION_HOURS` | `24` | Files older than this are deleted |
|
||||||
|
| **Dry Run** | `CLEANUP_DRY_RUN` | `false` | Test mode without actual deletion |
|
||||||
|
| **Max Keys** | `MAX_KEYS_CLEANUP` | `1000` | MinIO list operation page size |
|
||||||
|
|
||||||
|
#### Architecture Diagram
|
||||||
|
```mermaid
|
||||||
|
flowchart TD
|
||||||
|
A[Scheduled Trigger] --> B[cleanup_minio_files]
|
||||||
|
B --> C[cleanup_temp_directories]
|
||||||
|
|
||||||
|
B -.-> MinIO[MinIO Storage]
|
||||||
|
C -.-> FS[Local Filesystem]
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Retry Strategies
|
||||||
|
|
||||||
|
| Operation Type | Initial Interval | Max Interval | Backoff | Max Attempts | Use Case |
|
||||||
|
|---------------|------------------|--------------|---------|--------------|----------|
|
||||||
|
| **Network** | 1s | 10s | 2.0x | 5 | MinIO operations (transient network errors) |
|
||||||
|
| **No Retry** | - | - | - | 1 | Local filesystem operations (permanent errors) |
|
||||||
|
|
||||||
|
#### Cleanup Patterns
|
||||||
|
|
||||||
|
**MinIO Files:**
|
||||||
|
- Pattern: `{timestamp}-{filename}` where timestamp is milliseconds since epoch
|
||||||
|
- Example: `1638360000000-training_data.csv`
|
||||||
|
- Retention: Files older than `CLEANUP_RETENTION_HOURS` are deleted
|
||||||
|
|
||||||
|
**Local Directories:**
|
||||||
|
- Pattern: `{name}_{YYYYMMDD}_{HHMMSS}_{microseconds}`
|
||||||
|
- Example: `temp_20231201_143052_123456`
|
||||||
|
- Retention: Directories older than `CLEANUP_RETENTION_HOURS` are deleted
|
||||||
|
|
||||||
## Installation & Setup
|
## Installation & Setup
|
||||||
|
|
||||||
### Prerequisites
|
### Prerequisites
|
||||||
@@ -773,11 +860,23 @@ The Model Manager system exposes comprehensive Prometheus metrics for operationa
|
|||||||
- `app_up`: Application health status (1=healthy, 0=unhealthy)
|
- `app_up`: Application health status (1=healthy, 0=unhealthy)
|
||||||
- Labels: `pod_id`
|
- Labels: `pod_id`
|
||||||
|
|
||||||
|
### Workflow Execution Metrics
|
||||||
|
- `workflow_execution_total`: Total workflow executions
|
||||||
|
- Labels: `workflow_name`, `status` (success/failure)
|
||||||
|
- `activity_execution_total`: Total activity executions
|
||||||
|
- Labels: `activity_name`, `status` (success/failure)
|
||||||
|
|
||||||
### Training Metrics
|
### Training Metrics
|
||||||
- Training success/failure rates through notification system
|
- Training success/failure rates through notification system
|
||||||
- Model save performance metrics
|
- Model save performance metrics
|
||||||
- Experiment status tracking
|
- Experiment status tracking
|
||||||
|
|
||||||
|
### Cleanup Metrics
|
||||||
|
- Cleanup execution success/failure rates
|
||||||
|
- Number of files deleted from MinIO
|
||||||
|
- Number of directories cleaned from local filesystem
|
||||||
|
- Cleanup duration and performance
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
### Environment Variables
|
### Environment Variables
|
||||||
@@ -786,6 +885,9 @@ The Model Manager system exposes comprehensive Prometheus metrics for operationa
|
|||||||
|----------|-------------|---------|----------|
|
|----------|-------------|---------|----------|
|
||||||
| `TEMPORAL_HOST` | Temporal server address | `localhost:7233` | Yes |
|
| `TEMPORAL_HOST` | Temporal server address | `localhost:7233` | Yes |
|
||||||
| `TEMPORAL_NAMESPACE` | Temporal namespace | `model-manager` | No |
|
| `TEMPORAL_NAMESPACE` | Temporal namespace | `model-manager` | No |
|
||||||
|
| `TEMPORAL_USE_TLS` | Enable TLS for Temporal connection | `false` | No |
|
||||||
|
| `TRAIN_TASK_QUEUE` | Task queue for training workflows | `train_model-queue` | No |
|
||||||
|
| `CLEANUP_TASK_QUEUE` | Task queue for cleanup workflows | `cleanup-queue` | No |
|
||||||
| `POSTGRES_HOST` | PostgreSQL hostname | `localhost` | Yes |
|
| `POSTGRES_HOST` | PostgreSQL hostname | `localhost` | Yes |
|
||||||
| `POSTGRES_PORT` | PostgreSQL port | `5432` | Yes |
|
| `POSTGRES_PORT` | PostgreSQL port | `5432` | Yes |
|
||||||
| `POSTGRES_USER` | PostgreSQL username | `sientia` | Yes |
|
| `POSTGRES_USER` | PostgreSQL username | `sientia` | Yes |
|
||||||
@@ -811,6 +913,14 @@ The Model Manager system exposes comprehensive Prometheus metrics for operationa
|
|||||||
| `MONGODB_PASSWORD` | MongoDB password | `wKZDbMNU1c` | Yes |
|
| `MONGODB_PASSWORD` | MongoDB password | `wKZDbMNU1c` | Yes |
|
||||||
| `MONGODB_DATABASE_NAME` | MongoDB database name | `sientia` | Yes |
|
| `MONGODB_DATABASE_NAME` | MongoDB database name | `sientia` | Yes |
|
||||||
| `MONGODB_TTL_INDEX_HOURS` | MongoDB TTL index hours | `1` | No |
|
| `MONGODB_TTL_INDEX_HOURS` | MongoDB TTL index hours | `1` | No |
|
||||||
|
| `CLEANUP_SCHEDULE_ID` | Cleanup schedule identifier | `cleanup-files-daily` | No |
|
||||||
|
| `CLEANUP_CRON` | Cleanup cron expression | `0 0 * * *` | No |
|
||||||
|
| `CLEANUP_TIMEZONE` | Cleanup schedule timezone | `UTC` | No |
|
||||||
|
| `CLEANUP_EXECUTION_TIMEOUT_HOURS` | Cleanup execution timeout | `1` | No |
|
||||||
|
| `CLEANUP_RETENTION_HOURS` | File retention period (hours) | `24` | No |
|
||||||
|
| `CLEANUP_DRY_RUN` | Dry-run mode (no actual deletion) | `false` | No |
|
||||||
|
| `MAX_KEYS_CLEANUP` | MinIO list operation page size | `1000` | No |
|
||||||
|
| `DEFAULT_CLEANUP_BUCKET` | Default bucket for cleanup | `model-training` | No |
|
||||||
| `LOG_LEVEL` | Application log level | `INFO` | No |
|
| `LOG_LEVEL` | Application log level | `INFO` | No |
|
||||||
| `PROJECT_NAME` | Project name for metrics | `model-manager` | No |
|
| `PROJECT_NAME` | Project name for metrics | `model-manager` | No |
|
||||||
| `HTTP_METRICS_PORT` | Prometheus metrics port | `9090` | No |
|
| `HTTP_METRICS_PORT` | Prometheus metrics port | `9090` | No |
|
||||||
@@ -819,18 +929,24 @@ The Model Manager system exposes comprehensive Prometheus metrics for operationa
|
|||||||
|
|
||||||
#### Workflow Activity Timeouts
|
#### Workflow Activity Timeouts
|
||||||
|
|
||||||
These timeouts control how long each activity in the training workflow can run before timing out. All values are in seconds and are designed to handle large files (up to 200MB).
|
These timeouts control how long each activity in workflows can run before timing out. All values are in seconds.
|
||||||
|
|
||||||
|
**Training Workflow Timeouts:**
|
||||||
|
|
||||||
| Variable | Description | Default | Calculation Basis |
|
| Variable | Description | Default | Calculation Basis |
|
||||||
|----------|-------------|---------|-------------------|
|
|----------|-------------|---------|-------------------|
|
||||||
| `TIMEOUT_VALIDATE_PARAMS` | Parameter validation timeout | `30` | Fast operation, no I/O |
|
| `TIMEOUT_VALIDATE_PARAMS` | Parameter validation timeout | `30` | Fast operation, no I/O |
|
||||||
| `TIMEOUT_DOWNLOAD_FILE` | File download from MinIO timeout | `600` | 200MB @ 1MB/s with 3x buffer (10 min) |
|
| `TIMEOUT_TRAIN_MODEL` | Model training timeout | `2700` | Large dataset processing (45 min) |
|
||||||
| `TIMEOUT_TRAIN_MODEL` | Model training timeout | `1800` | Large dataset processing (30 min) |
|
| `TIMEOUT_DELETE_FILE` | Delete file from MinIO timeout | `120` | MinIO delete operation (2 min) |
|
||||||
| `TIMEOUT_SAVE_MODEL` | Save model to MLFlow timeout | `300` | Artifact upload and logging (5 min) |
|
|
||||||
| `TIMEOUT_CLEANUP_DIRECTORY` | Cleanup temporary directory timeout | `60` | Local filesystem operation (1 min) |
|
|
||||||
| `TIMEOUT_DELETE_FILE` | Delete file from MinIO timeout | `60` | MinIO delete operation (1 min) |
|
|
||||||
| `TIMEOUT_UPDATE_DATABASE` | Database update timeout | `30` | PostgreSQL update query (30 sec) |
|
| `TIMEOUT_UPDATE_DATABASE` | Database update timeout | `30` | PostgreSQL update query (30 sec) |
|
||||||
|
|
||||||
|
**Cleanup Workflow Timeouts:**
|
||||||
|
|
||||||
|
| Variable | Description | Default | Calculation Basis |
|
||||||
|
|----------|-------------|---------|-------------------|
|
||||||
|
| `TIMEOUT_CLEANUP_MINIO` | MinIO cleanup timeout | `300` | Scan and delete multiple files (5 min) |
|
||||||
|
| `TIMEOUT_CLEANUP_LOCAL` | Local cleanup timeout | `120` | Scan and delete directories (2 min) |
|
||||||
|
|
||||||
**Note**: These timeouts can be adjusted based on your infrastructure performance and file sizes. If you're processing files larger than 200MB or have slower network/compute resources, increase these values accordingly.
|
**Note**: These timeouts can be adjusted based on your infrastructure performance and file sizes. If you're processing files larger than 200MB or have slower network/compute resources, increase these values accordingly.
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
@@ -908,18 +1024,23 @@ model_manager/
|
|||||||
│ ├── __init__.py
|
│ ├── __init__.py
|
||||||
│ ├── activities.py # Main activities orchestrator (combines all activities)
|
│ ├── activities.py # Main activities orchestrator (combines all activities)
|
||||||
│ ├── experiment_tracking.py # Experiment status tracking and database operations
|
│ ├── experiment_tracking.py # Experiment status tracking and database operations
|
||||||
│ ├── training.py # ML model training operations
|
│ ├── training.py # ML model training operations (includes MLFlow & MinIO)
|
||||||
│ ├── minio.py # MinIO object storage operations
|
│ └── cleanup.py # File and directory cleanup operations
|
||||||
│ └── mlflow.py # MLFlow model saving and artifact management
|
|
||||||
├── workflows/ # Temporal workflow definitions
|
├── workflows/ # Temporal workflow definitions
|
||||||
│ ├── __init__.py
|
│ ├── __init__.py
|
||||||
│ └── train_model.py # Complete ML model training workflow
|
│ ├── train_model.py # Complete ML model training workflow
|
||||||
|
│ └── cleanup_files.py # Automated file cleanup workflow
|
||||||
|
├── schedules/ # Temporal schedule configurations
|
||||||
|
│ ├── __init__.py
|
||||||
|
│ └── cleanup_schedule.py # Cleanup schedule creation and management
|
||||||
├── worker/ # Worker implementation
|
├── worker/ # Worker implementation
|
||||||
│ ├── __init__.py
|
│ ├── __init__.py
|
||||||
│ └── worker.py # Main worker orchestrator (Temporal client setup)
|
│ └── worker.py # Main worker orchestrator (Temporal client setup)
|
||||||
├── utils/ # Utility functions and helpers
|
├── utils/ # Utility functions and helpers
|
||||||
│ ├── __init__.py
|
│ ├── __init__.py
|
||||||
│ ├── connectors_config.py # Environment-based configuration builders
|
│ ├── connectors_config.py # Environment-based configuration builders
|
||||||
|
│ ├── exceptions.py # Custom exception definitions
|
||||||
|
│ ├── logger_helper.py # Logger initialization utilities
|
||||||
│ ├── models/ # Data models and schemas
|
│ ├── models/ # Data models and schemas
|
||||||
│ │ ├── __init__.py
|
│ │ ├── __init__.py
|
||||||
│ │ ├── train_model_params.py # Training parameters model
|
│ │ ├── train_model_params.py # Training parameters model
|
||||||
@@ -928,7 +1049,19 @@ model_manager/
|
|||||||
│ └── repository/ # Data access layer
|
│ └── repository/ # Data access layer
|
||||||
│ ├── __init__.py
|
│ ├── __init__.py
|
||||||
│ ├── training_repository.py # Training business logic
|
│ ├── training_repository.py # Training business logic
|
||||||
│ └── model_repository.py # MLFlow artifact management
|
│ ├── model_repository.py # MLFlow artifact management
|
||||||
|
│ └── storage_repository.py # MinIO storage operations
|
||||||
|
├── sientia/ # Sientia-specific implementations
|
||||||
|
│ ├── __init__.py
|
||||||
|
│ ├── exceptions.py # Custom exceptions
|
||||||
|
│ ├── metrics.py # Business metrics
|
||||||
|
│ ├── models.py # ML model implementations
|
||||||
|
│ ├── model_serving.py # Model serving utilities
|
||||||
|
│ ├── reports.py # Report generation
|
||||||
|
│ └── utils.py # Utility functions
|
||||||
|
├── reports/ # Report templates and temporary files
|
||||||
|
│ ├── header.html # HTML report header template
|
||||||
|
│ └── temp/ # Temporary report files (cleaned up automatically)
|
||||||
├── metrics.py # Prometheus metrics definitions
|
├── metrics.py # Prometheus metrics definitions
|
||||||
└── __init__.py
|
└── __init__.py
|
||||||
```
|
```
|
||||||
@@ -994,16 +1127,22 @@ export LOG_LEVEL=DEBUG
|
|||||||
### Key Parameters
|
### Key Parameters
|
||||||
|
|
||||||
- **Worker Concurrency**: Adjust `max_concurrent_workflow_tasks` and `max_concurrent_activities`
|
- **Worker Concurrency**: Adjust `max_concurrent_workflow_tasks` and `max_concurrent_activities`
|
||||||
|
- Training workflows: 10 concurrent tasks/activities
|
||||||
|
- Cleanup workflows: 20 concurrent tasks/activities
|
||||||
- **Connection Pools**: Optimize database connection pool sizes
|
- **Connection Pools**: Optimize database connection pool sizes
|
||||||
- **Model Retention**: Configure MLFlow model retention based on requirements
|
- **Model Retention**: Configure MLFlow model retention based on requirements
|
||||||
- **Batch Sizes**: Adjust data processing batch sizes for optimal throughput
|
- **Batch Sizes**: Adjust data processing batch sizes for optimal throughput
|
||||||
|
- **Cleanup Performance**: Tune `MAX_KEYS_CLEANUP` for MinIO list operation page size
|
||||||
|
|
||||||
### Scaling Considerations
|
### Scaling Considerations
|
||||||
|
|
||||||
- **Horizontal Scaling**: Deploy multiple worker instances
|
- **Horizontal Scaling**: Deploy multiple worker instances
|
||||||
- **Task Queue Distribution**: Use multiple task queues for different workflow types
|
- **Task Queue Distribution**: Two dedicated task queues for workflow isolation
|
||||||
|
- `train_model-queue`: Training workflows
|
||||||
|
- `cleanup-queue`: Cleanup workflows
|
||||||
- **Database Performance**: Optimize indexes and connection pooling
|
- **Database Performance**: Optimize indexes and connection pooling
|
||||||
- **MLFlow Performance**: Configure appropriate model serving resources
|
- **MLFlow Performance**: Configure appropriate model serving resources
|
||||||
|
- **Storage Management**: Adjust cleanup retention period based on storage capacity and costs
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
|
|||||||
@@ -7,25 +7,25 @@ with workflow.unsafe.imports_passed_through():
|
|||||||
from sientia_do.observability.logger import Logger
|
from sientia_do.observability.logger import Logger
|
||||||
from sientia_do.observability.metrics_controller import MetricsController
|
from sientia_do.observability.metrics_controller import MetricsController
|
||||||
|
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
from model_manager.activities.experiment_tracking import ExperimentTracking
|
from model_manager.activities.experiment_tracking import ExperimentTracking
|
||||||
from model_manager.activities.training import Training
|
from model_manager.activities.training import Training
|
||||||
from model_manager.utils.repository.model_repository import ModelRepository
|
from model_manager.utils.repository.model_repository import ModelRepository
|
||||||
from model_manager.utils.repository.storage_repository import StorageRepository
|
from model_manager.utils.repository.storage_repository import StorageRepository
|
||||||
|
|
||||||
|
|
||||||
class Activities(ExperimentTracking, Training):
|
class Activities(ExperimentTracking, Training, Cleanup):
|
||||||
"""
|
"""
|
||||||
Main activities orchestrator for the Model Manager system.
|
Main activities orchestrator for the Model Manager system.
|
||||||
|
|
||||||
This class combines functionality from multiple activity classes to provide
|
This class combines functionality from multiple activity classes to provide
|
||||||
a unified interface for all workflow operations. It manages database connections,
|
a unified interface for all workflow operations. It manages database connections,
|
||||||
MLFlow model interactions, MinIO storage operations, and data quality validation.
|
MLFlow model interactions, MinIO storage operations, and cleanup operations.
|
||||||
|
|
||||||
The class implements multiple inheritance to combine specialized functionality:
|
The class implements multiple inheritance to combine specialized functionality:
|
||||||
- ExperimentTracking: ML experiment lifecycle tracking and database operations (extends Postgres)
|
- ExperimentTracking: ML experiment lifecycle tracking and database operations
|
||||||
- MLFlow: Model saving and artifact management operations
|
- Training: ML model training operations with MLFlow and MinIO integration
|
||||||
- MinIO: Object storage operations (file upload/download/delete)
|
- Cleanup: File and directory cleanup operations for MinIO and local filesystem
|
||||||
- Training: ML model training operations (extends BaseActivity)
|
|
||||||
|
|
||||||
Attributes:
|
Attributes:
|
||||||
postgres_config (dict): PostgreSQL connection configuration
|
postgres_config (dict): PostgreSQL connection configuration
|
||||||
@@ -109,6 +109,14 @@ class Activities(ExperimentTracking, Training):
|
|||||||
metrics_controller=metrics_controller,
|
metrics_controller=metrics_controller,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
Cleanup.__init__(
|
||||||
|
self,
|
||||||
|
storage_repository=self.storage_repository,
|
||||||
|
logger=logger,
|
||||||
|
notification_handler=notification_handler,
|
||||||
|
metrics_controller=metrics_controller,
|
||||||
|
)
|
||||||
|
|
||||||
def __del__(self):
|
def __del__(self):
|
||||||
"""
|
"""
|
||||||
Destructor to safely handle cleanup during garbage collection.
|
Destructor to safely handle cleanup during garbage collection.
|
||||||
@@ -143,3 +151,5 @@ class Activities(ExperimentTracking, Training):
|
|||||||
Prefer calling this method explicitly rather than relying on __del__.
|
Prefer calling this method explicitly rather than relying on __del__.
|
||||||
"""
|
"""
|
||||||
ExperimentTracking.close(self)
|
ExperimentTracking.close(self)
|
||||||
|
self.info('Postgres client closed')
|
||||||
|
self.storage_repository.close()
|
||||||
|
|||||||
338
model_manager/activities/cleanup.py
Normal file
338
model_manager/activities/cleanup.py
Normal file
@@ -0,0 +1,338 @@
|
|||||||
|
"""
|
||||||
|
Cleanup activities for removing stale files from MinIO and local filesystem.
|
||||||
|
|
||||||
|
This module provides activities for cleaning up temporary files and directories
|
||||||
|
that are older than the configured retention period. It operates independently
|
||||||
|
of the database, using timestamps embedded in filenames.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from temporalio import activity, workflow
|
||||||
|
|
||||||
|
with workflow.unsafe.imports_passed_through():
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import traceback
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||||
|
from sientia_do.notifications.models import NotificationLevel
|
||||||
|
from sientia_do.observability.logger import Logger
|
||||||
|
from sientia_do.observability.metrics_controller import MetricsController
|
||||||
|
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||||
|
|
||||||
|
from model_manager.metrics import ACTIVITY_EXECUTION_TOTAL, WORKFLOW_EXECUTION_TOTAL
|
||||||
|
from model_manager.utils.repository.storage_repository import StorageRepository
|
||||||
|
|
||||||
|
RETENTION_HOURS = int(os.getenv('CLEANUP_RETENTION_HOURS', '24'))
|
||||||
|
DRY_RUN = os.getenv('CLEANUP_DRY_RUN', 'false').lower() == 'true'
|
||||||
|
MAX_KEYS_CLEANUP = int(os.getenv('MAX_KEYS_CLEANUP', '1000'))
|
||||||
|
|
||||||
|
|
||||||
|
class Cleanup(SientiaMonitoring):
|
||||||
|
"""
|
||||||
|
Activity for cleaning up stale files and directories.
|
||||||
|
|
||||||
|
This activity extends SientiaMonitoring and handles cleanup of:
|
||||||
|
- MinIO files with timestamp prefixes (timestamp-filename pattern)
|
||||||
|
- Local temporary directories with timestamp suffixes
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
storage_repository: StorageRepository,
|
||||||
|
logger: Logger,
|
||||||
|
notification_handler: NotificationHandler,
|
||||||
|
metrics_controller: MetricsController,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Initialize Cleanup activity.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
storage_repository: Repository for MinIO operations
|
||||||
|
logger: Logger instance for observability
|
||||||
|
notification_handler: Handler for sending notifications
|
||||||
|
metrics_controller: Controller for metrics emission
|
||||||
|
"""
|
||||||
|
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||||
|
self.storage_repository = storage_repository
|
||||||
|
|
||||||
|
# Configuration from environment variables
|
||||||
|
self.retention_hours = RETENTION_HOURS
|
||||||
|
self.dry_run = DRY_RUN
|
||||||
|
|
||||||
|
# MinIO list operation page size
|
||||||
|
self.max_keys_cleanup = MAX_KEYS_CLEANUP
|
||||||
|
|
||||||
|
# Regex patterns for timestamp extraction
|
||||||
|
self.minio_timestamp_pattern = re.compile(r'^(\d{13})-(.+)') # timestamp-filename
|
||||||
|
self.dir_timestamp_pattern = re.compile(
|
||||||
|
r'^(.+)_(\d{8}_\d{6}_\d{6})$'
|
||||||
|
) # name_YYYYMMDD_HHMMSS_microseconds
|
||||||
|
|
||||||
|
@activity.defn(name='cleanup_minio_files')
|
||||||
|
async def cleanup_minio_files(self, input_data: dict[str, Any]) -> None:
|
||||||
|
"""
|
||||||
|
Clean up stale files from MinIO based on timestamp in filename.
|
||||||
|
|
||||||
|
This activity scans a single MinIO bucket for files following the pattern
|
||||||
|
'{timestamp}-{filename}' where timestamp is milliseconds since epoch.
|
||||||
|
Files older than the retention period are deleted.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
input_data: Cleanup configuration containing:
|
||||||
|
- metadata (dict): Workflow execution metadata
|
||||||
|
- bucket_name (str): Name of the bucket to scan
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None: Results are logged and tracked via metrics
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception: If cleanup fails (after sending notification)
|
||||||
|
"""
|
||||||
|
metadata = input_data.get('metadata', {})
|
||||||
|
bucket_name = input_data.get('bucket_name')
|
||||||
|
metrics_status = 'success'
|
||||||
|
|
||||||
|
if not bucket_name:
|
||||||
|
raise ValueError('bucket_name must be provided')
|
||||||
|
|
||||||
|
cutoff_time = datetime.now(UTC) - timedelta(hours=self.retention_hours)
|
||||||
|
cutoff_timestamp_ms = int(cutoff_time.timestamp() * 1000)
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.info(
|
||||||
|
f'Starting MinIO cleanup - Bucket: {bucket_name}, '
|
||||||
|
f'Retention: {self.retention_hours}h, Dry run: {self.dry_run}, '
|
||||||
|
f'Cutoff: {cutoff_time.isoformat()}',
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
files_scanned = 0
|
||||||
|
files_deleted = 0
|
||||||
|
errors = []
|
||||||
|
|
||||||
|
# List objects in the specified bucket
|
||||||
|
max_keys = self.max_keys_cleanup # Use environment variable for page size
|
||||||
|
objects = self.storage_repository.list_bucket_objects(bucket_name, max_keys)
|
||||||
|
|
||||||
|
for obj_key in objects:
|
||||||
|
files_scanned += 1
|
||||||
|
|
||||||
|
# Extract timestamp from filename
|
||||||
|
match = self.minio_timestamp_pattern.match(obj_key)
|
||||||
|
if not match:
|
||||||
|
self.debug(f'Skipping file without timestamp pattern: {obj_key}', metadata)
|
||||||
|
continue
|
||||||
|
|
||||||
|
file_timestamp_ms = int(match.group(1))
|
||||||
|
|
||||||
|
if file_timestamp_ms < cutoff_timestamp_ms:
|
||||||
|
if self.dry_run:
|
||||||
|
self.info(
|
||||||
|
f'[DRY RUN] Would delete: {obj_key} (age: {(cutoff_time.timestamp() - file_timestamp_ms / 1000) / 3600:.1f}h)',
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
files_deleted += 1
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
self.storage_repository.delete_file(bucket_name, obj_key)
|
||||||
|
self.info(f'Deleted stale file: {obj_key}', metadata)
|
||||||
|
files_deleted += 1
|
||||||
|
except OSError as e:
|
||||||
|
error_msg = f'Failed to delete {obj_key}: {str(e)}'
|
||||||
|
errors.append(error_msg)
|
||||||
|
self.error(error_msg, metadata)
|
||||||
|
else:
|
||||||
|
self.debug(
|
||||||
|
f'Keeping recent file: {obj_key} (age: {(cutoff_time.timestamp() - file_timestamp_ms / 1000) / 3600:.1f}h)',
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.info(
|
||||||
|
f'MinIO cleanup completed - Bucket: {bucket_name}, '
|
||||||
|
f'Scanned: {files_scanned}, Deleted: {files_deleted}, Errors: {len(errors)}',
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
metrics_status = 'error'
|
||||||
|
error_msg = f'Error in MinIO cleanup: {str(e)}'
|
||||||
|
trace = traceback.format_exc()
|
||||||
|
|
||||||
|
self.send_notification(
|
||||||
|
metadata=metadata,
|
||||||
|
notification_id='CLEANUP_MINIO_ERROR',
|
||||||
|
message=error_msg,
|
||||||
|
block='cleanup_minio_files',
|
||||||
|
level=NotificationLevel.ERROR,
|
||||||
|
attachment_content=trace,
|
||||||
|
)
|
||||||
|
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
await self._emit_metrics(
|
||||||
|
metadata=metadata,
|
||||||
|
metrics_status=metrics_status,
|
||||||
|
activity_name='cleanup_minio_files',
|
||||||
|
emit_workflow_metric=(metrics_status == 'error'),
|
||||||
|
)
|
||||||
|
|
||||||
|
@activity.defn(name='cleanup_temp_directories')
|
||||||
|
async def cleanup_temp_directories(self, input_data: dict[str, Any]) -> None:
|
||||||
|
"""
|
||||||
|
Clean up stale temporary directories based on timestamp in directory name.
|
||||||
|
|
||||||
|
This activity scans the reports/temp directory for subdirectories following
|
||||||
|
the pattern '{name}_{timestamp}' where timestamp is in YYYYMMDD_HHMMSS_microseconds format.
|
||||||
|
Directories older than the retention period are deleted.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
input_data: Cleanup configuration containing:
|
||||||
|
- metadata (dict): Workflow execution metadata
|
||||||
|
- temp_path (str): Path to temp directory (optional, defaults to reports/temp)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None: Results are logged and tracked via metrics
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception: If cleanup fails (after sending notification)
|
||||||
|
"""
|
||||||
|
metadata = input_data.get('metadata', {})
|
||||||
|
temp_path = input_data.get('temp_path', 'model_manager/reports/temp')
|
||||||
|
metrics_status = 'success'
|
||||||
|
|
||||||
|
cutoff_time = datetime.now() - timedelta(hours=self.retention_hours)
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.info(
|
||||||
|
f'Starting local directory cleanup - Path: {temp_path}, '
|
||||||
|
f'Retention: {self.retention_hours}h, Dry run: {self.dry_run}',
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not os.path.exists(temp_path):
|
||||||
|
self.warning(f'Temp directory does not exist: {temp_path}', metadata)
|
||||||
|
return
|
||||||
|
|
||||||
|
directories_scanned = 0
|
||||||
|
directories_deleted = 0
|
||||||
|
errors = []
|
||||||
|
|
||||||
|
for item_name in os.listdir(temp_path):
|
||||||
|
item_path = os.path.join(temp_path, item_name)
|
||||||
|
|
||||||
|
if not os.path.isdir(item_path):
|
||||||
|
continue
|
||||||
|
|
||||||
|
directories_scanned += 1
|
||||||
|
|
||||||
|
# Extract timestamp from directory name
|
||||||
|
match = self.dir_timestamp_pattern.match(item_name)
|
||||||
|
if not match:
|
||||||
|
self.debug(
|
||||||
|
f'Skipping directory without timestamp pattern: {item_name}', metadata
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
timestamp_str = match.group(2)
|
||||||
|
try:
|
||||||
|
# Parse YYYYMMDD_HHMMSS_microseconds
|
||||||
|
dir_time = datetime.strptime(timestamp_str, '%Y%m%d_%H%M%S_%f')
|
||||||
|
|
||||||
|
if dir_time < cutoff_time:
|
||||||
|
age_hours = (datetime.now() - dir_time).total_seconds() / 3600
|
||||||
|
|
||||||
|
if self.dry_run:
|
||||||
|
self.info(
|
||||||
|
f'[DRY RUN] Would delete directory: {item_name} (age: {age_hours:.1f}h)',
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
directories_deleted += 1
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
shutil.rmtree(item_path)
|
||||||
|
self.info(
|
||||||
|
f'Deleted stale directory: {item_name} (age: {age_hours:.1f}h)',
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
directories_deleted += 1
|
||||||
|
except OSError as e:
|
||||||
|
error_msg = f'Failed to delete directory {item_name}: {str(e)}'
|
||||||
|
errors.append(error_msg)
|
||||||
|
self.error(error_msg, metadata)
|
||||||
|
else:
|
||||||
|
age_hours = (datetime.now() - dir_time).total_seconds() / 3600
|
||||||
|
self.debug(
|
||||||
|
f'Keeping recent directory: {item_name} (age: {age_hours:.1f}h)',
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
except ValueError as e:
|
||||||
|
error_msg = f'Invalid timestamp format in directory {item_name}: {str(e)}'
|
||||||
|
errors.append(error_msg)
|
||||||
|
self.error(error_msg, metadata)
|
||||||
|
|
||||||
|
self.info(
|
||||||
|
f'Directory cleanup completed - Scanned: {directories_scanned}, '
|
||||||
|
f'Deleted: {directories_deleted}, Errors: {len(errors)}',
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
metrics_status = 'error'
|
||||||
|
error_msg = f'Error in directory cleanup: {str(e)}'
|
||||||
|
trace = traceback.format_exc()
|
||||||
|
|
||||||
|
self.send_notification(
|
||||||
|
metadata=metadata,
|
||||||
|
notification_id='CLEANUP_DIRECTORIES_ERROR',
|
||||||
|
message=error_msg,
|
||||||
|
block='cleanup_temp_directories',
|
||||||
|
level=NotificationLevel.ERROR,
|
||||||
|
attachment_content=trace,
|
||||||
|
)
|
||||||
|
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
await self._emit_metrics(
|
||||||
|
metadata=metadata,
|
||||||
|
metrics_status=metrics_status,
|
||||||
|
activity_name='cleanup_temp_directories',
|
||||||
|
emit_workflow_metric=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _emit_metrics(
|
||||||
|
self,
|
||||||
|
metadata: dict[str, Any],
|
||||||
|
metrics_status: str,
|
||||||
|
activity_name: str,
|
||||||
|
emit_workflow_metric: bool,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Emit workflow and activity execution metrics.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
metadata: Activity metadata containing pod_id and workflow_name
|
||||||
|
metrics_status: Execution status ('success' or 'error')
|
||||||
|
activity_name: Name of the activity being executed
|
||||||
|
"""
|
||||||
|
if emit_workflow_metric:
|
||||||
|
await self.emit_metric(
|
||||||
|
metric_object=WORKFLOW_EXECUTION_TOTAL,
|
||||||
|
tags={
|
||||||
|
'pod_id': metadata.get('pod_id'),
|
||||||
|
'workflow_name': metadata.get('workflow_name'),
|
||||||
|
'status': metrics_status,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.emit_metric(
|
||||||
|
metric_object=ACTIVITY_EXECUTION_TOTAL,
|
||||||
|
tags={
|
||||||
|
'pod_id': metadata.get('pod_id'),
|
||||||
|
'activity_name': activity_name,
|
||||||
|
'status': metrics_status,
|
||||||
|
},
|
||||||
|
)
|
||||||
@@ -76,7 +76,8 @@ class ExperimentTracking(Postgres):
|
|||||||
Raises:
|
Raises:
|
||||||
ConnectionError: If database connection cannot be established
|
ConnectionError: If database connection cannot be established
|
||||||
"""
|
"""
|
||||||
super().__init__(
|
Postgres.__init__(
|
||||||
|
self,
|
||||||
host=host,
|
host=host,
|
||||||
port=port,
|
port=port,
|
||||||
user=user,
|
user=user,
|
||||||
@@ -89,6 +90,8 @@ class ExperimentTracking(Postgres):
|
|||||||
metrics_controller=metrics_controller,
|
metrics_controller=metrics_controller,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
self.info(f'Postgres client initialized at {host}:{port}')
|
||||||
|
|
||||||
def __del__(self):
|
def __del__(self):
|
||||||
"""
|
"""
|
||||||
Destructor to safely handle cleanup during garbage collection.
|
Destructor to safely handle cleanup during garbage collection.
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ class Training(SientiaMonitoring):
|
|||||||
logger: Logger instance for observability
|
logger: Logger instance for observability
|
||||||
notification_handler: Handler for sending notifications
|
notification_handler: Handler for sending notifications
|
||||||
"""
|
"""
|
||||||
super().__init__(logger, notification_handler, metrics_controller)
|
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||||
self.training_repository = TrainingRepository(logger)
|
self.training_repository = TrainingRepository(logger)
|
||||||
self.model_repository = model_repository
|
self.model_repository = model_repository
|
||||||
self.storage_repository = storage_repository
|
self.storage_repository = storage_repository
|
||||||
|
|||||||
0
model_manager/schedules/__init__.py
Normal file
0
model_manager/schedules/__init__.py
Normal file
89
model_manager/schedules/cleanup_schedule.py
Normal file
89
model_manager/schedules/cleanup_schedule.py
Normal file
@@ -0,0 +1,89 @@
|
|||||||
|
"""Schedule configuration for cleanup workflow."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
|
from sientia_do.observability.logger import Logger as SientiaLogger
|
||||||
|
from temporalio.client import (
|
||||||
|
Client,
|
||||||
|
Schedule,
|
||||||
|
ScheduleActionStartWorkflow,
|
||||||
|
ScheduleSpec,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Schedule configuration from environment variables
|
||||||
|
SCHEDULE_ID = os.getenv('CLEANUP_SCHEDULE_ID', 'cleanup-files-daily')
|
||||||
|
CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 0 * * *') # Default: midnight UTC
|
||||||
|
CLEANUP_TIMEZONE = os.getenv('CLEANUP_TIMEZONE', 'UTC')
|
||||||
|
CLEANUP_TASK_QUEUE = os.getenv('CLEANUP_TASK_QUEUE', 'cleanup-queue')
|
||||||
|
CLEANUP_EXECUTION_TIMEOUT_HOURS = int(os.getenv('CLEANUP_EXECUTION_TIMEOUT_HOURS', '1'))
|
||||||
|
|
||||||
|
|
||||||
|
async def schedule_exists(
|
||||||
|
client: Client, schedule_id: str, logger: SientiaLogger, metadata: dict[str, str | None]
|
||||||
|
) -> bool:
|
||||||
|
"""
|
||||||
|
Check if a schedule already exists.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client: Temporal client instance
|
||||||
|
schedule_id: ID of the schedule to check
|
||||||
|
logger: Logger instance for error logging
|
||||||
|
metadata: Metadata dictionary for logging context
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if schedule exists, False otherwise
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
async for schedule in await client.list_schedules():
|
||||||
|
if schedule.id == schedule_id:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.custom_error(f'Error checking if schedule exists: {e}', metadata)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def create_cleanup_schedule(
|
||||||
|
client: Client, logger: SientiaLogger, metadata: dict[str, str | None]
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Create or update the cleanup files schedule.
|
||||||
|
|
||||||
|
This function is idempotent and can be called multiple times safely.
|
||||||
|
It will only create the schedule if it doesn't already exist.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client: Temporal client instance
|
||||||
|
logger: Logger instance for logging schedule operations
|
||||||
|
metadata: Metadata dictionary for logging context
|
||||||
|
"""
|
||||||
|
# Check if schedule already exists
|
||||||
|
if await schedule_exists(client, SCHEDULE_ID, logger, metadata):
|
||||||
|
logger.custom_info(
|
||||||
|
f"Schedule '{SCHEDULE_ID}' already configured, skipping creation", metadata
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
await client.create_schedule(
|
||||||
|
SCHEDULE_ID,
|
||||||
|
Schedule(
|
||||||
|
action=ScheduleActionStartWorkflow(
|
||||||
|
'cleanup_files',
|
||||||
|
{}, # Empty input, will use default bucket from environment
|
||||||
|
id=f'cleanup-files-scheduled-{SCHEDULE_ID}',
|
||||||
|
task_queue=CLEANUP_TASK_QUEUE,
|
||||||
|
execution_timeout=timedelta(hours=CLEANUP_EXECUTION_TIMEOUT_HOURS),
|
||||||
|
),
|
||||||
|
spec=ScheduleSpec(
|
||||||
|
cron_expressions=[CLEANUP_CRON],
|
||||||
|
time_zone_name=CLEANUP_TIMEZONE,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.custom_info(
|
||||||
|
f"Schedule '{SCHEDULE_ID}' created successfully. "
|
||||||
|
f'Cleanup will run at: {CLEANUP_CRON} ({CLEANUP_TIMEZONE})',
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
@@ -84,6 +84,7 @@ def build_mongodb_config() -> dict[str, Any]:
|
|||||||
'connection_string': connection_string,
|
'connection_string': connection_string,
|
||||||
'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
|
'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
|
||||||
'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600,
|
'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600,
|
||||||
|
'uri': uri,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -31,8 +31,8 @@ warnings.filterwarnings('ignore', category=FutureWarning, message=".*'squared' i
|
|||||||
class ModelRepository:
|
class ModelRepository:
|
||||||
def __init__(self, url, username, password, logger: Logger):
|
def __init__(self, url, username, password, logger: Logger):
|
||||||
self.model_serving = ModelServing(tracking_uri=url, username=username, password=password)
|
self.model_serving = ModelServing(tracking_uri=url, username=username, password=password)
|
||||||
|
|
||||||
self.logger = logger
|
self.logger = logger
|
||||||
|
self.logger.info(f'MLFlow client initialized at {url}')
|
||||||
|
|
||||||
def save_model(self, train_result: TrainModelResult) -> TrainModelResult:
|
def save_model(self, train_result: TrainModelResult) -> TrainModelResult:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -86,7 +86,11 @@ class StorageRepository:
|
|||||||
use_ssl=use_ssl,
|
use_ssl=use_ssl,
|
||||||
)
|
)
|
||||||
|
|
||||||
self.logger.info(f'MinIO client initialized successfully: {endpoint_url}')
|
self.logger.info(f'MinIO client initialized at {endpoint_url}')
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self.minio_client.close()
|
||||||
|
self.logger.info('MinIO client closed')
|
||||||
|
|
||||||
def fetch_file(self, bucket_name: str, file_name: str) -> BytesIO:
|
def fetch_file(self, bucket_name: str, file_name: str) -> BytesIO:
|
||||||
"""
|
"""
|
||||||
@@ -125,3 +129,36 @@ class StorageRepository:
|
|||||||
"""
|
"""
|
||||||
self.minio_client.delete_object(Bucket=bucket_name, Key=file_name)
|
self.minio_client.delete_object(Bucket=bucket_name, Key=file_name)
|
||||||
self.logger.info(f'File deleted successfully: {bucket_name}/{file_name}')
|
self.logger.info(f'File deleted successfully: {bucket_name}/{file_name}')
|
||||||
|
|
||||||
|
def list_bucket_objects(self, bucket_name: str, max_keys: int = 1000) -> list[str]:
|
||||||
|
"""
|
||||||
|
List objects in a MinIO bucket.
|
||||||
|
|
||||||
|
This method uses the MinIO/S3 list_objects_v2 API to retrieve objects
|
||||||
|
from the specified bucket. This is optimized for cleanup operations
|
||||||
|
by using configurable pagination.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
bucket_name: Name of the bucket to list objects from.
|
||||||
|
max_keys: Maximum number of keys per page (default: 1000).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List[str]: List of object keys (file names).
|
||||||
|
"""
|
||||||
|
# Use list_objects_v2 for efficient pagination
|
||||||
|
paginator = self.minio_client.get_paginator('list_objects_v2')
|
||||||
|
|
||||||
|
pages = paginator.paginate(Bucket=bucket_name, MaxKeys=max_keys)
|
||||||
|
|
||||||
|
objects = []
|
||||||
|
total_count = 0
|
||||||
|
|
||||||
|
for page in pages:
|
||||||
|
if 'Contents' in page:
|
||||||
|
for obj in page['Contents']:
|
||||||
|
objects.append(obj['Key'])
|
||||||
|
total_count += 1
|
||||||
|
|
||||||
|
self.logger.info(f'Listed {total_count} objects from bucket {bucket_name}')
|
||||||
|
|
||||||
|
return objects
|
||||||
|
|||||||
@@ -2,9 +2,11 @@
|
|||||||
|
|
||||||
This module provides the main worker implementation for the Sientia DataOps Model Manager system.
|
This module provides the main worker implementation for the Sientia DataOps Model Manager system.
|
||||||
It orchestrates Temporal workers, manages task queues, and handles the lifecycle of
|
It orchestrates Temporal workers, manages task queues, and handles the lifecycle of
|
||||||
model training workflows.
|
model training and cleanup workflows.
|
||||||
|
|
||||||
The worker supports the train_model-queue task queue for ML model training workflows.
|
The worker supports two task queues:
|
||||||
|
- train_model-queue: For ML model training workflows
|
||||||
|
- cleanup-queue: For file cleanup workflows
|
||||||
|
|
||||||
Key Features:
|
Key Features:
|
||||||
- Automatic scaling with PollerBehaviorAutoscaling
|
- Automatic scaling with PollerBehaviorAutoscaling
|
||||||
@@ -12,14 +14,18 @@ Key Features:
|
|||||||
- Comprehensive error handling and logging
|
- Comprehensive error handling and logging
|
||||||
- Graceful shutdown with cleanup
|
- Graceful shutdown with cleanup
|
||||||
- ML model training pipeline orchestration
|
- ML model training pipeline orchestration
|
||||||
|
- Automated cleanup schedule management
|
||||||
|
|
||||||
Environment Variables:
|
Environment Variables:
|
||||||
- TEMPORAL_HOST: Temporal server address (default: localhost:7233)
|
- TEMPORAL_HOST: Temporal server address (default: localhost:7233)
|
||||||
- TEMPORAL_NAMESPACE: Temporal namespace (default: model_manager)
|
- TEMPORAL_NAMESPACE: Temporal namespace (default: model-manager)
|
||||||
|
- TEMPORAL_USE_TLS: Enable TLS for Temporal connection (default: false)
|
||||||
|
- TRAIN_TASK_QUEUE: Task queue for training workflows (default: train_model-queue)
|
||||||
|
- CLEANUP_TASK_QUEUE: Task queue for cleanup workflows (default: cleanup-queue)
|
||||||
- POD_ID: Kubernetes pod identifier for metrics
|
- POD_ID: Kubernetes pod identifier for metrics
|
||||||
- HTTP_METRICS_PORT: Prometheus metrics server port (default: 9090)
|
- HTTP_METRICS_PORT: Prometheus metrics server port (default: 9090)
|
||||||
- HTTP_SDK_METRICS_PORT: Temporal SDK metrics port (default: 9091)
|
- HTTP_SDK_METRICS_PORT: Temporal SDK metrics port (default: 9091)
|
||||||
- PROJECT_NAME: Project name for notifications (default: model_manager)
|
- PROJECT_NAME: Project name for notifications (default: model-manager)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from temporalio import client, workflow
|
from temporalio import client, workflow
|
||||||
@@ -33,9 +39,11 @@ with workflow.unsafe.imports_passed_through():
|
|||||||
|
|
||||||
from prometheus_client import start_http_server
|
from prometheus_client import start_http_server
|
||||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||||
|
from sientia_do.observability.logger import Logger as SientiaLogger
|
||||||
|
|
||||||
from model_manager import metrics
|
from model_manager import metrics
|
||||||
from model_manager.activities.activities import Activities
|
from model_manager.activities.activities import Activities
|
||||||
|
from model_manager.schedules.cleanup_schedule import create_cleanup_schedule
|
||||||
from model_manager.utils.connectors_config import (
|
from model_manager.utils.connectors_config import (
|
||||||
build_minio_config,
|
build_minio_config,
|
||||||
build_mlflow_config,
|
build_mlflow_config,
|
||||||
@@ -43,10 +51,13 @@ with workflow.unsafe.imports_passed_through():
|
|||||||
build_postgres_config,
|
build_postgres_config,
|
||||||
)
|
)
|
||||||
from model_manager.utils.logger_helper import get_logger
|
from model_manager.utils.logger_helper import get_logger
|
||||||
|
from model_manager.workflows.cleanup_files import CleanupFiles
|
||||||
from model_manager.workflows.train_model import TrainModel
|
from model_manager.workflows.train_model import TrainModel
|
||||||
|
|
||||||
POD_ID = os.getenv('POD_ID')
|
POD_ID = os.getenv('POD_ID')
|
||||||
SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091'))
|
SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091'))
|
||||||
|
TRAIN_TASK_QUEUE = os.getenv('TRAIN_TASK_QUEUE', 'train_model-queue')
|
||||||
|
CLEANUP_TASK_QUEUE = os.getenv('CLEANUP_TASK_QUEUE', 'cleanup-queue')
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def main():
|
||||||
@@ -69,21 +80,16 @@ async def main():
|
|||||||
SystemExit: On graceful shutdown or error conditions
|
SystemExit: On graceful shutdown or error conditions
|
||||||
"""
|
"""
|
||||||
host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
|
host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
|
||||||
|
use_tls = os.getenv('TEMPORAL_USE_TLS', 'false').lower() == 'true'
|
||||||
logger = get_logger(__name__)
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
metadata = {
|
metadata = {
|
||||||
'pod_id': POD_ID,
|
'pod_id': POD_ID,
|
||||||
'workflow_name': 'train_model',
|
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.custom_info(f'Starting Worker with POD_ID: {POD_ID}', metadata)
|
start_prometheus_server(logger, metadata)
|
||||||
|
|
||||||
logger.custom_info('Starting prometheus client...', metadata)
|
|
||||||
start_prometheus_server()
|
|
||||||
|
|
||||||
logger.custom_info('Starting Notification Handler...', metadata)
|
|
||||||
|
|
||||||
mongo_config = build_mongodb_config()
|
mongo_config = build_mongodb_config()
|
||||||
|
|
||||||
notification_handler = NotificationHandler(
|
notification_handler = NotificationHandler(
|
||||||
connection_string=mongo_config['connection_string'],
|
connection_string=mongo_config['connection_string'],
|
||||||
database=mongo_config['database_name'],
|
database=mongo_config['database_name'],
|
||||||
@@ -91,7 +97,7 @@ async def main():
|
|||||||
project_name=os.getenv('PROJECT_NAME', 'model-manager'),
|
project_name=os.getenv('PROJECT_NAME', 'model-manager'),
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.custom_info('Starting Activities...', metadata)
|
logger.custom_info(f'MongoDB client initialized at {mongo_config["uri"]}', metadata)
|
||||||
|
|
||||||
activities = Activities(
|
activities = Activities(
|
||||||
postgres_config=build_postgres_config(),
|
postgres_config=build_postgres_config(),
|
||||||
@@ -101,28 +107,35 @@ async def main():
|
|||||||
notification_handler=notification_handler,
|
notification_handler=notification_handler,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.custom_info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata)
|
|
||||||
|
|
||||||
new_runtime = Runtime(
|
new_runtime = Runtime(
|
||||||
telemetry=TelemetryConfig(
|
telemetry=TelemetryConfig(
|
||||||
metrics=PrometheusConfig(bind_address=f'0.0.0.0:{SDK_METRICS_PORT}')
|
metrics=PrometheusConfig(bind_address=f'0.0.0.0:{SDK_METRICS_PORT}')
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.custom_info(f'Starting Temporal Client at {host}...', metadata)
|
logger.custom_info(f'SDK metrics server initialized on port {SDK_METRICS_PORT}', metadata)
|
||||||
|
|
||||||
temporal_client = await client.Client.connect(
|
temporal_client = await client.Client.connect(
|
||||||
target_host=host,
|
target_host=host,
|
||||||
namespace=os.getenv('TEMPORAL_NAMESPACE', 'model-manager'),
|
namespace=os.getenv('TEMPORAL_NAMESPACE', 'model-manager'),
|
||||||
runtime=new_runtime,
|
runtime=new_runtime,
|
||||||
|
tls=use_tls,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.custom_info('Starting Workers...', metadata)
|
logger.custom_info(f'Temporal client initialized at {host}', metadata)
|
||||||
|
|
||||||
|
# Create cleanup schedule (idempotent - only creates if doesn't exist)
|
||||||
|
try:
|
||||||
|
await create_cleanup_schedule(temporal_client, logger, metadata)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.custom_error(f'Failed to configure cleanup schedule: {e}', metadata)
|
||||||
|
# Don't fail the worker startup if schedule creation fails
|
||||||
|
# The schedule can be created manually if needed
|
||||||
|
|
||||||
workers = [
|
workers = [
|
||||||
Worker(
|
Worker(
|
||||||
temporal_client,
|
temporal_client,
|
||||||
task_queue='train_model-queue',
|
task_queue=TRAIN_TASK_QUEUE,
|
||||||
workflows=[TrainModel],
|
workflows=[TrainModel],
|
||||||
activities=[
|
activities=[
|
||||||
activities.update_experiment_run,
|
activities.update_experiment_run,
|
||||||
@@ -130,20 +143,36 @@ async def main():
|
|||||||
activities.train_model,
|
activities.train_model,
|
||||||
activities.cleanup_resources,
|
activities.cleanup_resources,
|
||||||
],
|
],
|
||||||
max_concurrent_workflow_tasks=50,
|
max_concurrent_workflow_tasks=10,
|
||||||
max_concurrent_activities=50,
|
max_concurrent_activities=10,
|
||||||
max_concurrent_local_activities=50,
|
max_concurrent_local_activities=10,
|
||||||
max_cached_workflows=200,
|
max_cached_workflows=100,
|
||||||
|
workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
|
||||||
|
activity_task_poller_behavior=PollerBehaviorAutoscaling(),
|
||||||
|
),
|
||||||
|
Worker(
|
||||||
|
temporal_client,
|
||||||
|
task_queue=CLEANUP_TASK_QUEUE,
|
||||||
|
workflows=[CleanupFiles],
|
||||||
|
activities=[
|
||||||
|
activities.cleanup_minio_files,
|
||||||
|
activities.cleanup_temp_directories,
|
||||||
|
],
|
||||||
|
max_concurrent_workflow_tasks=20,
|
||||||
|
max_concurrent_activities=20,
|
||||||
|
max_concurrent_local_activities=20,
|
||||||
|
max_cached_workflows=100,
|
||||||
workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
|
workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
|
||||||
activity_task_poller_behavior=PollerBehaviorAutoscaling(),
|
activity_task_poller_behavior=PollerBehaviorAutoscaling(),
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
handlers = []
|
handlers = []
|
||||||
|
|
||||||
for w in workers:
|
for w in workers:
|
||||||
handlers.append(w.run())
|
handlers.append(w.run())
|
||||||
|
|
||||||
logger.custom_info('Workers started successfully', metadata)
|
logger.custom_info('Model manager workers initialized', metadata)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# This will run the workers and wait for them to complete.
|
# This will run the workers and wait for them to complete.
|
||||||
@@ -153,13 +182,14 @@ async def main():
|
|||||||
logger.custom_error(f'An unhandled exception occurred: {e}', metadata)
|
logger.custom_error(f'An unhandled exception occurred: {e}', metadata)
|
||||||
finally:
|
finally:
|
||||||
notification_handler.shutdown()
|
notification_handler.shutdown()
|
||||||
|
logger.custom_info('MongoDB client closed', metadata)
|
||||||
await activities.shutdown()
|
await activities.shutdown()
|
||||||
# Exit with a non-zero status code to indicate failure to Kubernetes
|
# Exit with a non-zero status code to indicate failure to Kubernetes
|
||||||
metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN
|
metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
def start_prometheus_server():
|
def start_prometheus_server(logger: SientiaLogger, metadata: dict[str, str | None]):
|
||||||
"""
|
"""
|
||||||
Starts the Prometheus metrics server for monitoring and observability.
|
Starts the Prometheus metrics server for monitoring and observability.
|
||||||
|
|
||||||
@@ -179,10 +209,10 @@ def start_prometheus_server():
|
|||||||
try:
|
try:
|
||||||
port = int(os.getenv('HTTP_METRICS_PORT', 9090))
|
port = int(os.getenv('HTTP_METRICS_PORT', 9090))
|
||||||
start_http_server(port)
|
start_http_server(port)
|
||||||
print(f'Prometheus server started on port {port}.')
|
logger.custom_info(f'Prometheus server initialized on port {port}.', metadata)
|
||||||
metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP
|
metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP
|
||||||
except Exception as e: # noqa: BLE001
|
except Exception as e: # noqa: BLE001
|
||||||
print(f'Failed to start Prometheus server: {e}')
|
logger.custom_critical(f'Failed to start Prometheus server: {e}', metadata)
|
||||||
os._exit(1)
|
os._exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
83
model_manager/workflows/cleanup_files.py
Normal file
83
model_manager/workflows/cleanup_files.py
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
"""
|
||||||
|
Cleanup workflow for removing stale files from MinIO and local filesystem.
|
||||||
|
|
||||||
|
This module provides a Temporal cron workflow that runs daily to clean up
|
||||||
|
temporary files and directories older than the configured retention period.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from temporalio import workflow
|
||||||
|
|
||||||
|
with workflow.unsafe.imports_passed_through():
|
||||||
|
import os
|
||||||
|
from datetime import timedelta
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from model_manager.activities.activities import Activities
|
||||||
|
from model_manager.workflows.train_model import POD_ID, network_retry_policy, no_retry_policy
|
||||||
|
|
||||||
|
TIMEOUT_CLEANUP_MINIO = int(os.getenv('TIMEOUT_CLEANUP_MINIO', '300'))
|
||||||
|
TIMEOUT_CLEANUP_LOCAL = int(os.getenv('TIMEOUT_CLEANUP_LOCAL', '120'))
|
||||||
|
DEFAULT_CLEANUP_BUCKET = os.getenv('DEFAULT_CLEANUP_BUCKET', 'model-training')
|
||||||
|
|
||||||
|
|
||||||
|
@workflow.defn(name='cleanup_files')
|
||||||
|
class CleanupFiles:
|
||||||
|
"""
|
||||||
|
Cleanup workflow for removing stale files.
|
||||||
|
|
||||||
|
This workflow cleans up:
|
||||||
|
- MinIO files with timestamp prefixes
|
||||||
|
- Local temporary directories with timestamp suffixes
|
||||||
|
|
||||||
|
The workflow is designed to be simple and robust, with error handling
|
||||||
|
delegated to the individual activities.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@workflow.run
|
||||||
|
async def run(self, input_data: dict[str, Any]) -> None:
|
||||||
|
"""
|
||||||
|
Execute the cleanup workflow.
|
||||||
|
|
||||||
|
This method orchestrates the cleanup of MinIO files and local directories
|
||||||
|
in sequence. No exception handling is needed as activities handle their
|
||||||
|
own errors and notifications.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
input_data: Workflow configuration containing optional:
|
||||||
|
- bucket_name (str): Bucket to clean (defaults to environment variable)
|
||||||
|
"""
|
||||||
|
# Get bucket name from input or environment
|
||||||
|
bucket_name = input_data.get('bucket_name', DEFAULT_CLEANUP_BUCKET)
|
||||||
|
|
||||||
|
# Default temp path for local cleanup
|
||||||
|
temp_path = 'model_manager/reports/temp'
|
||||||
|
|
||||||
|
# Metadata for tracking
|
||||||
|
metadata = {
|
||||||
|
'metadata': {
|
||||||
|
'pod_id': POD_ID,
|
||||||
|
'workflow_name': 'cleanup_files',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Execute MinIO cleanup
|
||||||
|
await workflow.execute_activity_method(
|
||||||
|
Activities.cleanup_minio_files,
|
||||||
|
{
|
||||||
|
**metadata,
|
||||||
|
'bucket_name': bucket_name,
|
||||||
|
},
|
||||||
|
retry_policy=network_retry_policy,
|
||||||
|
start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_MINIO),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Execute local directory cleanup
|
||||||
|
await workflow.execute_activity_method(
|
||||||
|
Activities.cleanup_temp_directories,
|
||||||
|
{
|
||||||
|
**metadata,
|
||||||
|
'temp_path': temp_path,
|
||||||
|
},
|
||||||
|
retry_policy=no_retry_policy,
|
||||||
|
start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_LOCAL),
|
||||||
|
)
|
||||||
@@ -3,7 +3,7 @@ psycopg2-binary==2.9.11
|
|||||||
sqlalchemy==2.0.44
|
sqlalchemy==2.0.44
|
||||||
boto3==1.40.55
|
boto3==1.40.55
|
||||||
botocore==1.40.55
|
botocore==1.40.55
|
||||||
git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.5.2
|
git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.6.1
|
||||||
prometheus-client==0.23.1
|
prometheus-client==0.23.1
|
||||||
mlflow==2.10.1
|
mlflow==2.10.1
|
||||||
evidently==0.4.21
|
evidently==0.4.21
|
||||||
|
|||||||
12
run_local.sh
12
run_local.sh
@@ -3,19 +3,13 @@
|
|||||||
# Exit on any error
|
# Exit on any error
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
#echo "Activating virtual environment..."
|
|
||||||
|
|
||||||
#conda activate ./venv
|
|
||||||
|
|
||||||
echo "Loading environment variables from .env..."
|
|
||||||
|
|
||||||
if [ -f .env ]; then
|
if [ -f .env ]; then
|
||||||
export $(cat .env | grep -v '^#' | xargs)
|
set -a
|
||||||
|
source <(cat .env | grep -v '^#' | grep -v '^$')
|
||||||
|
set +a
|
||||||
echo "Environment variables loaded from .env"
|
echo "Environment variables loaded from .env"
|
||||||
else
|
else
|
||||||
echo "Warning: .env file not found. Continuing without environment variables."
|
echo "Warning: .env file not found. Continuing without environment variables."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "Starting ingestor application..."
|
|
||||||
|
|
||||||
python -m model_manager.worker.worker
|
python -m model_manager.worker.worker
|
||||||
|
|||||||
88
scripts/run_cleanup_test.py
Normal file
88
scripts/run_cleanup_test.py
Normal file
@@ -0,0 +1,88 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Run cleanup_files workflow once for manual testing.
|
||||||
|
|
||||||
|
This script starts the Temporal workflow `cleanup_files` a single time,
|
||||||
|
using the same Temporal namespace and task queue as the main worker.
|
||||||
|
|
||||||
|
It is intended only for local/manual testing; scheduling (cron) must be
|
||||||
|
configured separately in Temporal.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from datetime import timedelta
|
||||||
|
from typing import Any
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from pathlib import Path
|
||||||
|
from temporalio.client import Client
|
||||||
|
|
||||||
|
# Ensure project root is on PYTHONPATH when running directly
|
||||||
|
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
if ROOT_DIR not in sys.path:
|
||||||
|
sys.path.insert(0, ROOT_DIR)
|
||||||
|
|
||||||
|
from model_manager.workflows.cleanup_files import CleanupFiles
|
||||||
|
|
||||||
|
|
||||||
|
# Carrega variáveis de ambiente do arquivo .env na raiz do projeto
|
||||||
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
ENV_PATH = PROJECT_ROOT / '.env'
|
||||||
|
if ENV_PATH.exists():
|
||||||
|
load_dotenv(dotenv_path=ENV_PATH)
|
||||||
|
|
||||||
|
|
||||||
|
async def main(argv: list[str]) -> None:
|
||||||
|
"""Entry point for manual cleanup workflow execution.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
argv: Command-line arguments (excluding program name).
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Config from environment / defaults
|
||||||
|
temporal_host = os.getenv('TEMPORAL_HOST')
|
||||||
|
temporal_namespace = os.getenv('TEMPORAL_NAMESPACE')
|
||||||
|
task_queue = os.getenv('CLEANUP_TASK_QUEUE')
|
||||||
|
default_bucket = os.getenv('DEFAULT_CLEANUP_BUCKET')
|
||||||
|
use_tls = os.getenv('TEMPORAL_USE_TLS', 'false').lower() == 'true'
|
||||||
|
|
||||||
|
# Optional CLI: bucket name override
|
||||||
|
bucket_name = default_bucket
|
||||||
|
if argv:
|
||||||
|
bucket_name = argv[0]
|
||||||
|
|
||||||
|
print(f"Connecting to Temporal at {temporal_host} (namespace={temporal_namespace})...")
|
||||||
|
client = await Client.connect(
|
||||||
|
target_host=temporal_host,
|
||||||
|
namespace=temporal_namespace,
|
||||||
|
tls=use_tls,
|
||||||
|
)
|
||||||
|
|
||||||
|
input_data: dict[str, Any] = {
|
||||||
|
'bucket_name': bucket_name,
|
||||||
|
}
|
||||||
|
|
||||||
|
workflow_id = f"cleanup-files-manual-{int(asyncio.get_event_loop().time())}"
|
||||||
|
|
||||||
|
print(f"Starting cleanup_files workflow once...\n"
|
||||||
|
f" workflow_id = {workflow_id}\n"
|
||||||
|
f" task_queue = {task_queue}\n"
|
||||||
|
f" bucket_name = {bucket_name}")
|
||||||
|
|
||||||
|
handle = await client.start_workflow(
|
||||||
|
CleanupFiles.run,
|
||||||
|
input_data,
|
||||||
|
id=workflow_id,
|
||||||
|
task_queue=task_queue,
|
||||||
|
run_timeout=timedelta(minutes=10),
|
||||||
|
)
|
||||||
|
|
||||||
|
print("Workflow started, waiting for completion...")
|
||||||
|
await handle.result()
|
||||||
|
print("cleanup_files workflow completed successfully.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__': # pragma: no cover - manual utility script
|
||||||
|
asyncio.run(main(sys.argv[1:]))
|
||||||
@@ -24,29 +24,35 @@ import uuid
|
|||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from dotenv import load_dotenv
|
||||||
import psycopg2
|
import psycopg2
|
||||||
from psycopg2.extras import Json
|
from psycopg2.extras import Json
|
||||||
from temporalio import client
|
from temporalio import client
|
||||||
|
|
||||||
|
|
||||||
|
# Carrega variáveis de ambiente do arquivo .env na raiz do projeto
|
||||||
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
ENV_PATH = PROJECT_ROOT / '.env'
|
||||||
|
if ENV_PATH.exists():
|
||||||
|
load_dotenv(dotenv_path=ENV_PATH)
|
||||||
|
|
||||||
|
|
||||||
DOCS_PATH = Path('docs/test-model-data.csv')
|
DOCS_PATH = Path('docs/test-model-data.csv')
|
||||||
MINIO_ALIAS = os.getenv('MINIO_ALIAS', 'suse')
|
MINIO_ALIAS = 'suse'
|
||||||
MINIO_BUCKET = os.getenv('MINIO_BUCKET', 'model-training')
|
MINIO_BUCKET = 'model-training'
|
||||||
|
|
||||||
POSTGRES_CONFIG = {
|
POSTGRES_CONFIG = {
|
||||||
'host': os.getenv('POSTGRES_HOST', 'localhost'),
|
'host': os.getenv('POSTGRES_HOST'),
|
||||||
'port': os.getenv('POSTGRES_PORT', '55432'),
|
'port': os.getenv('POSTGRES_PORT'),
|
||||||
'user': os.getenv('POSTGRES_USER', 'postgres'),
|
'user': os.getenv('POSTGRES_USER'),
|
||||||
'password': os.getenv(
|
'password': os.getenv('POSTGRES_PASSWORD'),
|
||||||
'POSTGRES_PASSWORD',
|
'dbname': os.getenv('POSTGRES_DBNAME'),
|
||||||
'nFqc81y6kwmr2zuAIx43DhiOosFCVPpeEfTtTWZflkNjB2j1KtEeIANkhFR9mAX3',
|
|
||||||
),
|
|
||||||
'dbname': os.getenv('POSTGRES_DBNAME', 'sientia-core-mlops-bff'),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
TEMPORAL_HOST = os.getenv('TEMPORAL_HOST', 'localhost:37463')
|
TEMPORAL_HOST = os.getenv('TEMPORAL_HOST')
|
||||||
TEMPORAL_NAMESPACE = os.getenv('TEMPORAL_NAMESPACE', 'model-manager')
|
TEMPORAL_NAMESPACE = os.getenv('TEMPORAL_NAMESPACE')
|
||||||
TEMPORAL_TASK_QUEUE = os.getenv('TEMPORAL_TASK_QUEUE', 'train_model-queue')
|
TRAIN_TASK_QUEUE = os.getenv('TRAIN_TASK_QUEUE')
|
||||||
TEMPORAL_WORKFLOW = os.getenv('TEMPORAL_WORKFLOW', 'train_model')
|
TEMPORAL_WORKFLOW = 'train_model'
|
||||||
|
|
||||||
BASE_REQUEST_DATA = {
|
BASE_REQUEST_DATA = {
|
||||||
'experimentName': 'model-manager-test-01',
|
'experimentName': 'model-manager-test-01',
|
||||||
@@ -169,6 +175,7 @@ async def trigger_temporal_workflow(workflow_input: dict) -> str:
|
|||||||
temporal_client = await client.Client.connect(
|
temporal_client = await client.Client.connect(
|
||||||
target_host=TEMPORAL_HOST,
|
target_host=TEMPORAL_HOST,
|
||||||
namespace=TEMPORAL_NAMESPACE,
|
namespace=TEMPORAL_NAMESPACE,
|
||||||
|
tls=os.getenv('TEMPORAL_USE_TLS', False),
|
||||||
)
|
)
|
||||||
|
|
||||||
workflow_id = f'train-model-test-{uuid.uuid4()}'
|
workflow_id = f'train-model-test-{uuid.uuid4()}'
|
||||||
@@ -176,7 +183,7 @@ async def trigger_temporal_workflow(workflow_input: dict) -> str:
|
|||||||
TEMPORAL_WORKFLOW,
|
TEMPORAL_WORKFLOW,
|
||||||
workflow_input,
|
workflow_input,
|
||||||
id=workflow_id,
|
id=workflow_id,
|
||||||
task_queue=TEMPORAL_TASK_QUEUE,
|
task_queue=TRAIN_TASK_QUEUE,
|
||||||
execution_timeout=timedelta(minutes=5),
|
execution_timeout=timedelta(minutes=5),
|
||||||
run_timeout=timedelta(minutes=5),
|
run_timeout=timedelta(minutes=5),
|
||||||
task_timeout=timedelta(minutes=5),
|
task_timeout=timedelta(minutes=5),
|
||||||
|
|||||||
569
tests/activities/test_cleanup.py
Normal file
569
tests/activities/test_cleanup.py
Normal file
@@ -0,0 +1,569 @@
|
|||||||
|
"""Unit tests for the Cleanup activity, ensuring 100% code coverage."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from importlib import reload
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
# Define mocks at the top level to be accessible by all tests
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_logger():
|
||||||
|
"""Fixture for a mock logger."""
|
||||||
|
return MagicMock()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_notification_handler():
|
||||||
|
"""Fixture for a mock notification handler."""
|
||||||
|
return MagicMock()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_metrics_controller():
|
||||||
|
"""Fixture for a mock metrics controller with async methods."""
|
||||||
|
controller = MagicMock()
|
||||||
|
controller.shutdown = AsyncMock()
|
||||||
|
controller.emit = AsyncMock()
|
||||||
|
return controller
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_storage_repository():
|
||||||
|
"""Fixture for a mock storage repository."""
|
||||||
|
repo = MagicMock()
|
||||||
|
repo.delete_file = MagicMock()
|
||||||
|
repo.list_bucket_objects = MagicMock()
|
||||||
|
return repo
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def temp_dir():
|
||||||
|
"""Fixture to create and clean up a temporary directory."""
|
||||||
|
path = tempfile.mkdtemp()
|
||||||
|
yield path
|
||||||
|
shutil.rmtree(path)
|
||||||
|
|
||||||
|
|
||||||
|
# --- Initialization Tests ---
|
||||||
|
|
||||||
|
|
||||||
|
@patch.dict(
|
||||||
|
'model_manager.activities.cleanup.os.environ',
|
||||||
|
{
|
||||||
|
'CLEANUP_RETENTION_HOURS': '24',
|
||||||
|
'CLEANUP_DRY_RUN': 'false',
|
||||||
|
'MAX_KEYS_CLEANUP': '1000',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
def test_cleanup_init_default_values(
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test Cleanup initialization uses default environment values."""
|
||||||
|
import model_manager.activities.cleanup
|
||||||
|
|
||||||
|
reload(model_manager.activities.cleanup)
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert cleanup.retention_hours == 24
|
||||||
|
assert cleanup.dry_run is False
|
||||||
|
assert cleanup.max_keys_cleanup == 1000
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_init_custom_env_values(
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test Cleanup initialization with custom environment values."""
|
||||||
|
with patch.dict(
|
||||||
|
os.environ,
|
||||||
|
{
|
||||||
|
'CLEANUP_RETENTION_HOURS': '48',
|
||||||
|
'CLEANUP_DRY_RUN': 'true',
|
||||||
|
'MAX_KEYS_CLEANUP': '500',
|
||||||
|
},
|
||||||
|
):
|
||||||
|
import model_manager.activities.cleanup
|
||||||
|
|
||||||
|
reload(model_manager.activities.cleanup)
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert cleanup.retention_hours == 48
|
||||||
|
assert cleanup.dry_run is True
|
||||||
|
assert cleanup.max_keys_cleanup == 500
|
||||||
|
|
||||||
|
|
||||||
|
@patch.dict(os.environ, {'CLEANUP_RETENTION_HOURS': 'invalid'})
|
||||||
|
def test_cleanup_init_invalid_env_value_raises_error():
|
||||||
|
"""Test Cleanup module raises ValueError for invalid environment variables on import."""
|
||||||
|
import model_manager.activities.cleanup
|
||||||
|
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
reload(model_manager.activities.cleanup)
|
||||||
|
|
||||||
|
|
||||||
|
# --- MinIO Cleanup Tests ---
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_minio_files_missing_bucket_name(
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test cleanup_minio_files raises ValueError if bucket_name is missing."""
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics = AsyncMock()
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match='bucket_name must be provided'):
|
||||||
|
asyncio.run(cleanup.cleanup_minio_files({'metadata': {}}))
|
||||||
|
|
||||||
|
|
||||||
|
@patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'})
|
||||||
|
def test_cleanup_minio_files_success_with_deletions(
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test successful deletion of old files from MinIO."""
|
||||||
|
import model_manager.activities.cleanup
|
||||||
|
|
||||||
|
reload(model_manager.activities.cleanup)
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics = AsyncMock()
|
||||||
|
|
||||||
|
old_ts = int((datetime.now(UTC) - timedelta(hours=48)).timestamp() * 1000)
|
||||||
|
recent_ts = int((datetime.now(UTC) - timedelta(hours=1)).timestamp() * 1000)
|
||||||
|
|
||||||
|
mock_storage_repository.list_bucket_objects.return_value = [
|
||||||
|
f'{old_ts}-old-file.txt',
|
||||||
|
f'{recent_ts}-recent-file.txt',
|
||||||
|
'no-timestamp-file.txt',
|
||||||
|
]
|
||||||
|
|
||||||
|
asyncio.run(cleanup.cleanup_minio_files({'bucket_name': 'test-bucket', 'metadata': {}}))
|
||||||
|
|
||||||
|
mock_storage_repository.delete_file.assert_called_once_with(
|
||||||
|
'test-bucket', f'{old_ts}-old-file.txt'
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_minio_files_dry_run(
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test MinIO cleanup in dry_run mode does not delete files."""
|
||||||
|
with patch.dict(os.environ, {'CLEANUP_DRY_RUN': 'true'}):
|
||||||
|
import model_manager.activities.cleanup
|
||||||
|
|
||||||
|
reload(model_manager.activities.cleanup)
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics = AsyncMock()
|
||||||
|
|
||||||
|
old_ts = int((datetime.now(UTC) - timedelta(hours=48)).timestamp() * 1000)
|
||||||
|
mock_storage_repository.list_bucket_objects.return_value = [f'{old_ts}-old-file.txt']
|
||||||
|
|
||||||
|
asyncio.run(cleanup.cleanup_minio_files({'bucket_name': 'test-bucket', 'metadata': {}}))
|
||||||
|
|
||||||
|
mock_storage_repository.delete_file.assert_not_called()
|
||||||
|
cleanup._emit_metrics.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
@patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'})
|
||||||
|
def test_cleanup_minio_files_delete_error(
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test error during MinIO file deletion is handled gracefully."""
|
||||||
|
import model_manager.activities.cleanup
|
||||||
|
|
||||||
|
reload(model_manager.activities.cleanup)
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics = AsyncMock()
|
||||||
|
cleanup.error = MagicMock()
|
||||||
|
|
||||||
|
old_ts = int((datetime.now(UTC) - timedelta(hours=48)).timestamp() * 1000)
|
||||||
|
mock_storage_repository.list_bucket_objects.return_value = [f'{old_ts}-old-file.txt']
|
||||||
|
mock_storage_repository.delete_file.side_effect = OSError('Permission Denied')
|
||||||
|
|
||||||
|
asyncio.run(cleanup.cleanup_minio_files({'bucket_name': 'test-bucket', 'metadata': {}}))
|
||||||
|
|
||||||
|
cleanup.error.assert_called_once()
|
||||||
|
cleanup._emit_metrics.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_minio_files_exception_handling(
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test exception during MinIO cleanup triggers notification and metrics."""
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup.send_notification = MagicMock()
|
||||||
|
cleanup._emit_metrics = AsyncMock()
|
||||||
|
|
||||||
|
mock_storage_repository.list_bucket_objects.side_effect = Exception('Connection Error')
|
||||||
|
|
||||||
|
with pytest.raises(Exception, match='Connection Error'):
|
||||||
|
asyncio.run(cleanup.cleanup_minio_files({'bucket_name': 'test-bucket', 'metadata': {}}))
|
||||||
|
|
||||||
|
cleanup.send_notification.assert_called_once()
|
||||||
|
cleanup._emit_metrics.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
# --- Temp Directory Cleanup Tests ---
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_temp_directories_nonexistent_path(
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test temp directory cleanup with a non-existent path."""
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics = AsyncMock()
|
||||||
|
cleanup.warning = MagicMock()
|
||||||
|
|
||||||
|
asyncio.run(
|
||||||
|
cleanup.cleanup_temp_directories({'temp_path': '/nonexistent/path', 'metadata': {}})
|
||||||
|
)
|
||||||
|
|
||||||
|
cleanup.warning.assert_called_once()
|
||||||
|
cleanup._emit_metrics.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
@patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'})
|
||||||
|
def test_cleanup_temp_directories_success_with_deletions(
|
||||||
|
temp_dir,
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test successful deletion of old temporary directories."""
|
||||||
|
import model_manager.activities.cleanup
|
||||||
|
|
||||||
|
reload(model_manager.activities.cleanup)
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics = AsyncMock()
|
||||||
|
|
||||||
|
old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000')
|
||||||
|
old_dir = os.path.join(temp_dir, f'old_dir_{old_time}')
|
||||||
|
os.makedirs(old_dir)
|
||||||
|
|
||||||
|
recent_time = (datetime.now() - timedelta(hours=1)).strftime('%Y%m%d_%H%M%S_000000')
|
||||||
|
recent_dir = os.path.join(temp_dir, f'recent_dir_{recent_time}')
|
||||||
|
os.makedirs(recent_dir)
|
||||||
|
|
||||||
|
asyncio.run(cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}))
|
||||||
|
|
||||||
|
assert not os.path.exists(old_dir)
|
||||||
|
assert os.path.exists(recent_dir)
|
||||||
|
cleanup._emit_metrics.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
@patch.dict(os.environ, {'CLEANUP_DRY_RUN': 'true'})
|
||||||
|
def test_cleanup_temp_directories_dry_run(
|
||||||
|
temp_dir,
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test temp directory cleanup in dry_run mode does not delete."""
|
||||||
|
import model_manager.activities.cleanup
|
||||||
|
|
||||||
|
reload(model_manager.activities.cleanup)
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics = AsyncMock()
|
||||||
|
|
||||||
|
old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000')
|
||||||
|
old_dir = os.path.join(temp_dir, f'old_dir_{old_time}')
|
||||||
|
os.makedirs(old_dir)
|
||||||
|
|
||||||
|
asyncio.run(cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}))
|
||||||
|
|
||||||
|
assert os.path.exists(old_dir)
|
||||||
|
cleanup._emit_metrics.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
@patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'})
|
||||||
|
def test_cleanup_temp_directories_delete_error(
|
||||||
|
temp_dir,
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test graceful handling of errors during directory deletion."""
|
||||||
|
import model_manager.activities.cleanup
|
||||||
|
|
||||||
|
reload(model_manager.activities.cleanup)
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics = AsyncMock()
|
||||||
|
cleanup.error = MagicMock()
|
||||||
|
|
||||||
|
old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000')
|
||||||
|
old_dir = os.path.join(temp_dir, f'old_dir_{old_time}')
|
||||||
|
os.makedirs(old_dir)
|
||||||
|
|
||||||
|
with patch('shutil.rmtree', side_effect=OSError('Permission Denied')):
|
||||||
|
asyncio.run(cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}))
|
||||||
|
|
||||||
|
cleanup.error.assert_called_once()
|
||||||
|
cleanup._emit_metrics.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
# --- Metrics and Utility Tests ---
|
||||||
|
|
||||||
|
|
||||||
|
def test_emit_metrics(
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test that _emit_metrics calls the public emit_metric method."""
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup.emit_metric = AsyncMock()
|
||||||
|
|
||||||
|
asyncio.run(
|
||||||
|
cleanup._emit_metrics(
|
||||||
|
metadata={'pod_id': 'p1', 'workflow_name': 'wf1'},
|
||||||
|
metrics_status='success',
|
||||||
|
activity_name='test_activity',
|
||||||
|
emit_workflow_metric=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert cleanup.emit_metric.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_temp_directories_with_files_and_unmatched_dirs(
|
||||||
|
temp_dir,
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test that files and directories with non-matching names are skipped."""
|
||||||
|
import model_manager.activities.cleanup
|
||||||
|
|
||||||
|
reload(model_manager.activities.cleanup)
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics = AsyncMock()
|
||||||
|
cleanup.debug = MagicMock()
|
||||||
|
|
||||||
|
# Create a file and a directory with a non-matching name
|
||||||
|
with open(os.path.join(temp_dir, 'a_file.txt'), 'w') as f:
|
||||||
|
f.write('hello')
|
||||||
|
os.makedirs(os.path.join(temp_dir, 'a_directory_with_no_timestamp'))
|
||||||
|
|
||||||
|
asyncio.run(cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}))
|
||||||
|
|
||||||
|
# Ensure the debug message for skipping was called for the unmatched directory
|
||||||
|
cleanup.debug.assert_called_with(
|
||||||
|
'Skipping directory without timestamp pattern: a_directory_with_no_timestamp', {}
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_temp_directories_invalid_timestamp_format(
|
||||||
|
temp_dir,
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test that a directory with an invalid timestamp format is handled correctly."""
|
||||||
|
import model_manager.activities.cleanup
|
||||||
|
|
||||||
|
reload(model_manager.activities.cleanup)
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics = AsyncMock()
|
||||||
|
cleanup.error = MagicMock()
|
||||||
|
|
||||||
|
# Create a directory with a malformed timestamp that matches the regex but fails parsing
|
||||||
|
malformed_dir_name = 'dir_20239999_999999_999999'
|
||||||
|
os.makedirs(os.path.join(temp_dir, malformed_dir_name))
|
||||||
|
|
||||||
|
asyncio.run(cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}))
|
||||||
|
|
||||||
|
cleanup.error.assert_called_once()
|
||||||
|
cleanup._emit_metrics.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_temp_directories_generic_exception(
|
||||||
|
temp_dir,
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test that a generic exception during directory cleanup is handled."""
|
||||||
|
import model_manager.activities.cleanup
|
||||||
|
|
||||||
|
reload(model_manager.activities.cleanup)
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup._emit_metrics = AsyncMock()
|
||||||
|
cleanup.send_notification = MagicMock()
|
||||||
|
|
||||||
|
with patch('os.listdir', side_effect=Exception('Unexpected OS Error')):
|
||||||
|
with pytest.raises(Exception, match='Unexpected OS Error'):
|
||||||
|
asyncio.run(cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}))
|
||||||
|
|
||||||
|
cleanup.send_notification.assert_called_once()
|
||||||
|
cleanup._emit_metrics.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_emit_metrics_activity_only(
|
||||||
|
mock_storage_repository,
|
||||||
|
mock_logger,
|
||||||
|
mock_notification_handler,
|
||||||
|
mock_metrics_controller,
|
||||||
|
):
|
||||||
|
"""Test that _emit_metrics can emit only the activity metric."""
|
||||||
|
from model_manager.activities.cleanup import Cleanup
|
||||||
|
|
||||||
|
cleanup = Cleanup(
|
||||||
|
storage_repository=mock_storage_repository,
|
||||||
|
logger=mock_logger,
|
||||||
|
notification_handler=mock_notification_handler,
|
||||||
|
metrics_controller=mock_metrics_controller,
|
||||||
|
)
|
||||||
|
cleanup.emit_metric = AsyncMock()
|
||||||
|
|
||||||
|
asyncio.run(
|
||||||
|
cleanup._emit_metrics(
|
||||||
|
metadata={'pod_id': 'p1', 'workflow_name': 'wf1'},
|
||||||
|
metrics_status='success',
|
||||||
|
activity_name='test_activity',
|
||||||
|
emit_workflow_metric=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
cleanup.emit_metric.assert_called_once()
|
||||||
0
tests/schedules/__init__.py
Normal file
0
tests/schedules/__init__.py
Normal file
362
tests/schedules/test_cleanup_schedule.py
Normal file
362
tests/schedules/test_cleanup_schedule.py
Normal file
@@ -0,0 +1,362 @@
|
|||||||
|
"""Tests for cleanup schedule management."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from datetime import timedelta
|
||||||
|
from importlib import reload
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_temporal_client():
|
||||||
|
"""Fixture for a mock Temporal client."""
|
||||||
|
client = AsyncMock()
|
||||||
|
client.list_schedules = AsyncMock()
|
||||||
|
client.create_schedule = AsyncMock()
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_logger():
|
||||||
|
"""Fixture for a mock Sientia logger."""
|
||||||
|
logger = MagicMock()
|
||||||
|
logger.custom_info = MagicMock()
|
||||||
|
logger.custom_error = MagicMock()
|
||||||
|
return logger
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def metadata():
|
||||||
|
"""Fixture for metadata dict."""
|
||||||
|
return {'pod_id': 'test-pod', 'project_name': 'test-project'}
|
||||||
|
|
||||||
|
|
||||||
|
# --- schedule_exists Tests ---
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_schedule_exists_returns_true_when_schedule_found(
|
||||||
|
mock_temporal_client, mock_logger, metadata
|
||||||
|
):
|
||||||
|
"""Test that schedule_exists returns True when schedule is found."""
|
||||||
|
from model_manager.schedules.cleanup_schedule import schedule_exists
|
||||||
|
|
||||||
|
# Mock schedule list with matching schedule
|
||||||
|
mock_schedule = MagicMock()
|
||||||
|
mock_schedule.id = 'test-schedule-id'
|
||||||
|
|
||||||
|
async def mock_list_schedules():
|
||||||
|
yield mock_schedule
|
||||||
|
|
||||||
|
mock_temporal_client.list_schedules.return_value = mock_list_schedules()
|
||||||
|
|
||||||
|
result = await schedule_exists(mock_temporal_client, 'test-schedule-id', mock_logger, metadata)
|
||||||
|
|
||||||
|
assert result is True
|
||||||
|
mock_temporal_client.list_schedules.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_schedule_exists_returns_false_when_schedule_not_found(
|
||||||
|
mock_temporal_client, mock_logger, metadata
|
||||||
|
):
|
||||||
|
"""Test that schedule_exists returns False when schedule is not found."""
|
||||||
|
from model_manager.schedules.cleanup_schedule import schedule_exists
|
||||||
|
|
||||||
|
# Mock empty schedule list
|
||||||
|
async def mock_list_schedules():
|
||||||
|
return
|
||||||
|
yield # Make it an async generator
|
||||||
|
|
||||||
|
mock_temporal_client.list_schedules.return_value = mock_list_schedules()
|
||||||
|
|
||||||
|
result = await schedule_exists(
|
||||||
|
mock_temporal_client, 'nonexistent-schedule', mock_logger, metadata
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
mock_temporal_client.list_schedules.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_schedule_exists_returns_false_when_different_schedule_found(
|
||||||
|
mock_temporal_client, mock_logger, metadata
|
||||||
|
):
|
||||||
|
"""Test that schedule_exists returns False when only different schedules exist."""
|
||||||
|
from model_manager.schedules.cleanup_schedule import schedule_exists
|
||||||
|
|
||||||
|
# Mock schedule list with non-matching schedule
|
||||||
|
mock_schedule = MagicMock()
|
||||||
|
mock_schedule.id = 'different-schedule-id'
|
||||||
|
|
||||||
|
async def mock_list_schedules():
|
||||||
|
yield mock_schedule
|
||||||
|
|
||||||
|
mock_temporal_client.list_schedules.return_value = mock_list_schedules()
|
||||||
|
|
||||||
|
result = await schedule_exists(mock_temporal_client, 'test-schedule-id', mock_logger, metadata)
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
mock_temporal_client.list_schedules.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_schedule_exists_handles_exception(mock_temporal_client, mock_logger, metadata):
|
||||||
|
"""Test that schedule_exists handles exceptions gracefully."""
|
||||||
|
from model_manager.schedules.cleanup_schedule import schedule_exists
|
||||||
|
|
||||||
|
# Mock list_schedules to raise an exception
|
||||||
|
mock_temporal_client.list_schedules.side_effect = Exception('Connection error')
|
||||||
|
|
||||||
|
result = await schedule_exists(mock_temporal_client, 'test-schedule-id', mock_logger, metadata)
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
mock_logger.custom_error.assert_called_once()
|
||||||
|
assert 'Error checking if schedule exists' in mock_logger.custom_error.call_args[0][0]
|
||||||
|
|
||||||
|
|
||||||
|
# --- create_cleanup_schedule Tests ---
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_create_cleanup_schedule_skips_when_exists(
|
||||||
|
mock_temporal_client, mock_logger, metadata
|
||||||
|
):
|
||||||
|
"""Test that create_cleanup_schedule skips creation when schedule already exists."""
|
||||||
|
import model_manager.schedules.cleanup_schedule
|
||||||
|
|
||||||
|
reload(model_manager.schedules.cleanup_schedule)
|
||||||
|
from model_manager.schedules.cleanup_schedule import create_cleanup_schedule
|
||||||
|
|
||||||
|
# Mock schedule already exists
|
||||||
|
mock_schedule = MagicMock()
|
||||||
|
mock_schedule.id = 'cleanup-files-daily'
|
||||||
|
|
||||||
|
async def mock_list_schedules():
|
||||||
|
yield mock_schedule
|
||||||
|
|
||||||
|
mock_temporal_client.list_schedules.return_value = mock_list_schedules()
|
||||||
|
|
||||||
|
await create_cleanup_schedule(mock_temporal_client, mock_logger, metadata)
|
||||||
|
|
||||||
|
# Verify schedule creation was NOT called
|
||||||
|
mock_temporal_client.create_schedule.assert_not_called()
|
||||||
|
|
||||||
|
# Verify info log was called
|
||||||
|
mock_logger.custom_info.assert_called_once()
|
||||||
|
assert 'already configured' in mock_logger.custom_info.call_args[0][0]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@patch.dict(
|
||||||
|
'model_manager.schedules.cleanup_schedule.os.environ',
|
||||||
|
{
|
||||||
|
'CLEANUP_SCHEDULE_ID': 'test-cleanup-schedule',
|
||||||
|
'CLEANUP_CRON': '0 2 * * *',
|
||||||
|
'CLEANUP_TIMEZONE': 'America/Sao_Paulo',
|
||||||
|
'CLEANUP_TASK_QUEUE': 'test-cleanup-queue',
|
||||||
|
'CLEANUP_EXECUTION_TIMEOUT_HOURS': '2',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
async def test_create_cleanup_schedule_creates_with_custom_config(
|
||||||
|
mock_temporal_client, mock_logger, metadata
|
||||||
|
):
|
||||||
|
"""Test that create_cleanup_schedule creates schedule with custom configuration."""
|
||||||
|
import model_manager.schedules.cleanup_schedule
|
||||||
|
|
||||||
|
reload(model_manager.schedules.cleanup_schedule)
|
||||||
|
from model_manager.schedules.cleanup_schedule import create_cleanup_schedule
|
||||||
|
|
||||||
|
# Mock schedule does not exist (empty list)
|
||||||
|
async def mock_list_schedules():
|
||||||
|
return
|
||||||
|
yield # Make it an async generator
|
||||||
|
|
||||||
|
mock_temporal_client.list_schedules.return_value = mock_list_schedules()
|
||||||
|
|
||||||
|
await create_cleanup_schedule(mock_temporal_client, mock_logger, metadata)
|
||||||
|
|
||||||
|
# Verify schedule creation was called
|
||||||
|
mock_temporal_client.create_schedule.assert_called_once()
|
||||||
|
|
||||||
|
# Verify schedule parameters
|
||||||
|
call_args = mock_temporal_client.create_schedule.call_args
|
||||||
|
schedule_id = call_args[0][0]
|
||||||
|
schedule_obj = call_args[0][1]
|
||||||
|
|
||||||
|
assert schedule_id == 'test-cleanup-schedule'
|
||||||
|
assert schedule_obj.action.workflow == 'cleanup_files'
|
||||||
|
assert schedule_obj.action.task_queue == 'test-cleanup-queue'
|
||||||
|
assert schedule_obj.action.execution_timeout == timedelta(hours=2)
|
||||||
|
assert schedule_obj.spec.cron_expressions == ['0 2 * * *']
|
||||||
|
assert schedule_obj.spec.time_zone_name == 'America/Sao_Paulo'
|
||||||
|
|
||||||
|
# Verify success log was called
|
||||||
|
assert mock_logger.custom_info.call_count == 1
|
||||||
|
assert 'created successfully' in mock_logger.custom_info.call_args[0][0]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@patch.dict(
|
||||||
|
'model_manager.schedules.cleanup_schedule.os.environ',
|
||||||
|
{
|
||||||
|
'CLEANUP_SCHEDULE_ID': 'default-schedule',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
async def test_create_cleanup_schedule_uses_defaults(mock_temporal_client, mock_logger, metadata):
|
||||||
|
"""Test that create_cleanup_schedule uses default values when env vars not set."""
|
||||||
|
import model_manager.schedules.cleanup_schedule
|
||||||
|
|
||||||
|
# Remove optional env vars to test defaults
|
||||||
|
for key in [
|
||||||
|
'CLEANUP_CRON',
|
||||||
|
'CLEANUP_TIMEZONE',
|
||||||
|
'CLEANUP_TASK_QUEUE',
|
||||||
|
'CLEANUP_EXECUTION_TIMEOUT_HOURS',
|
||||||
|
]:
|
||||||
|
os.environ.pop(key, None)
|
||||||
|
|
||||||
|
reload(model_manager.schedules.cleanup_schedule)
|
||||||
|
from model_manager.schedules.cleanup_schedule import create_cleanup_schedule
|
||||||
|
|
||||||
|
# Mock schedule does not exist (empty list)
|
||||||
|
async def mock_list_schedules():
|
||||||
|
return
|
||||||
|
yield # Make it an async generator
|
||||||
|
|
||||||
|
mock_temporal_client.list_schedules.return_value = mock_list_schedules()
|
||||||
|
|
||||||
|
await create_cleanup_schedule(mock_temporal_client, mock_logger, metadata)
|
||||||
|
|
||||||
|
# Verify schedule creation was called
|
||||||
|
mock_temporal_client.create_schedule.assert_called_once()
|
||||||
|
|
||||||
|
# Verify default parameters
|
||||||
|
call_args = mock_temporal_client.create_schedule.call_args
|
||||||
|
schedule_obj = call_args[0][1]
|
||||||
|
|
||||||
|
assert schedule_obj.spec.cron_expressions == ['0 0 * * *'] # Default midnight
|
||||||
|
assert schedule_obj.spec.time_zone_name == 'UTC' # Default UTC
|
||||||
|
assert schedule_obj.action.task_queue == 'cleanup-queue' # Default queue
|
||||||
|
assert schedule_obj.action.execution_timeout == timedelta(hours=1) # Default 1 hour
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_create_cleanup_schedule_workflow_id_format(
|
||||||
|
mock_temporal_client, mock_logger, metadata
|
||||||
|
):
|
||||||
|
"""Test that workflow ID is correctly formatted with schedule ID."""
|
||||||
|
import model_manager.schedules.cleanup_schedule
|
||||||
|
|
||||||
|
reload(model_manager.schedules.cleanup_schedule)
|
||||||
|
from model_manager.schedules.cleanup_schedule import (
|
||||||
|
SCHEDULE_ID,
|
||||||
|
create_cleanup_schedule,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Mock schedule does not exist (empty list)
|
||||||
|
async def mock_list_schedules():
|
||||||
|
return
|
||||||
|
yield # Make it an async generator
|
||||||
|
|
||||||
|
mock_temporal_client.list_schedules.return_value = mock_list_schedules()
|
||||||
|
|
||||||
|
await create_cleanup_schedule(mock_temporal_client, mock_logger, metadata)
|
||||||
|
|
||||||
|
# Verify workflow ID format
|
||||||
|
call_args = mock_temporal_client.create_schedule.call_args
|
||||||
|
schedule_obj = call_args[0][1]
|
||||||
|
|
||||||
|
expected_workflow_id = f'cleanup-files-scheduled-{SCHEDULE_ID}'
|
||||||
|
assert schedule_obj.action.id == expected_workflow_id
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_create_cleanup_schedule_empty_workflow_args(
|
||||||
|
mock_temporal_client, mock_logger, metadata
|
||||||
|
):
|
||||||
|
"""Test that workflow is created with empty args (uses env defaults)."""
|
||||||
|
import model_manager.schedules.cleanup_schedule
|
||||||
|
|
||||||
|
reload(model_manager.schedules.cleanup_schedule)
|
||||||
|
from model_manager.schedules.cleanup_schedule import create_cleanup_schedule
|
||||||
|
|
||||||
|
# Mock schedule does not exist (empty list)
|
||||||
|
async def mock_list_schedules():
|
||||||
|
return
|
||||||
|
yield # Make it an async generator
|
||||||
|
|
||||||
|
mock_temporal_client.list_schedules.return_value = mock_list_schedules()
|
||||||
|
|
||||||
|
await create_cleanup_schedule(mock_temporal_client, mock_logger, metadata)
|
||||||
|
|
||||||
|
# Verify workflow args are empty (it's a list with one empty dict)
|
||||||
|
call_args = mock_temporal_client.create_schedule.call_args
|
||||||
|
schedule_obj = call_args[0][1]
|
||||||
|
|
||||||
|
# The args are passed as positional args, so it's a list with one element
|
||||||
|
assert schedule_obj.action.args == [{}]
|
||||||
|
|
||||||
|
|
||||||
|
# --- Environment Variable Configuration Tests ---
|
||||||
|
|
||||||
|
|
||||||
|
@patch.dict(
|
||||||
|
'model_manager.schedules.cleanup_schedule.os.environ',
|
||||||
|
{
|
||||||
|
'CLEANUP_SCHEDULE_ID': 'custom-id',
|
||||||
|
'CLEANUP_CRON': '30 3 * * 1',
|
||||||
|
'CLEANUP_TIMEZONE': 'Europe/London',
|
||||||
|
'CLEANUP_TASK_QUEUE': 'custom-queue',
|
||||||
|
'CLEANUP_EXECUTION_TIMEOUT_HOURS': '3',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
def test_environment_variables_loaded_correctly():
|
||||||
|
"""Test that environment variables are loaded correctly."""
|
||||||
|
import model_manager.schedules.cleanup_schedule
|
||||||
|
|
||||||
|
reload(model_manager.schedules.cleanup_schedule)
|
||||||
|
from model_manager.schedules.cleanup_schedule import (
|
||||||
|
CLEANUP_CRON,
|
||||||
|
CLEANUP_EXECUTION_TIMEOUT_HOURS,
|
||||||
|
CLEANUP_TASK_QUEUE,
|
||||||
|
CLEANUP_TIMEZONE,
|
||||||
|
SCHEDULE_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert SCHEDULE_ID == 'custom-id'
|
||||||
|
assert CLEANUP_CRON == '30 3 * * 1'
|
||||||
|
assert CLEANUP_TIMEZONE == 'Europe/London'
|
||||||
|
assert CLEANUP_TASK_QUEUE == 'custom-queue'
|
||||||
|
assert CLEANUP_EXECUTION_TIMEOUT_HOURS == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_environment_variables_use_defaults_when_not_set():
|
||||||
|
"""Test that default values are used when environment variables are not set."""
|
||||||
|
import model_manager.schedules.cleanup_schedule
|
||||||
|
|
||||||
|
# Remove all env vars
|
||||||
|
for key in [
|
||||||
|
'CLEANUP_SCHEDULE_ID',
|
||||||
|
'CLEANUP_CRON',
|
||||||
|
'CLEANUP_TIMEZONE',
|
||||||
|
'CLEANUP_TASK_QUEUE',
|
||||||
|
'CLEANUP_EXECUTION_TIMEOUT_HOURS',
|
||||||
|
]:
|
||||||
|
os.environ.pop(key, None)
|
||||||
|
|
||||||
|
reload(model_manager.schedules.cleanup_schedule)
|
||||||
|
from model_manager.schedules.cleanup_schedule import (
|
||||||
|
CLEANUP_CRON,
|
||||||
|
CLEANUP_EXECUTION_TIMEOUT_HOURS,
|
||||||
|
CLEANUP_TASK_QUEUE,
|
||||||
|
CLEANUP_TIMEZONE,
|
||||||
|
SCHEDULE_ID,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert SCHEDULE_ID == 'cleanup-files-daily'
|
||||||
|
assert CLEANUP_CRON == '0 0 * * *'
|
||||||
|
assert CLEANUP_TIMEZONE == 'UTC'
|
||||||
|
assert CLEANUP_TASK_QUEUE == 'cleanup-queue'
|
||||||
|
assert CLEANUP_EXECUTION_TIMEOUT_HOURS == 1
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
"""Unit tests for ModelRepository with 100% coverage."""
|
"""Unit tests for ModelRepository with 100% coverage."""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import shutil
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -8,6 +9,29 @@ import pandas as pd
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def cleanup_temp_directories():
|
||||||
|
"""Clean up temporary directories after each test."""
|
||||||
|
# Get the temp directory path
|
||||||
|
current_file_dir = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
model_manager_dir = os.path.dirname(os.path.dirname(os.path.dirname(current_file_dir)))
|
||||||
|
temp_dir = os.path.join(model_manager_dir, 'reports', 'temp')
|
||||||
|
|
||||||
|
# Run the test
|
||||||
|
yield
|
||||||
|
|
||||||
|
# Clean up after test
|
||||||
|
if os.path.exists(temp_dir):
|
||||||
|
for item in os.listdir(temp_dir):
|
||||||
|
item_path = os.path.join(temp_dir, item)
|
||||||
|
if os.path.isdir(item_path) and item.startswith('test_run_'):
|
||||||
|
try:
|
||||||
|
shutil.rmtree(item_path)
|
||||||
|
except (OSError, PermissionError):
|
||||||
|
# Ignore cleanup errors
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def mock_logger():
|
def mock_logger():
|
||||||
"""Create a mock logger."""
|
"""Create a mock logger."""
|
||||||
@@ -79,6 +103,9 @@ def test_save_model_success(mock_model_serving_class, mock_logger, mock_train_re
|
|||||||
url='http://mlflow.test', username='user', password='pass', logger=mock_logger
|
url='http://mlflow.test', username='user', password='pass', logger=mock_logger
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Reset mock after initialization to focus on method-specific calls
|
||||||
|
mock_logger.reset_mock()
|
||||||
|
|
||||||
repo._get_next_run_name = MagicMock(return_value='test_experiment-1')
|
repo._get_next_run_name = MagicMock(return_value='test_experiment-1')
|
||||||
repo._generate_artifacts = MagicMock(return_value=mock_train_result)
|
repo._generate_artifacts = MagicMock(return_value=mock_train_result)
|
||||||
repo._save_run = MagicMock()
|
repo._save_run = MagicMock()
|
||||||
@@ -105,6 +132,9 @@ def test_cleanup_run_directory_exists(
|
|||||||
url='http://mlflow.test', username='user', password='pass', logger=mock_logger
|
url='http://mlflow.test', username='user', password='pass', logger=mock_logger
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Reset mock after initialization to focus on method-specific calls
|
||||||
|
mock_logger.reset_mock()
|
||||||
|
|
||||||
mock_exists.return_value = True
|
mock_exists.return_value = True
|
||||||
|
|
||||||
repo.cleanup_run_directory('/tmp/test_run') # noqa: S108
|
repo.cleanup_run_directory('/tmp/test_run') # noqa: S108
|
||||||
@@ -124,6 +154,9 @@ def test_cleanup_run_directory_not_exists(mock_exists, mock_model_serving_class,
|
|||||||
url='http://mlflow.test', username='user', password='pass', logger=mock_logger
|
url='http://mlflow.test', username='user', password='pass', logger=mock_logger
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Reset mock after initialization to focus on method-specific calls
|
||||||
|
mock_logger.reset_mock()
|
||||||
|
|
||||||
mock_exists.return_value = False
|
mock_exists.return_value = False
|
||||||
|
|
||||||
repo.cleanup_run_directory('/tmp/test_run') # noqa: S108
|
repo.cleanup_run_directory('/tmp/test_run') # noqa: S108
|
||||||
@@ -141,6 +174,9 @@ def test_cleanup_run_directory_empty_path(mock_model_serving_class, mock_logger)
|
|||||||
url='http://mlflow.test', username='user', password='pass', logger=mock_logger
|
url='http://mlflow.test', username='user', password='pass', logger=mock_logger
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Reset mock after initialization to focus on method-specific calls
|
||||||
|
mock_logger.reset_mock()
|
||||||
|
|
||||||
repo.cleanup_run_directory('')
|
repo.cleanup_run_directory('')
|
||||||
|
|
||||||
mock_logger.info.assert_called_once_with('No run directory specified, skipping cleanup')
|
mock_logger.info.assert_called_once_with('No run directory specified, skipping cleanup')
|
||||||
@@ -756,9 +792,14 @@ def test_generate_artifacts_no_run_name(
|
|||||||
)
|
)
|
||||||
|
|
||||||
mock_train_result.run_name = None
|
mock_train_result.run_name = None
|
||||||
|
mock_exists.return_value = True # Mock reports directory exists
|
||||||
|
|
||||||
with pytest.raises(ValueError, match='run_name must be set'):
|
# Mock _create_run_directory to avoid creating real directories
|
||||||
repo._generate_artifacts(mock_train_result)
|
with patch.object(repo, '_create_run_directory') as mock_create_dir:
|
||||||
|
mock_create_dir.return_value = '/mock/run/dir'
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match='run_name must be set'):
|
||||||
|
repo._generate_artifacts(mock_train_result)
|
||||||
|
|
||||||
|
|
||||||
@patch('model_manager.utils.repository.model_repository.ModelServing')
|
@patch('model_manager.utils.repository.model_repository.ModelServing')
|
||||||
@@ -800,8 +841,12 @@ def test_generate_artifacts_header_not_found(
|
|||||||
|
|
||||||
mock_exists.side_effect = exists_side_effect
|
mock_exists.side_effect = exists_side_effect
|
||||||
|
|
||||||
with pytest.raises(FileNotFoundError, match='Header file does not exist'):
|
# Mock _create_run_directory to avoid creating real directories
|
||||||
repo._generate_artifacts(mock_train_result)
|
with patch.object(repo, '_create_run_directory') as mock_create_dir:
|
||||||
|
mock_create_dir.return_value = '/mock/run/dir'
|
||||||
|
|
||||||
|
with pytest.raises(FileNotFoundError, match='Header file does not exist'):
|
||||||
|
repo._generate_artifacts(mock_train_result)
|
||||||
|
|
||||||
|
|
||||||
@patch('model_manager.utils.repository.model_repository.ModelServing')
|
@patch('model_manager.utils.repository.model_repository.ModelServing')
|
||||||
|
|||||||
@@ -462,3 +462,52 @@ def test_fetch_file_logs_file_size(mock_boto3, mock_logger, storage_config):
|
|||||||
# Verify logging includes file size
|
# Verify logging includes file size
|
||||||
log_calls = [str(call) for call in mock_logger.info.call_args_list]
|
log_calls = [str(call) for call in mock_logger.info.call_args_list]
|
||||||
assert any('12345 bytes' in str(call) for call in log_calls)
|
assert any('12345 bytes' in str(call) for call in log_calls)
|
||||||
|
|
||||||
|
|
||||||
|
@patch('model_manager.utils.repository.storage_repository.boto3')
|
||||||
|
def test_close_method(mock_boto3, mock_logger, storage_config):
|
||||||
|
"""Test that the close method calls the underlying client's close method."""
|
||||||
|
from model_manager.utils.repository.storage_repository import StorageRepository
|
||||||
|
|
||||||
|
mock_s3_client = Mock()
|
||||||
|
mock_boto3.client.return_value = mock_s3_client
|
||||||
|
|
||||||
|
repo = StorageRepository(logger=mock_logger, **storage_config)
|
||||||
|
repo.close()
|
||||||
|
|
||||||
|
mock_s3_client.close.assert_called_once()
|
||||||
|
mock_logger.info.assert_called_with('MinIO client closed')
|
||||||
|
|
||||||
|
|
||||||
|
@patch('model_manager.utils.repository.storage_repository.boto3')
|
||||||
|
def test_list_bucket_objects_with_pagination(mock_boto3, mock_logger, storage_config):
|
||||||
|
"""Test list_bucket_objects with a paginated response."""
|
||||||
|
from model_manager.utils.repository.storage_repository import StorageRepository
|
||||||
|
|
||||||
|
mock_s3_client = Mock()
|
||||||
|
mock_paginator = Mock()
|
||||||
|
page1 = {
|
||||||
|
'Contents': [
|
||||||
|
{'Key': 'file1.txt'},
|
||||||
|
{'Key': 'file2.txt'},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
page2 = {
|
||||||
|
'Contents': [
|
||||||
|
{'Key': 'file3.txt'},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
page3 = {}
|
||||||
|
|
||||||
|
mock_paginator.paginate.return_value = [page1, page2, page3]
|
||||||
|
mock_s3_client.get_paginator.return_value = mock_paginator
|
||||||
|
mock_boto3.client.return_value = mock_s3_client
|
||||||
|
|
||||||
|
repo = StorageRepository(logger=mock_logger, **storage_config)
|
||||||
|
objects = repo.list_bucket_objects('test-bucket', max_keys=2)
|
||||||
|
|
||||||
|
assert objects == ['file1.txt', 'file2.txt', 'file3.txt']
|
||||||
|
assert len(objects) == 3
|
||||||
|
mock_s3_client.get_paginator.assert_called_once_with('list_objects_v2')
|
||||||
|
mock_paginator.paginate.assert_called_once_with(Bucket='test-bucket', MaxKeys=2)
|
||||||
|
mock_logger.info.assert_any_call('Listed 3 objects from bucket test-bucket')
|
||||||
|
|||||||
@@ -96,6 +96,7 @@ def test_build_mongo_db_config_with_env_vars():
|
|||||||
'connection_string': 'mongodb://sientia1:sientia1@localhost:27018',
|
'connection_string': 'mongodb://sientia1:sientia1@localhost:27018',
|
||||||
'database_name': 'test_db',
|
'database_name': 'test_db',
|
||||||
'ttl_index_seconds': 3600,
|
'ttl_index_seconds': 3600,
|
||||||
|
'uri': 'localhost:27018',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -109,6 +110,7 @@ def test_build_mongo_db_config_with_defaults():
|
|||||||
'connection_string': 'mongodb://root:wKZDbMNU1c@localhost:27018',
|
'connection_string': 'mongodb://root:wKZDbMNU1c@localhost:27018',
|
||||||
'database_name': 'sientia',
|
'database_name': 'sientia',
|
||||||
'ttl_index_seconds': 3600,
|
'ttl_index_seconds': 3600,
|
||||||
|
'uri': 'localhost:27018',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ def mock_env_vars():
|
|||||||
'TEMPORAL_HOST': 'localhost:7233',
|
'TEMPORAL_HOST': 'localhost:7233',
|
||||||
'TEMPORAL_NAMESPACE': 'test-namespace',
|
'TEMPORAL_NAMESPACE': 'test-namespace',
|
||||||
'PROJECT_NAME': 'test-project',
|
'PROJECT_NAME': 'test-project',
|
||||||
|
'TRAIN_TASK_QUEUE': 'train_model-local_queue',
|
||||||
|
'CLEANUP_TASK_QUEUE': 'cleanup-local_queue',
|
||||||
}
|
}
|
||||||
|
|
||||||
with patch.dict(os.environ, env_vars, clear=False):
|
with patch.dict(os.environ, env_vars, clear=False):
|
||||||
@@ -109,14 +111,18 @@ def test_sdk_metrics_port_from_env():
|
|||||||
@patch('model_manager.worker.worker.POD_ID', 'test-pod-123')
|
@patch('model_manager.worker.worker.POD_ID', 'test-pod-123')
|
||||||
@patch('model_manager.worker.worker.start_http_server')
|
@patch('model_manager.worker.worker.start_http_server')
|
||||||
@patch('model_manager.worker.worker.metrics')
|
@patch('model_manager.worker.worker.metrics')
|
||||||
def test_start_prometheus_server_success(mock_metrics, mock_start_http_server, mock_env_vars):
|
def test_start_prometheus_server_success(
|
||||||
|
mock_metrics, mock_start_http_server, mock_env_vars, mock_logger
|
||||||
|
):
|
||||||
"""Test successful Prometheus server startup."""
|
"""Test successful Prometheus server startup."""
|
||||||
from model_manager.worker.worker import start_prometheus_server
|
from model_manager.worker.worker import start_prometheus_server
|
||||||
|
|
||||||
mock_app_up = Mock()
|
mock_app_up = Mock()
|
||||||
mock_metrics.APP_UP.labels.return_value = mock_app_up
|
mock_metrics.APP_UP.labels.return_value = mock_app_up
|
||||||
|
|
||||||
start_prometheus_server()
|
metadata = {'pod_id': 'test-pod-123', 'workflow_name': 'train_model'}
|
||||||
|
|
||||||
|
start_prometheus_server(mock_logger, metadata)
|
||||||
|
|
||||||
# Verify HTTP server started
|
# Verify HTTP server started
|
||||||
mock_start_http_server.assert_called_once_with(9090)
|
mock_start_http_server.assert_called_once_with(9090)
|
||||||
@@ -124,11 +130,12 @@ def test_start_prometheus_server_success(mock_metrics, mock_start_http_server, m
|
|||||||
# Verify APP_UP metric was set to 1
|
# Verify APP_UP metric was set to 1
|
||||||
mock_metrics.APP_UP.labels.assert_called_once_with(pod_id='test-pod-123')
|
mock_metrics.APP_UP.labels.assert_called_once_with(pod_id='test-pod-123')
|
||||||
mock_app_up.set.assert_called_once_with(1)
|
mock_app_up.set.assert_called_once_with(1)
|
||||||
|
mock_logger.custom_info.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
@patch('model_manager.worker.worker.start_http_server')
|
@patch('model_manager.worker.worker.start_http_server')
|
||||||
@patch('model_manager.worker.worker.metrics')
|
@patch('model_manager.worker.worker.metrics')
|
||||||
def test_start_prometheus_server_custom_port(mock_metrics, mock_start_http_server):
|
def test_start_prometheus_server_custom_port(mock_metrics, mock_start_http_server, mock_logger):
|
||||||
"""Test Prometheus server startup with custom port."""
|
"""Test Prometheus server startup with custom port."""
|
||||||
from model_manager.worker.worker import start_prometheus_server
|
from model_manager.worker.worker import start_prometheus_server
|
||||||
|
|
||||||
@@ -136,7 +143,9 @@ def test_start_prometheus_server_custom_port(mock_metrics, mock_start_http_serve
|
|||||||
mock_app_up = Mock()
|
mock_app_up = Mock()
|
||||||
mock_metrics.APP_UP.labels.return_value = mock_app_up
|
mock_metrics.APP_UP.labels.return_value = mock_app_up
|
||||||
|
|
||||||
start_prometheus_server()
|
metadata = {'pod_id': 'custom-pod', 'workflow_name': 'train_model'}
|
||||||
|
|
||||||
|
start_prometheus_server(mock_logger, metadata)
|
||||||
|
|
||||||
mock_start_http_server.assert_called_once_with(8080)
|
mock_start_http_server.assert_called_once_with(8080)
|
||||||
|
|
||||||
@@ -145,17 +154,20 @@ def test_start_prometheus_server_custom_port(mock_metrics, mock_start_http_serve
|
|||||||
@patch('model_manager.worker.worker.metrics')
|
@patch('model_manager.worker.worker.metrics')
|
||||||
@patch('model_manager.worker.worker.os._exit')
|
@patch('model_manager.worker.worker.os._exit')
|
||||||
def test_start_prometheus_server_failure(
|
def test_start_prometheus_server_failure(
|
||||||
mock_exit, mock_metrics, mock_start_http_server, mock_env_vars
|
mock_exit, mock_metrics, mock_start_http_server, mock_env_vars, mock_logger
|
||||||
):
|
):
|
||||||
"""Test Prometheus server startup failure."""
|
"""Test Prometheus server startup failure."""
|
||||||
from model_manager.worker.worker import start_prometheus_server
|
from model_manager.worker.worker import start_prometheus_server
|
||||||
|
|
||||||
mock_start_http_server.side_effect = OSError('Port already in use')
|
mock_start_http_server.side_effect = OSError('Port already in use')
|
||||||
|
|
||||||
start_prometheus_server()
|
metadata = {'pod_id': 'test-pod-123', 'workflow_name': 'train_model'}
|
||||||
|
|
||||||
# Verify exit was called with code 1
|
start_prometheus_server(mock_logger, metadata)
|
||||||
|
|
||||||
|
# Verify exit was called with code 1 e log crítico emitido
|
||||||
mock_exit.assert_called_once_with(1)
|
mock_exit.assert_called_once_with(1)
|
||||||
|
mock_logger.custom_critical.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -199,6 +211,7 @@ async def test_main_successful_startup(
|
|||||||
mock_build_mongodb.return_value = {
|
mock_build_mongodb.return_value = {
|
||||||
'connection_string': 'mongodb://test',
|
'connection_string': 'mongodb://test',
|
||||||
'database_name': 'test_db',
|
'database_name': 'test_db',
|
||||||
|
'uri': 'localhost:27018',
|
||||||
}
|
}
|
||||||
mock_build_postgres.return_value = {}
|
mock_build_postgres.return_value = {}
|
||||||
mock_build_mlflow.return_value = {}
|
mock_build_mlflow.return_value = {}
|
||||||
@@ -234,7 +247,8 @@ async def test_main_successful_startup(
|
|||||||
mock_notification_handler_class.assert_called_once()
|
mock_notification_handler_class.assert_called_once()
|
||||||
mock_activities_class.assert_called_once()
|
mock_activities_class.assert_called_once()
|
||||||
mock_client_class.connect.assert_called_once()
|
mock_client_class.connect.assert_called_once()
|
||||||
mock_worker_class.assert_called_once()
|
# Agora são criados dois Workers: um para train_model-queue e outro para cleanup-queue
|
||||||
|
assert mock_worker_class.call_count == 2
|
||||||
|
|
||||||
# Verify cleanup was performed
|
# Verify cleanup was performed
|
||||||
mock_notification_handler.shutdown.assert_called_once()
|
mock_notification_handler.shutdown.assert_called_once()
|
||||||
@@ -279,6 +293,7 @@ async def test_main_handles_exception(
|
|||||||
mock_build_mongodb.return_value = {
|
mock_build_mongodb.return_value = {
|
||||||
'connection_string': 'mongodb://test',
|
'connection_string': 'mongodb://test',
|
||||||
'database_name': 'test_db',
|
'database_name': 'test_db',
|
||||||
|
'uri': 'localhost:27018',
|
||||||
}
|
}
|
||||||
mock_build_postgres.return_value = {}
|
mock_build_postgres.return_value = {}
|
||||||
mock_build_mlflow.return_value = {}
|
mock_build_mlflow.return_value = {}
|
||||||
@@ -322,6 +337,7 @@ async def test_main_handles_exception(
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
@patch('model_manager.worker.worker.create_cleanup_schedule')
|
||||||
@patch('model_manager.worker.worker.Worker')
|
@patch('model_manager.worker.worker.Worker')
|
||||||
@patch('model_manager.worker.worker.client.Client')
|
@patch('model_manager.worker.worker.client.Client')
|
||||||
@patch('model_manager.worker.worker.Runtime')
|
@patch('model_manager.worker.worker.Runtime')
|
||||||
@@ -347,20 +363,28 @@ async def test_main_temporal_client_configuration(
|
|||||||
mock_runtime_class,
|
mock_runtime_class,
|
||||||
mock_client_class,
|
mock_client_class,
|
||||||
mock_worker_class,
|
mock_worker_class,
|
||||||
|
mock_create_cleanup_schedule,
|
||||||
mock_logger,
|
mock_logger,
|
||||||
):
|
):
|
||||||
"""Test that Temporal client is configured correctly."""
|
"""Test that Temporal client is configured correctly."""
|
||||||
from model_manager.worker.worker import main
|
from model_manager.worker.worker import main
|
||||||
|
|
||||||
|
mock_create_cleanup_schedule.return_value = AsyncMock()
|
||||||
|
|
||||||
with patch.dict(
|
with patch.dict(
|
||||||
os.environ,
|
os.environ,
|
||||||
{'TEMPORAL_HOST': 'temporal.example.com:7233', 'TEMPORAL_NAMESPACE': 'production'},
|
{
|
||||||
|
'TEMPORAL_HOST': 'temporal.example.com:7233',
|
||||||
|
'TEMPORAL_NAMESPACE': 'production',
|
||||||
|
'TEMPORAL_USE_TLS': 'true',
|
||||||
|
},
|
||||||
):
|
):
|
||||||
# Setup mocks
|
# Setup mocks
|
||||||
mock_get_logger.return_value = mock_logger
|
mock_get_logger.return_value = mock_logger
|
||||||
mock_build_mongodb.return_value = {
|
mock_build_mongodb.return_value = {
|
||||||
'connection_string': 'mongodb://test',
|
'connection_string': 'mongodb://test',
|
||||||
'database_name': 'test_db',
|
'database_name': 'test_db',
|
||||||
|
'uri': 'localhost:27018',
|
||||||
}
|
}
|
||||||
mock_build_postgres.return_value = {}
|
mock_build_postgres.return_value = {}
|
||||||
mock_build_mlflow.return_value = {}
|
mock_build_mlflow.return_value = {}
|
||||||
@@ -393,11 +417,15 @@ async def test_main_temporal_client_configuration(
|
|||||||
|
|
||||||
# Verify Temporal client was configured with correct parameters
|
# Verify Temporal client was configured with correct parameters
|
||||||
mock_client_class.connect.assert_called_once_with(
|
mock_client_class.connect.assert_called_once_with(
|
||||||
target_host='temporal.example.com:7233', namespace='production', runtime=mock_runtime
|
target_host='temporal.example.com:7233',
|
||||||
|
namespace='production',
|
||||||
|
runtime=mock_runtime,
|
||||||
|
tls=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
@patch('model_manager.worker.worker.create_cleanup_schedule')
|
||||||
@patch('model_manager.worker.worker.Worker')
|
@patch('model_manager.worker.worker.Worker')
|
||||||
@patch('model_manager.worker.worker.client.Client')
|
@patch('model_manager.worker.worker.client.Client')
|
||||||
@patch('model_manager.worker.worker.Runtime')
|
@patch('model_manager.worker.worker.Runtime')
|
||||||
@@ -423,30 +451,142 @@ async def test_main_worker_configuration(
|
|||||||
mock_runtime_class,
|
mock_runtime_class,
|
||||||
mock_client_class,
|
mock_client_class,
|
||||||
mock_worker_class,
|
mock_worker_class,
|
||||||
|
mock_create_cleanup_schedule,
|
||||||
mock_env_vars,
|
mock_env_vars,
|
||||||
mock_logger,
|
mock_logger,
|
||||||
):
|
):
|
||||||
"""Test that Temporal worker is configured with correct parameters."""
|
"""Test that Temporal worker is configured with correct parameters."""
|
||||||
from model_manager.worker.worker import main
|
from model_manager.worker.worker import main
|
||||||
|
|
||||||
|
mock_create_cleanup_schedule.return_value = AsyncMock()
|
||||||
|
|
||||||
|
# Patch the task queue constants directly
|
||||||
|
with (
|
||||||
|
patch('model_manager.worker.worker.TRAIN_TASK_QUEUE', 'train_model-local_queue'),
|
||||||
|
patch('model_manager.worker.worker.CLEANUP_TASK_QUEUE', 'cleanup-local_queue'),
|
||||||
|
):
|
||||||
|
# Setup mocks
|
||||||
|
mock_get_logger.return_value = mock_logger
|
||||||
|
mock_build_mongodb.return_value = {
|
||||||
|
'connection_string': 'mongodb://test',
|
||||||
|
'database_name': 'test_db',
|
||||||
|
'uri': 'localhost:27018',
|
||||||
|
}
|
||||||
|
mock_build_postgres.return_value = {}
|
||||||
|
mock_build_mlflow.return_value = {}
|
||||||
|
mock_build_minio.return_value = {}
|
||||||
|
|
||||||
|
mock_notification_handler = Mock()
|
||||||
|
mock_notification_handler_class.return_value = mock_notification_handler
|
||||||
|
|
||||||
|
mock_activities = AsyncMock()
|
||||||
|
mock_activities.update_experiment_run = Mock()
|
||||||
|
mock_activities.validate_train_params = Mock()
|
||||||
|
mock_activities.train_model = Mock()
|
||||||
|
mock_activities.cleanup_resources = Mock()
|
||||||
|
mock_activities.shutdown = AsyncMock()
|
||||||
|
mock_activities_class.return_value = mock_activities
|
||||||
|
|
||||||
|
mock_runtime = Mock()
|
||||||
|
mock_runtime_class.return_value = mock_runtime
|
||||||
|
|
||||||
|
mock_client_instance = AsyncMock()
|
||||||
|
mock_client_class.connect = AsyncMock(return_value=mock_client_instance)
|
||||||
|
|
||||||
|
mock_worker_instance = Mock()
|
||||||
|
mock_worker_instance.run = AsyncMock(side_effect=asyncio.CancelledError())
|
||||||
|
mock_worker_class.return_value = mock_worker_instance
|
||||||
|
|
||||||
|
mock_app_up = Mock()
|
||||||
|
mock_metrics.APP_UP.labels.return_value = mock_app_up
|
||||||
|
|
||||||
|
# Run main()
|
||||||
|
with pytest.raises(SystemExit):
|
||||||
|
await main()
|
||||||
|
|
||||||
|
# Verify Worker was created with correct configuration
|
||||||
|
assert mock_worker_class.call_count == 2
|
||||||
|
|
||||||
|
# Primeira chamada: worker de treinamento (train_model-local_queue)
|
||||||
|
train_call_args = mock_worker_class.call_args_list[0]
|
||||||
|
assert train_call_args[0][0] == mock_client_instance # temporal_client
|
||||||
|
assert train_call_args[1]['task_queue'] == 'train_model-local_queue'
|
||||||
|
assert train_call_args[1]['max_concurrent_workflow_tasks'] == 10
|
||||||
|
assert train_call_args[1]['max_concurrent_activities'] == 10
|
||||||
|
assert train_call_args[1]['max_concurrent_local_activities'] == 10
|
||||||
|
assert train_call_args[1]['max_cached_workflows'] == 100
|
||||||
|
|
||||||
|
train_activities_list = train_call_args[1]['activities']
|
||||||
|
assert mock_activities.update_experiment_run in train_activities_list
|
||||||
|
assert mock_activities.validate_train_params in train_activities_list
|
||||||
|
assert mock_activities.train_model in train_activities_list
|
||||||
|
assert mock_activities.cleanup_resources in train_activities_list
|
||||||
|
|
||||||
|
# Segunda chamada: worker de cleanup (cleanup-local_queue)
|
||||||
|
cleanup_call_args = mock_worker_class.call_args_list[1]
|
||||||
|
assert cleanup_call_args[0][0] == mock_client_instance # temporal_client
|
||||||
|
assert cleanup_call_args[1]['task_queue'] == 'cleanup-local_queue'
|
||||||
|
assert cleanup_call_args[1]['max_concurrent_workflow_tasks'] == 20
|
||||||
|
assert cleanup_call_args[1]['max_concurrent_activities'] == 20
|
||||||
|
assert cleanup_call_args[1]['max_concurrent_local_activities'] == 20
|
||||||
|
assert cleanup_call_args[1]['max_cached_workflows'] == 100
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@patch('model_manager.worker.worker.create_cleanup_schedule')
|
||||||
|
@patch('model_manager.worker.worker.Worker')
|
||||||
|
@patch('model_manager.worker.worker.client.Client')
|
||||||
|
@patch('model_manager.worker.worker.Runtime')
|
||||||
|
@patch('model_manager.worker.worker.Activities')
|
||||||
|
@patch('model_manager.worker.worker.NotificationHandler')
|
||||||
|
@patch('model_manager.worker.worker.build_mongodb_config')
|
||||||
|
@patch('model_manager.worker.worker.build_postgres_config')
|
||||||
|
@patch('model_manager.worker.worker.build_mlflow_config')
|
||||||
|
@patch('model_manager.worker.worker.build_minio_config')
|
||||||
|
@patch('model_manager.worker.worker.get_logger')
|
||||||
|
@patch('model_manager.worker.worker.start_prometheus_server')
|
||||||
|
@patch('model_manager.worker.worker.metrics')
|
||||||
|
async def test_main_schedule_creation_failure_does_not_stop_worker(
|
||||||
|
mock_metrics,
|
||||||
|
mock_start_prometheus,
|
||||||
|
mock_get_logger,
|
||||||
|
mock_build_minio,
|
||||||
|
mock_build_mlflow,
|
||||||
|
mock_build_postgres,
|
||||||
|
mock_build_mongodb,
|
||||||
|
mock_notification_handler_class,
|
||||||
|
mock_activities_class,
|
||||||
|
mock_runtime_class,
|
||||||
|
mock_client_class,
|
||||||
|
mock_worker_class,
|
||||||
|
mock_create_cleanup_schedule,
|
||||||
|
mock_logger,
|
||||||
|
):
|
||||||
|
"""Test that schedule creation failure does not prevent worker startup."""
|
||||||
|
from model_manager.worker.worker import main
|
||||||
|
|
||||||
|
# Mock schedule creation to raise an exception (as coroutine)
|
||||||
|
async def mock_schedule_error(*args, **kwargs):
|
||||||
|
raise Exception('Schedule creation failed')
|
||||||
|
|
||||||
|
mock_create_cleanup_schedule.side_effect = mock_schedule_error
|
||||||
|
|
||||||
# Setup mocks
|
# Setup mocks
|
||||||
mock_get_logger.return_value = mock_logger
|
mock_get_logger.return_value = mock_logger
|
||||||
mock_build_mongodb.return_value = {
|
mock_build_mongodb.return_value = {
|
||||||
'connection_string': 'mongodb://test',
|
'connection_string': 'mongodb://test',
|
||||||
'database_name': 'test_db',
|
'database_name': 'test_db',
|
||||||
|
'uri': 'localhost:27018',
|
||||||
}
|
}
|
||||||
mock_build_postgres.return_value = {}
|
mock_build_postgres.return_value = {}
|
||||||
mock_build_mlflow.return_value = {}
|
mock_build_mlflow.return_value = {}
|
||||||
mock_build_minio.return_value = {}
|
mock_build_minio.return_value = {}
|
||||||
|
|
||||||
mock_notification_handler = Mock()
|
mock_notification_handler = Mock()
|
||||||
|
mock_notification_handler.shutdown = Mock()
|
||||||
mock_notification_handler_class.return_value = mock_notification_handler
|
mock_notification_handler_class.return_value = mock_notification_handler
|
||||||
|
|
||||||
mock_activities = AsyncMock()
|
mock_activities = AsyncMock()
|
||||||
mock_activities.update_experiment_run = Mock()
|
|
||||||
mock_activities.validate_train_params = Mock()
|
|
||||||
mock_activities.train_model = Mock()
|
|
||||||
mock_activities.cleanup_resources = Mock()
|
|
||||||
mock_activities.shutdown = AsyncMock()
|
mock_activities.shutdown = AsyncMock()
|
||||||
mock_activities_class.return_value = mock_activities
|
mock_activities_class.return_value = mock_activities
|
||||||
|
|
||||||
@@ -463,27 +603,27 @@ async def test_main_worker_configuration(
|
|||||||
mock_app_up = Mock()
|
mock_app_up = Mock()
|
||||||
mock_metrics.APP_UP.labels.return_value = mock_app_up
|
mock_metrics.APP_UP.labels.return_value = mock_app_up
|
||||||
|
|
||||||
# Run main()
|
# Run main() - should not fail despite schedule creation error
|
||||||
with pytest.raises(SystemExit):
|
with pytest.raises(SystemExit):
|
||||||
await main()
|
await main()
|
||||||
|
|
||||||
# Verify Worker was created with correct configuration
|
# Verify schedule creation was attempted
|
||||||
mock_worker_class.assert_called_once()
|
mock_create_cleanup_schedule.assert_called_once()
|
||||||
call_args = mock_worker_class.call_args
|
|
||||||
|
|
||||||
assert call_args[0][0] == mock_client_instance # temporal_client
|
# Verify error was logged - check all custom_error calls
|
||||||
assert call_args[1]['task_queue'] == 'train_model-queue'
|
assert mock_logger.custom_error.call_count >= 1
|
||||||
assert call_args[1]['max_concurrent_workflow_tasks'] == 50
|
|
||||||
assert call_args[1]['max_concurrent_activities'] == 50
|
|
||||||
assert call_args[1]['max_concurrent_local_activities'] == 50
|
|
||||||
assert call_args[1]['max_cached_workflows'] == 200
|
|
||||||
|
|
||||||
# Verify activities are included
|
# Find the call that contains the schedule error message
|
||||||
activities_list = call_args[1]['activities']
|
schedule_error_logged = False
|
||||||
assert mock_activities.update_experiment_run in activities_list
|
for call in mock_logger.custom_error.call_args_list:
|
||||||
assert mock_activities.validate_train_params in activities_list
|
if 'Failed to configure cleanup schedule' in call[0][0]:
|
||||||
assert mock_activities.train_model in activities_list
|
schedule_error_logged = True
|
||||||
assert mock_activities.cleanup_resources in activities_list
|
break
|
||||||
|
|
||||||
|
assert schedule_error_logged, 'Schedule creation error should be logged'
|
||||||
|
|
||||||
|
# Verify workers were still created (startup continued)
|
||||||
|
assert mock_worker_class.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
@patch('model_manager.worker.worker.asyncio.run')
|
@patch('model_manager.worker.worker.asyncio.run')
|
||||||
@@ -513,7 +653,7 @@ def test_worker_module_docstring():
|
|||||||
@patch('model_manager.worker.worker.start_http_server')
|
@patch('model_manager.worker.worker.start_http_server')
|
||||||
@patch('model_manager.worker.worker.metrics')
|
@patch('model_manager.worker.worker.metrics')
|
||||||
def test_start_prometheus_server_prints_success(
|
def test_start_prometheus_server_prints_success(
|
||||||
mock_metrics, mock_start_http_server, capsys, mock_env_vars
|
mock_metrics, mock_start_http_server, capsys, mock_env_vars, mock_logger
|
||||||
):
|
):
|
||||||
"""Test that start_prometheus_server prints success message."""
|
"""Test that start_prometheus_server prints success message."""
|
||||||
from model_manager.worker.worker import start_prometheus_server
|
from model_manager.worker.worker import start_prometheus_server
|
||||||
@@ -521,25 +661,29 @@ def test_start_prometheus_server_prints_success(
|
|||||||
mock_app_up = Mock()
|
mock_app_up = Mock()
|
||||||
mock_metrics.APP_UP.labels.return_value = mock_app_up
|
mock_metrics.APP_UP.labels.return_value = mock_app_up
|
||||||
|
|
||||||
start_prometheus_server()
|
metadata = {'pod_id': 'test-pod-123', 'workflow_name': 'train_model'}
|
||||||
|
|
||||||
captured = capsys.readouterr()
|
start_prometheus_server(mock_logger, metadata)
|
||||||
assert 'Prometheus server started on port 9090' in captured.out
|
|
||||||
|
# Agora a mensagem é enviada via logger
|
||||||
|
mock_logger.custom_info.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
@patch('model_manager.worker.worker.start_http_server')
|
@patch('model_manager.worker.worker.start_http_server')
|
||||||
@patch('model_manager.worker.worker.metrics')
|
@patch('model_manager.worker.worker.metrics')
|
||||||
@patch('model_manager.worker.worker.os._exit')
|
@patch('model_manager.worker.worker.os._exit')
|
||||||
def test_start_prometheus_server_prints_failure(
|
def test_start_prometheus_server_prints_failure(
|
||||||
mock_exit, mock_metrics, mock_start_http_server, capsys, mock_env_vars
|
mock_exit, mock_metrics, mock_start_http_server, capsys, mock_env_vars, mock_logger
|
||||||
):
|
):
|
||||||
"""Test that start_prometheus_server prints failure message."""
|
"""Test that start_prometheus_server prints failure message."""
|
||||||
from model_manager.worker.worker import start_prometheus_server
|
from model_manager.worker.worker import start_prometheus_server
|
||||||
|
|
||||||
mock_start_http_server.side_effect = Exception('Test error')
|
mock_start_http_server.side_effect = Exception('Test error')
|
||||||
|
|
||||||
start_prometheus_server()
|
metadata = {'pod_id': 'test-pod-123', 'workflow_name': 'train_model'}
|
||||||
|
|
||||||
captured = capsys.readouterr()
|
start_prometheus_server(mock_logger, metadata)
|
||||||
assert 'Failed to start Prometheus server' in captured.out
|
|
||||||
assert 'Test error' in captured.out
|
# Agora o erro é logado via logger crítico
|
||||||
|
mock_logger.custom_critical.assert_called_once()
|
||||||
|
mock_exit.assert_called_once_with(1)
|
||||||
|
|||||||
76
tests/workflows/test_cleanup_files.py
Normal file
76
tests/workflows/test_cleanup_files.py
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
"""Unit tests for the CleanupFiles workflow."""
|
||||||
|
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@patch('model_manager.workflows.cleanup_files.workflow')
|
||||||
|
@patch('model_manager.workflows.cleanup_files.POD_ID', 'temporal-pod')
|
||||||
|
async def test_cleanup_files_workflow_with_input_bucket(mock_workflow_module):
|
||||||
|
"""Test the CleanupFiles workflow when bucket_name is provided in the input."""
|
||||||
|
from model_manager.workflows.cleanup_files import CleanupFiles
|
||||||
|
|
||||||
|
# Mock execute_activity_method
|
||||||
|
mock_workflow_module.execute_activity_method = AsyncMock()
|
||||||
|
|
||||||
|
# Instantiate and run the workflow
|
||||||
|
workflow_instance = CleanupFiles()
|
||||||
|
await workflow_instance.run({'bucket_name': 'input-bucket'})
|
||||||
|
|
||||||
|
# Verify that the activities were called with the correct parameters
|
||||||
|
calls = mock_workflow_module.execute_activity_method.call_args_list
|
||||||
|
assert len(calls) == 2
|
||||||
|
|
||||||
|
# Check cleanup_minio_files call
|
||||||
|
minio_call_args = calls[0][0][1]
|
||||||
|
assert minio_call_args['bucket_name'] == 'input-bucket'
|
||||||
|
assert minio_call_args['metadata'] == {
|
||||||
|
'pod_id': 'temporal-pod',
|
||||||
|
'workflow_name': 'cleanup_files',
|
||||||
|
}
|
||||||
|
|
||||||
|
# Check cleanup_temp_directories call
|
||||||
|
local_call_args = calls[1][0][1]
|
||||||
|
assert local_call_args['temp_path'] == 'model_manager/reports/temp'
|
||||||
|
assert local_call_args['metadata'] == {
|
||||||
|
'pod_id': 'temporal-pod',
|
||||||
|
'workflow_name': 'cleanup_files',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@patch('model_manager.workflows.cleanup_files.workflow')
|
||||||
|
@patch('model_manager.workflows.cleanup_files.POD_ID', 'temporal-pod')
|
||||||
|
@patch('model_manager.workflows.cleanup_files.DEFAULT_CLEANUP_BUCKET', 'env-var-bucket')
|
||||||
|
async def test_cleanup_files_workflow_with_default_bucket(mock_workflow_module):
|
||||||
|
"""Test the CleanupFiles workflow when using the default bucket from environment variables."""
|
||||||
|
from model_manager.workflows.cleanup_files import CleanupFiles
|
||||||
|
|
||||||
|
# Mock execute_activity_method
|
||||||
|
mock_workflow_module.execute_activity_method = AsyncMock()
|
||||||
|
|
||||||
|
# Instantiate and run the workflow
|
||||||
|
workflow_instance = CleanupFiles()
|
||||||
|
await workflow_instance.run({}) # Empty input
|
||||||
|
|
||||||
|
# Verify that the activities were called
|
||||||
|
calls = mock_workflow_module.execute_activity_method.call_args_list
|
||||||
|
assert len(calls) == 2
|
||||||
|
|
||||||
|
# Check cleanup_minio_files call
|
||||||
|
minio_call_args = calls[0][0][1]
|
||||||
|
assert minio_call_args['bucket_name'] == 'env-var-bucket'
|
||||||
|
assert minio_call_args['metadata'] == {
|
||||||
|
'pod_id': 'temporal-pod',
|
||||||
|
'workflow_name': 'cleanup_files',
|
||||||
|
}
|
||||||
|
|
||||||
|
# Check cleanup_temp_directories call
|
||||||
|
local_call_args = calls[1][0][1]
|
||||||
|
assert local_call_args['temp_path'] == 'model_manager/reports/temp'
|
||||||
|
assert local_call_args['metadata'] == {
|
||||||
|
'pod_id': 'temporal-pod',
|
||||||
|
'workflow_name': 'cleanup_files',
|
||||||
|
}
|
||||||
@@ -1,7 +1,5 @@
|
|||||||
- Criar o dashboard do grafana.
|
- Criar um gráfico no grafana para cada nova atividade.
|
||||||
|
|
||||||
- Atualizar o .github/workflows/quality-gate.yml para usar os pipelines genéricos do github;
|
- Atualizar o .github/workflows/quality-gate.yml para usar os pipelines genéricos do github;
|
||||||
Criar um workflow para fazer o deploy no suse.
|
Criar um workflow para fazer o deploy no suse.
|
||||||
Criar um workflow para criar o release no github.
|
Criar um workflow para criar o release no github.
|
||||||
|
|
||||||
- Atualizar a documentação do projeto.
|
|
||||||
|
|||||||
29
values.yaml
29
values.yaml
@@ -187,6 +187,12 @@ env:
|
|||||||
value: "temporal-frontend.temporal.svc.cluster.local:7233"
|
value: "temporal-frontend.temporal.svc.cluster.local:7233"
|
||||||
- name: TEMPORAL_NAMESPACE
|
- name: TEMPORAL_NAMESPACE
|
||||||
value: "model-manager"
|
value: "model-manager"
|
||||||
|
- name: TRAIN_TASK_QUEUE
|
||||||
|
value: "train_model-queue"
|
||||||
|
- name: CLEANUP_TASK_QUEUE
|
||||||
|
value: "cleanup-queue"
|
||||||
|
- name: TEMPORAL_USE_TLS
|
||||||
|
value: "false"
|
||||||
|
|
||||||
- name: MONGODB_USERNAME
|
- name: MONGODB_USERNAME
|
||||||
value: "root"
|
value: "root"
|
||||||
@@ -227,6 +233,29 @@ env:
|
|||||||
- name: TIMEOUT_UPDATE_DATABASE
|
- name: TIMEOUT_UPDATE_DATABASE
|
||||||
value: "30"
|
value: "30"
|
||||||
|
|
||||||
|
- name: CLEANUP_RETENTION_HOURS
|
||||||
|
value: "24"
|
||||||
|
- name: CLEANUP_DRY_RUN
|
||||||
|
value: "false"
|
||||||
|
- name: TIMEOUT_CLEANUP_MINIO
|
||||||
|
value: "300"
|
||||||
|
- name: TIMEOUT_CLEANUP_LOCAL
|
||||||
|
value: "120"
|
||||||
|
- name: MAX_KEYS_CLEANUP
|
||||||
|
value: "1000"
|
||||||
|
- name: DEFAULT_CLEANUP_BUCKET
|
||||||
|
value: "model-training"
|
||||||
|
|
||||||
|
# Cleanup Schedule Configuration
|
||||||
|
- name: CLEANUP_SCHEDULE_ID
|
||||||
|
value: "cleanup-files-daily"
|
||||||
|
- name: CLEANUP_CRON
|
||||||
|
value: "0 0 * * *" # Midnight UTC
|
||||||
|
- name: CLEANUP_TIMEZONE
|
||||||
|
value: "UTC"
|
||||||
|
- name: CLEANUP_EXECUTION_TIMEOUT_HOURS
|
||||||
|
value: "1"
|
||||||
|
|
||||||
- name: EXTRA_PIP_REQUIREMENTS
|
- name: EXTRA_PIP_REQUIREMENTS
|
||||||
value: "git+https://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@github.com/Aignosi/sientia-mlops-library.git"
|
value: "git+https://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@github.com/Aignosi/sientia-mlops-library.git"
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user