Update dependencies, improve CI workflow, and enhance code formatting

- Updated the `sientia-dataops-library` dependency version from 1.4.3 to 1.4.6 in `requirements.txt`.
- Modified the GitHub Actions workflow to install development and runtime dependencies separately, improving clarity and organization.
- Added code formatting and linting checks using Ruff, along with type checking using mypy, to ensure code quality.
- Updated `.gitignore` to include additional cache directories and log files.
- Refactored code in various files for consistency in string formatting and improved logging messages.

validate.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+# Model Manager Code Validation Script
+# This script runs all code quality checks before committing or deploying
+
+set -e  # Exit on any error
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Args
+FIX_MODE=false
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --fix)
+            FIX_MODE=true
+            shift
+            ;;
+        -h|--help)
+            echo "Usage: $0 [--fix]"
+            echo "  --fix   Apply Ruff auto-fixes (format and lint fixes)."
+            exit 0
+            ;;
+        *)
+            echo -e "${RED}Unknown option: $1${NC}"
+            echo "Usage: $0 [--fix]"
+            exit 2
+            ;;
+    esac
+done
+
+echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
+echo -e "${BLUE}║  Model Manager - Code Validation Suite                ║${NC}"
+echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
+echo ""
+
+# Check if virtual environment is activated
+if [[ -z "${VIRTUAL_ENV}" ]] && [[ -z "${CONDA_DEFAULT_ENV}" ]]; then
+    echo -e "${YELLOW}⚠️  Warning: No virtual environment detected${NC}"
+    echo -e "${YELLOW}   Consider activating your venv/conda environment${NC}"
+    echo ""
+fi
+
+# Function to run a validation step
+run_step() {
+    local step_name=$1
+    local step_command=$2
+    
+    echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+    echo -e "${BLUE}▶ ${step_name}${NC}"
+    echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+    
+    if eval "$step_command"; then
+        echo -e "${GREEN}✅ ${step_name} - PASSED${NC}"
+        echo ""
+        return 0
+    else
+        echo -e "${RED}❌ ${step_name} - FAILED${NC}"
+        echo ""
+        return 1
+    fi
+}
+
+# Track failures
+FAILED_STEPS=()
+
+# Step 1: Code Formatting Check (Ruff)
+# - default: check only
+# - --fix: write changes
+if ! run_step "1. Code Formatting (Ruff)" "if \$FIX_MODE; then ruff format ingestor/ tests/; else ruff format --check ingestor/ tests/; fi"; then
+    FAILED_STEPS+=("Code Formatting")
+fi
+
+# Step 2: Linting (Ruff)
+# - default: check only
+# - --fix: apply autofixes
+if ! run_step "2. Code Linting (Ruff)" "if \$FIX_MODE; then ruff check --fix ingestor/ tests/; else ruff check ingestor/ tests/; fi"; then
+    FAILED_STEPS+=("Linting")
+fi
+
+# Step 3: Type Checking (mypy)
+if ! run_step "3. Type Checking (mypy)" "mypy ingestor/"; then
+    FAILED_STEPS+=("Type Checking")
+fi
+
+# Step 4: Security Analysis (Bandit)
+if ! run_step "4. Security Analysis (Bandit)" "bandit -r ingestor/ -ll -q"; then
+    FAILED_STEPS+=("Security Analysis")
+fi
+
+# Step 5: Unit Tests (pytest)
+if ! run_step "5. Unit Tests (pytest)" "pytest tests/ --cov=ingestor --cov-report=term-missing --cov-report=xml --cov-report=html --cov-fail-under=80 -q"; then
+    FAILED_STEPS+=("Unit Tests")
+fi
+
+# Summary
+echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
+echo -e "${BLUE}║  Validation Summary                                    ║${NC}"
+echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
+echo ""
+
+if [ ${#FAILED_STEPS[@]} -eq 0 ]; then
+    echo -e "${GREEN}✅ All validation checks passed!${NC}"
+    echo -e "${GREEN}   Your code is ready for commit/deployment.${NC}"
+    echo ""
+    exit 0
+else
+    echo -e "${RED}❌ Validation failed for the following steps:${NC}"
+    for step in "${FAILED_STEPS[@]}"; do
+        echo -e "${RED}   • ${step}${NC}"
+    done
+    echo ""
+    echo -e "${YELLOW}💡 Tips:${NC}"
+    echo -e "${YELLOW}   • Run 'ruff format ingestor/ tests/' to auto-fix formatting${NC}"
+    echo -e "${YELLOW}   • Run 'ruff check --fix ingestor/ tests/' to auto-fix linting issues${NC}"
+    echo -e "${YELLOW}   • Review mypy errors and add type hints where needed${NC}"
+    echo -e "${YELLOW}   • Check bandit warnings for security issues${NC}"
+    echo -e "${YELLOW}   • Fix failing tests or improve test coverage${NC}"
+    echo ""
+    exit 1
+fi