Merge pull request #12 from Aignosi/feature/SIENTIAPDE-1308

SIENTIAPDE-1308: Dockerize project, update documentation, and improve dependency management

.dockerignore

@@ -0,0 +1,204 @@
+# Git
+.git
+.gitignore
+.gitattributes
+
+# Documentation
+*.md
+docs/
+README*
+
+# Tests and development
+tests/
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+.mypy_cache/
+.pyre/
+coverage.xml
+*.cover
+.hypothesis/
+
+# Python cache and compiled files
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+venv/
+env/
+ENV/
+.venv/
+.env/
+
+# IDE and editors
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+Thumbs.db
+
+# OS files
+.dockerignore
+.dockerignore.dockerignore
+
+# CI/CD
+.github/
+.gitlab-ci.yml
+.travis.yml
+.circleci/
+Jenkinsfile
+
+# Local configuration
+.env
+.env.local
+.env.*.local
+config/local/
+*.local
+
+# Logs
+*.log
+logs/
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids/
+*.pid
+*.seed
+*.pid.lock
+
+# Temporary files
+tmp/
+temp/
+*.tmp
+*.temp
+
+# Node.js (if any frontend tools)
+node_modules/
+npm-debug.log*
+
+# Database
+*.db
+*.sqlite
+*.sqlite3
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# pipenv
+Pipfile.lock
+
+# PEP 582
+__pypackages__/
+
+# Celery
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# Backup files
+*.bak
+*.backup
+*.old
+
+# Local development scripts
+scripts/local/
+dev-*
+
+# Docker files (excluding the main ones)
+docker-compose*.yml
+docker-compose*.yaml
+Dockerfile.*
+!Dockerfile
+
+# Helm charts (already in .gitignore but reinforcing)
+charts/
+
+# Kubernetes manifests
+k8s/
+kube-*
+
+# Terraform
+*.tfstate
+*.tfstate.*
+.terraform/
+
+# Monitoring and profiling
+*.prof
+*.profile
+.perf
+
+# Security
+*.pem
+*.key
+*.crt
+*.p12
+secrets/
+*.secret
+
+# Large binaries and datasets
+*.bin
+*.pkl
+*.pickle
+*.joblib
+data/
+datasets/
+models/pre-trained/
+
+# Build artifacts
+build/
+dist/
+target/
+out/
+
+# Package manager lock files (keeping requirements.txt)
+package-lock.json
+yarn.lock
+Pipfile.lock
+
+# Local tools
+tools/local/
+bin/local/
+
+# Cache directories
+.cache/
+cache/

Dockerfile

@@ -0,0 +1,78 @@
+# Multi-stage build for optimized Python application
+FROM python:3.11-slim AS builder
+
+# Set build-time environment variables
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+# Install build dependencies only
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    git \
+    && rm -rf /var/lib/apt/lists/* && \
+    apt-get clean
+
+# Configure SSH to trust GitHub host key
+RUN mkdir -p ~/.ssh && \
+    ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts && \
+    chmod 600 ~/.ssh/known_hosts
+
+# Create virtual environment
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Upgrade pip and wheel for better caching
+RUN pip install --upgrade pip setuptools wheel
+
+# Copy requirements files for better Docker layer caching
+COPY requirements.txt requirements-dev.txt ./
+
+# Install only production dependencies with no cache
+RUN --mount=type=ssh echo "=== Installing dependencies ===" && \
+    pip install --no-cache-dir -r requirements.txt && \
+    echo "=== Dependencies installed successfully ===" && \
+    pip list | wc -l && \
+    echo "=== Cleaning cache files ===" && \
+    find /opt/venv -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \
+    find /opt/venv -name "*.pyc" -delete 2>/dev/null || true && \
+    rm -rf /root/.cache/pip/* && \
+    echo "=== Cleaning venv site-packages ===" && \
+    find /opt/venv/lib/python3.11/site-packages/ -type f -name "*.md" -delete 2>/dev/null || true && \
+    echo "=== Stripping .so files ===" && \
+    find /opt/venv -name "*.so" -exec strip {} + 2>/dev/null || true
+
+# Production stage using python-slim for better functionality
+FROM python:3.11-slim AS production
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PATH="/opt/venv/bin:$PATH" \
+    POD_ID=unknown \
+    HOME="/app"
+
+# Copy virtual environment from builder stage
+COPY --from=builder /opt/venv /opt/venv
+
+# Set working directory
+WORKDIR /app
+
+# Copy application code
+COPY . .
+
+# Create necessary directories for runtime file creation
+RUN mkdir -p /app/model_manager/reports /app/logs /app/temp /app/models /app/data && \
+    chmod 755 /app/model_manager/reports /app/logs /app/temp /app/models /app/data
+
+# Create non-root user
+RUN groupadd -r appuser && useradd -r -g appuser appuser && \
+    chown -R appuser:appuser /app
+
+# Switch to non-root user
+USER appuser
+
+# Set entrypoint for proper signal handling and PID 1
+ENTRYPOINT ["/opt/venv/bin/python", "-m", "model_manager.worker.worker"]

README.md

@@ -494,13 +494,7 @@ kubectl exec -n temporal <temporal-admin-tools-pod-name> -- \
     gh auth login
     ```
 
-  3. **Run the install_dependencies.sh script**
+  3. **Install Python dependencies**
-    ```bash
-    chmod +x install_dependencies.sh
-    ./install_dependencies.sh
-    ```
-
-  4. **Install Python dependencies**
     ```bash
     python -m pip install --upgrade pip
     # Install production dependencies
@@ -1037,6 +1031,34 @@ For support and questions:
 - Open an issue in the project repository
 - Contact the development team
 
+## Docker
+
+### Create image
+
+```shell
+$ docker build --ssh default --no-cache --progress=plain -t aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0 .
+```
+
+### Create container
+
+```shell
+$ docker run --env-file .env --network="host" --name sientia-dataops-model-manager -d aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0
+
+$ docker logs -f sientia-dataops-model-manager
+```
+
+### Login using access token
+
+```shell
+$ docker login -u bruno-aignosi -p LD/IyZ4vtDI7khRYnH4HzfdTx3toorg6hlCetJM54n+ACRDim3xO aignosi.azurecr.io
+```
+
+### Push image to repository
+
+```shell
+$ docker push aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0
+```
+
 ---
 
 **Note**: The Model Manager system is designed for production use in industrial ML environments. Ensure proper security configuration and network isolation for production deployments.

model_manager/activities/training.py

@@ -51,7 +51,7 @@ class Training(SientiaMonitoring):
             logger: Logger instance for observability
             notification_handler: Handler for sending notifications
         """
-        super().__init__(logger, notification_handler, metrics_controller, set_error_counter=True)
+        super().__init__(logger, notification_handler, metrics_controller)
         self.training_repository = TrainingRepository(logger)
         self.model_repository = model_repository
         self.storage_repository = storage_repository

pyproject.toml

@@ -168,3 +168,14 @@ directory = "htmlcov"
 [tool.bandit]
 exclude_dirs = ["tests", "venv", ".venv"]
 skips = ["B101", "B601"]  # Skip assert and shell injection in controlled environments
+
+[tool.deptry]
+known_first_party = [
+    "model_manager"
+]
+requirements_files = [
+    "requirements.txt"
+]
+requirements_files_dev = [
+    "requirements-dev.txt"
+]

requirements.txt

@@ -3,7 +3,7 @@ psycopg2-binary==2.9.11
 sqlalchemy==2.0.44
 boto3==1.40.55
 botocore==1.40.55
-git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.5.1
+git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.5.2
 prometheus-client==0.23.1
 mlflow==2.10.1
 evidently==0.4.21

todo-list.txt

@@ -1,7 +1,7 @@
 - Criar o dashboard do grafana.
 
-- Atualizar a lib do sientia-dataops-library para a versão 1.5.2.
-
 - Atualizar o .github/workflows/quality-gate.yml para usar os pipelines genéricos do github;
   Criar um workflow para fazer o deploy no suse.
   Criar um workflow para criar o release no github.
+
+- Atualizar a documentação do projeto.