From d9d3df5723f3bdd9ce9d154cf6aff08b9b99eb43 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Tue, 4 Nov 2025 14:07:37 -0300 Subject: [PATCH] SIENTIAPDE-1308: Initial Docker setup, improved .dockerignore, updated README, and minor fixes. --- .dockerignore | 204 +++++++++++++++++++++++++++++++++++++++++++++++++ Dockerfile | 78 +++++++++++++++++++ README.md | 28 +++++++ pyproject.toml | 11 +++ todo-list.txt | 2 - 5 files changed, 321 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..430ab98 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,204 @@ +# Git +.git +.gitignore +.gitattributes + +# Documentation +*.md +docs/ +README* + +# Tests and development +tests/ +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +.nox/ +.mypy_cache/ +.pyre/ +coverage.xml +*.cover +.hypothesis/ + +# Python cache and compiled files +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Virtual environments +venv/ +env/ +ENV/ +.venv/ +.env/ + +# IDE and editors +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store +Thumbs.db + +# OS files +.dockerignore +.dockerignore.dockerignore + +# CI/CD +.github/ +.gitlab-ci.yml +.travis.yml +.circleci/ +Jenkinsfile + +# Local configuration +.env +.env.local +.env.*.local +config/local/ +*.local + +# Logs +*.log +logs/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids/ +*.pid +*.seed +*.pid.lock + +# Temporary files +tmp/ +temp/ +*.tmp +*.temp + +# Node.js (if any frontend tools) +node_modules/ +npm-debug.log* + +# Database +*.db +*.sqlite +*.sqlite3 + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# pipenv +Pipfile.lock + +# PEP 582 +__pypackages__/ + +# Celery +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# Backup files +*.bak +*.backup +*.old + +# Local development scripts +scripts/local/ +dev-* + +# Docker files (excluding the main ones) +docker-compose*.yml +docker-compose*.yaml +Dockerfile.* +!Dockerfile + +# Helm charts (already in .gitignore but reinforcing) +charts/ + +# Kubernetes manifests +k8s/ +kube-* + +# Terraform +*.tfstate +*.tfstate.* +.terraform/ + +# Monitoring and profiling +*.prof +*.profile +.perf + +# Security +*.pem +*.key +*.crt +*.p12 +secrets/ +*.secret + +# Large binaries and datasets +*.bin +*.pkl +*.pickle +*.joblib +data/ +datasets/ +models/pre-trained/ + +# Build artifacts +build/ +dist/ +target/ +out/ + +# Package manager lock files (keeping requirements.txt) +package-lock.json +yarn.lock +Pipfile.lock + +# Local tools +tools/local/ +bin/local/ + +# Cache directories +.cache/ +cache/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..725a605 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,78 @@ +# Multi-stage build for optimized Python application +FROM python:3.11-slim AS builder + +# Set build-time environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +# Install build dependencies only +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + && rm -rf /var/lib/apt/lists/* && \ + apt-get clean + +# Configure SSH to trust GitHub host key +RUN mkdir -p ~/.ssh && \ + ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts && \ + chmod 600 ~/.ssh/known_hosts + +# Create virtual environment +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +# Upgrade pip and wheel for better caching +RUN pip install --upgrade pip setuptools wheel + +# Copy requirements files for better Docker layer caching +COPY requirements.txt requirements-dev.txt ./ + +# Install only production dependencies with no cache +RUN --mount=type=ssh echo "=== Installing dependencies ===" && \ + pip install --no-cache-dir -r requirements.txt && \ + echo "=== Dependencies installed successfully ===" && \ + pip list | wc -l && \ + echo "=== Cleaning cache files ===" && \ + find /opt/venv -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \ + find /opt/venv -name "*.pyc" -delete 2>/dev/null || true && \ + rm -rf /root/.cache/pip/* && \ + echo "=== Cleaning venv site-packages ===" && \ + find /opt/venv/lib/python3.11/site-packages/ -type f -name "*.md" -delete 2>/dev/null || true && \ + echo "=== Stripping .so files ===" && \ + find /opt/venv -name "*.so" -exec strip {} + 2>/dev/null || true + +# Production stage using python-slim for better functionality +FROM python:3.11-slim AS production + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH="/opt/venv/bin:$PATH" \ + POD_ID=unknown \ + HOME="/app" + +# Copy virtual environment from builder stage +COPY --from=builder /opt/venv /opt/venv + +# Set working directory +WORKDIR /app + +# Copy application code +COPY . . + +# Create necessary directories for runtime file creation +RUN mkdir -p /app/model_manager/reports /app/logs /app/temp /app/models /app/data && \ + chmod 755 /app/model_manager/reports /app/logs /app/temp /app/models /app/data + +# Create non-root user +RUN groupadd -r appuser && useradd -r -g appuser appuser && \ + chown -R appuser:appuser /app + +# Switch to non-root user +USER appuser + +# Set entrypoint for proper signal handling and PID 1 +ENTRYPOINT ["/opt/venv/bin/python", "-m", "model_manager.worker.worker"] diff --git a/README.md b/README.md index d41faff..1fa12ae 100644 --- a/README.md +++ b/README.md @@ -1031,6 +1031,34 @@ For support and questions: - Open an issue in the project repository - Contact the development team +## Docker + +### Create image + +```shell +$ docker build --ssh default --no-cache --progress=plain -t aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0 . +``` + +### Create container + +```shell +$ docker run --env-file .env --network="host" --name sientia-dataops-model-manager -d aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0 + +$ docker logs -f sientia-dataops-model-manager +``` + +### Login using access token + +```shell +$ docker login -u bruno-aignosi -p LD/IyZ4vtDI7khRYnH4HzfdTx3toorg6hlCetJM54n+ACRDim3xO aignosi.azurecr.io +``` + +### Push image to repository + +```shell +$ docker push aignosi.azurecr.io/sientia-dataops-model-manager:0.0.0 +``` + --- **Note**: The Model Manager system is designed for production use in industrial ML environments. Ensure proper security configuration and network isolation for production deployments. diff --git a/pyproject.toml b/pyproject.toml index 7356343..cc7c55e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -168,3 +168,14 @@ directory = "htmlcov" [tool.bandit] exclude_dirs = ["tests", "venv", ".venv"] skips = ["B101", "B601"] # Skip assert and shell injection in controlled environments + +[tool.deptry] +known_first_party = [ + "model_manager" +] +requirements_files = [ + "requirements.txt" +] +requirements_files_dev = [ + "requirements-dev.txt" +] diff --git a/todo-list.txt b/todo-list.txt index ef8d4fa..f9fee32 100644 --- a/todo-list.txt +++ b/todo-list.txt @@ -1,7 +1,5 @@ - Criar o dashboard do grafana. -- Atualizar a lib do sientia-dataops-library para a versão 1.5.2. - - Atualizar o .github/workflows/quality-gate.yml para usar os pipelines genéricos do github; Criar um workflow para fazer o deploy no suse. Criar um workflow para criar o release no github.