diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..3ce122d --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,52 @@ +name: Deploy Python Application + +on: + pull_request: + types: + - closed + branches: + - main + +jobs: + get-version: + name: Determine Next Version + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + outputs: + version: ${{ steps.version.outputs.next_version }} + steps: + - uses: actions/checkout@v4 + + - name: Determine next version + id: version + uses: Aignosi/github_workflow_templates/.github/actions/determine-next-version@main + with: + branch_name: ${{ github.event.pull_request.head.ref }} + + deploy: + name: Deploy + needs: [get-version] + uses: Aignosi/github_workflow_templates/.github/workflows/reusable-deploy.yml@feature/SIENTIAPDE-1352 + with: + version: ${{ needs.get-version.outputs.version }} + project_type: 'python' + image_name: 'sientia-dataops-model-manager' + helm_chart_path: 'sientia-module' + helm_release_name: 'sientia-dataops-model-manager' + helm_namespace: 'sientia' + app_owner: 'Aignosi' + private_repos: 'sientia-dataops-library' + helm_repo_name: 'sientia' + helm_repo_url: 'https://raw.githubusercontent.com/Aignosi/sientia-dataops-helm-repo/refs/heads/main/' + helm_chart_version: '0.6.0' + helm_values_file: './values.yaml' + secrets: + AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + REGISTRY_LOGIN_SERVER: ${{ secrets.REGISTRY_LOGIN_SERVER }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + SUSE_RKE2_KUBE_CONFIG: ${{ secrets.SUSE_RKE2_KUBE_CONFIG }} + CI_DEPS_APP_ID: ${{ secrets.CI_DEPS_APP_ID }} + CI_DEPS_PRIVATE_KEY: ${{ secrets.CI_DEPS_PRIVATE_KEY }} + HELM_REPO_USERNAME: ${{ secrets.HELM_REPO_USERNAME }} + HELM_REPO_PASSWORD: ${{ secrets.HELM_REPO_PASSWORD }} diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 7d433b0..6a48281 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -1,237 +1,16 @@ name: Quality gate on: -# push: -# branches: -# - main pull_request: branches: - main types: [ opened, synchronize, reopened ] jobs: - sonar: - name: SonarQube Analysis - runs-on: ubuntu-latest + quality-gate: + uses: Aignosi/github_workflow_templates/.github/workflows/dataops-module-quality-gate.yml@feature/SIENTIAPDE-1352 permissions: write-all - steps: - - name: ⬇️ Checkout Code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Calculate Version - id: calculate-version - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - - // Função para calcular nova versão baseada no branch - function calculateVersion(lastVersion, branchName) { - const parseVersion = (v) => { - const match = v.match(/^(\d+)\.(\d+)\.(\d+)(?:-rc(\d+))?$/); - if (!match) throw new Error(`Invalid version format: ${v}`); - return { - major: parseInt(match[1]), - minor: parseInt(match[2]), - patch: parseInt(match[3]), - rc: match[4] ? parseInt(match[4]) : null - }; - }; - - const current = parseVersion(lastVersion); - - if (branchName.startsWith('release/')) { - return `${current.major + 1}.0.0`; - } else if (branchName.startsWith('feature/')) { - return `${current.major}.${current.minor + 1}.0`; - } else if (branchName.startsWith('fix/')) { - return `${current.major}.${current.minor}.${current.patch + 1}`; - } else if (branchName.startsWith('rc/')) { - if (current.rc !== null) { - return `${current.major}.${current.minor}.${current.patch}-rc${current.rc + 1}`; - } else { - return `${current.major}.${current.minor}.${current.patch}-rc1`; - } - } - - return null; // Não sugerir para outros tipos de branch - } - - try { - const branchName = context.payload.pull_request.head.ref; - console.log(`Branch name: ${branchName}`); - - // Validar se o branch segue os padrões aceitos - const validPrefixes = ['release/', 'feature/', 'fix/', 'rc/']; - const isValidBranch = validPrefixes.some(prefix => branchName.startsWith(prefix)); - - if (!isValidBranch) { - const errorMessage = `## 🚨 Erro: Nome do Branch Inválido\n\n` + - `O branch \`${branchName}\` não segue os padrões de nomenclatura aceitos.\n\n` + - `### 📝 Padrões Aceitos:\n` + - `- \`release/*\`: Para releases de major version (ex: release/v2.0.0)\n` + - `- \`feature/*\`: Para novas funcionalidades (ex: feature/nova-funcionalidade)\n` + - `- \`fix/*\`: Para correções de bugs (ex: fix/correcao-bug)\n` + - `- \`rc/*\`: Para release candidates (ex: rc/v1.2.0-rc1)\n\n` + - `### 🔧 Como corrigir:\n` + - `1. Renomeie o branch para seguir um dos padrões acima\n` + - `2. Ou crie um novo branch com o nome correto\n`; - - const prNumber = context.issue.number; - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: errorMessage - }); - - core.setFailed(`Invalid branch name: ${branchName}. Must start with release/, feature/, fix/, or rc/`); - return; - } - - // Obter a última tag de release - console.log('Fetching latest release tag...'); - const { data: releases } = await github.rest.repos.listReleases({ - owner: context.repo.owner, - repo: context.repo.repo, - per_page: 1 - }); - - let lastReleaseVersion = null; - let newVersion = null; - - if (releases.length > 0) { - lastReleaseVersion = releases[0].tag_name.replace(/^v/, ''); - console.log(`Latest release version: ${lastReleaseVersion}`); - // Calcular a nova versão baseada na última release - newVersion = calculateVersion(lastReleaseVersion, branchName); - } else { - console.log('No releases found, starting from 0.0.0'); - lastReleaseVersion = 'N/A'; - // Primeira release: começar com 0.0.0 independente do tipo de branch - newVersion = '0.0.0'; - } - console.log(`Calculated version: ${newVersion}`); - - // Exportar a versão como output - core.setOutput('version', newVersion); - - // Adicionar comentário informativo no PR - const prNumber = context.issue.number; - const infoMessage = `## ✅ Versão Calculada Automaticamente\n\n` + - `**Branch:** \`${branchName}\`\n` + - `**Última release:** \`${lastReleaseVersion}\`\n` + - `**Nova versão:** \`${newVersion}\`\n\n` + - `### 📝 Regras Aplicadas:\n` + - `- \`release/*\`: Aumenta major, zera minor e patch (ex: 2.0.0)\n` + - `- \`feature/*\`: Mantém major, aumenta minor, zera patch (ex: 1.2.0)\n` + - `- \`fix/*\`: Mantém major e minor, aumenta patch (ex: 1.1.3)\n` + - `- \`rc/*\`: Mantém versão base, aumenta RC (ex: 1.1.2-rc2)\n`; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: infoMessage - }); - - } catch (error) { - console.error('Error during version calculation:', error); - - const prNumber = context.issue.number; - const errorMessage = `## 🚨 Erro no Cálculo de Versão\n\n` + - `Ocorreu um erro durante o cálculo da versão:\n\n\`\`\`\n${error.message}\n\`\`\`\n\n` + - `Por favor, verifique se o nome do branch está correto e tente novamente.`; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: errorMessage - }); - - core.setFailed(`Version calculation error: ${error.message}`); - } - - - name: Generate App Token - id: generate-app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - owner: 'Aignosi' - repositories: 'sientia-dataops-library' - - - name: Prepare requirements.txt - id: prepare-requirements - run: | - sed -e "s|git+ssh://git@github.com/|git+https://github.com/|g" \ - -e "s|git@github.com:|git+https://github.com/|g" \ - requirements.txt > requirements_prepared.txt - echo "PROCESSED_REQUIREMENTS_FILE=requirements_prepared.txt" >> $GITHUB_OUTPUT - - - name: Configure Git to use App Token - env: - GH_APP_TOKEN: ${{ steps.generate-app-token.outputs.token }} - run: | - git config --global url."https://oauth2:${GH_APP_TOKEN}@github.com/".insteadOf "https://github.com/" - - - name: 🔧 Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: 💾 Cache pip packages - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements_prepared.txt', 'requirements-dev.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: 📦 Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r ${{ steps.prepare-requirements.outputs.PROCESSED_REQUIREMENTS_FILE }} - pip install -r requirements-dev.txt - - - name: 📝 Code Formatting Check (Ruff) - run: | - echo "Checking code formatting..." - ruff format --check model_manager/ tests/ - continue-on-error: false - - - name: 🔎 Code Linting (Ruff) - run: | - echo "Running linting checks..." - ruff check model_manager/ tests/ - continue-on-error: false - - - name: 🏷️ Type Checking (mypy) - run: | - echo "Running type checks..." - mypy model_manager/ - continue-on-error: true - - - name: 🔒 Security Analysis (Bandit) - run: | - echo "Running security analysis..." - bandit -r model_manager/ -ll -q - continue-on-error: true - - - name: 🧪 Run Tests with Pytest - run: | - pytest tests --junitxml=pytest.xml --cov=model_manager --cov-report=xml --cov-report=term - - - name: Run SonarQube Analysis - uses: SonarSource/sonarqube-scan-action@v6 - env: - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} - with: - args: > - -Dsonar.projectVersion=${{ steps.calculate-version.outputs.version || '0.0.0' }} + with: + project_name: 'model_manager' + repositories: 'sientia-dataops-library' + secrets: inherit diff --git a/.gitignore b/.gitignore index 2135453..571dd6e 100644 --- a/.gitignore +++ b/.gitignore @@ -244,3 +244,5 @@ git_key* git_log tmp/ sientia-module/ +.secrets +.event.json diff --git a/README.md b/README.md index 65884e4..977bfda 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,12 @@ An enterprise-grade ML model training orchestration platform built on Temporal. - [Code Quality Standards](#code-quality-standards) - [License](#license) - [Support](#support) +- [Local GitHub Actions Testing (act)](#local-github-actions-testing-act) + - [What is act?](#what-is-act) + - [Installation](#installation) + - [Configuration](#configuration-2) + - [Usage](#usage) + - [Command Reference](#command-reference) - [Docker](#docker) - [Helm Chart](#helm-chart) @@ -1172,6 +1178,176 @@ For support and questions: - Open an issue in the project repository - Contact the development team +## Local GitHub Actions Testing (act) + +### What is act? + +[act](https://nektosact.com/) is a tool that allows you to run GitHub Actions locally using Docker. This is useful for: + +- **Testing workflows** before pushing to the repository +- **Debugging issues** in workflows without creating commits +- **Speeding up development** by avoiding push/wait/check cycles +- **Saving GitHub Actions minutes** during development + +### Installation + +#### Prerequisites + +- Docker installed and running +- Go (for installation via `go install`) + +#### Installation Steps + +```bash +# 1. Update packages +sudo apt-get update + +# 2. Install Go (if not already installed) +sudo apt-get install golang + +# 3. Install act +go install github.com/nektos/act@latest + +# 4. Add Go bin to PATH +echo 'export PATH="$PATH:$HOME/go/bin"' >> ~/.bashrc +source ~/.bashrc + +# 5. Verify installation +act --version +``` + +On first run, `act` will ask which Docker image to use: +- **Large** (~17GB): Full image, compatible with almost all actions +- **Medium** (~500MB): Balanced image, compatible with most actions ✅ Recommended +- **Micro** (<200MB): Minimal image, Node.js only + +### Configuration + +#### `.secrets` File + +Create a `.secrets` file in the project root to store tokens and credentials: + +```bash +# .secrets +GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxx +SONAR_TOKEN=sqp_xxxxxxxxxxxxxxxxxxxx +SONAR_HOST_URL=https://sonarqube.example.com +CI_DEPS_APP_ID=123456 +CI_DEPS_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" +``` + +> ⚠️ **Important**: The `.secrets` file is already in `.gitignore`. Never commit this file! + +#### `.event.json` File + +Create a `.event.json` file to simulate GitHub events (e.g., pull request): + +```json +{ + "pull_request": { + "head": { + "ref": "feature/my-feature" + }, + "number": 1 + } +} +``` + +> ⚠️ **Important**: The `.event.json` file is already in `.gitignore`. Never commit this file! + +### Usage + +#### List Available Jobs + +```bash +act -l +``` + +This command lists all available workflows and jobs in the repository. + +#### Run Quality Gate Locally + +```bash +act pull_request -j quality-gate \ + --secret-file .secrets \ + --env SONAR_SCANNER_OPTS="-Dsonar.ci.autoconfig.disabled=true" \ + --eventpath .event.json +``` + +#### Run Deploy Workflow with Local Repository + +If you have workflows that reference external repositories (e.g., reusable workflows), you can map them locally: + +```bash +act pull_request \ + -e .event.json \ + --secret-file .secrets \ + -W .github/workflows/deploy.yml \ + --local-repository Aignosi/github_workflow_templates=/path/to/local/github_workflow_templates \ + --container-daemon-socket /var/run/docker.sock \ + --container-options "--user $(id -u):$(id -g)" +``` + +#### Docker Socket Permissions + +If you encounter permission issues with Docker socket: + +```bash +# Grant temporary access to Docker socket +sudo chmod 666 /var/run/docker.sock + +# Fix file ownership after running act (if needed) +sudo chown -R $USER:$USER /path/to/project +``` + +### Command Reference + +#### Command Parameters + +| Parameter | Description | +|-----------|-------------| +| `pull_request` | Event type to simulate (can be `push`, `pull_request`, `workflow_dispatch`, etc.) | +| `-j quality-gate` | Specific job name to execute (use `act -l` to see available jobs) | +| `--secret-file .secrets` | File containing secrets (tokens, credentials) | +| `--env VAR=value` | Sets environment variables for execution | +| `--eventpath .event.json` | JSON file with the simulated event payload | + +#### Special Parameter: `SONAR_SCANNER_OPTS` + +```bash +--env SONAR_SCANNER_OPTS="-Dsonar.ci.autoconfig.disabled=true" +``` + +This parameter is required because SonarQube tries to automatically detect the CI environment. When running locally with `act`, the complete GitHub Actions context is not available, causing errors. The `-Dsonar.ci.autoconfig.disabled=true` flag disables this automatic detection. + +#### Other Useful Commands + +```bash +# List all jobs +act -l + +# Run with verbose output +act pull_request -j quality-gate --secret-file .secrets -v + +# Run a push event +act push -j build --secret-file .secrets + +# Use a specific Docker image +act -P ubuntu-latest=catthehacker/ubuntu:act-latest + +# Dry-run (doesn't execute, only shows what would be done) +act -n +``` + +#### Troubleshooting + +| Problem | Solution | +|---------|----------| +| `SyntaxError: Unexpected end of JSON input` | Check if `.event.json` is properly formatted | +| `NullPointerException` in SonarQube | Add `--env SONAR_SCANNER_OPTS="-Dsonar.ci.autoconfig.disabled=true"` | +| `Not Found` when accessing GitHub API | Check if `GITHUB_TOKEN` in `.secrets` is valid | +| Job not found | Use `act -l` to see the correct job names | + ## Docker ### Create image