From 2b729c65e608e25b2ced53c3c1496d7d5ee8b0f9 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Tue, 2 Dec 2025 17:20:31 -0300 Subject: [PATCH 1/3] SIENTIAPDE-1352: Refactor quality gate workflow to use reusable dataops-module-quality-gate workflow and update .gitignore. ( -230, +19 lines) --- .github/workflows/quality-gate.yml | 233 +---------------------------- .gitignore | 2 + 2 files changed, 8 insertions(+), 227 deletions(-) diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 7d433b0..6a48281 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -1,237 +1,16 @@ name: Quality gate on: -# push: -# branches: -# - main pull_request: branches: - main types: [ opened, synchronize, reopened ] jobs: - sonar: - name: SonarQube Analysis - runs-on: ubuntu-latest + quality-gate: + uses: Aignosi/github_workflow_templates/.github/workflows/dataops-module-quality-gate.yml@feature/SIENTIAPDE-1352 permissions: write-all - steps: - - name: ⬇️ Checkout Code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Calculate Version - id: calculate-version - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - - // Função para calcular nova versão baseada no branch - function calculateVersion(lastVersion, branchName) { - const parseVersion = (v) => { - const match = v.match(/^(\d+)\.(\d+)\.(\d+)(?:-rc(\d+))?$/); - if (!match) throw new Error(`Invalid version format: ${v}`); - return { - major: parseInt(match[1]), - minor: parseInt(match[2]), - patch: parseInt(match[3]), - rc: match[4] ? parseInt(match[4]) : null - }; - }; - - const current = parseVersion(lastVersion); - - if (branchName.startsWith('release/')) { - return `${current.major + 1}.0.0`; - } else if (branchName.startsWith('feature/')) { - return `${current.major}.${current.minor + 1}.0`; - } else if (branchName.startsWith('fix/')) { - return `${current.major}.${current.minor}.${current.patch + 1}`; - } else if (branchName.startsWith('rc/')) { - if (current.rc !== null) { - return `${current.major}.${current.minor}.${current.patch}-rc${current.rc + 1}`; - } else { - return `${current.major}.${current.minor}.${current.patch}-rc1`; - } - } - - return null; // Não sugerir para outros tipos de branch - } - - try { - const branchName = context.payload.pull_request.head.ref; - console.log(`Branch name: ${branchName}`); - - // Validar se o branch segue os padrões aceitos - const validPrefixes = ['release/', 'feature/', 'fix/', 'rc/']; - const isValidBranch = validPrefixes.some(prefix => branchName.startsWith(prefix)); - - if (!isValidBranch) { - const errorMessage = `## 🚨 Erro: Nome do Branch Inválido\n\n` + - `O branch \`${branchName}\` não segue os padrões de nomenclatura aceitos.\n\n` + - `### 📝 Padrões Aceitos:\n` + - `- \`release/*\`: Para releases de major version (ex: release/v2.0.0)\n` + - `- \`feature/*\`: Para novas funcionalidades (ex: feature/nova-funcionalidade)\n` + - `- \`fix/*\`: Para correções de bugs (ex: fix/correcao-bug)\n` + - `- \`rc/*\`: Para release candidates (ex: rc/v1.2.0-rc1)\n\n` + - `### 🔧 Como corrigir:\n` + - `1. Renomeie o branch para seguir um dos padrões acima\n` + - `2. Ou crie um novo branch com o nome correto\n`; - - const prNumber = context.issue.number; - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: errorMessage - }); - - core.setFailed(`Invalid branch name: ${branchName}. Must start with release/, feature/, fix/, or rc/`); - return; - } - - // Obter a última tag de release - console.log('Fetching latest release tag...'); - const { data: releases } = await github.rest.repos.listReleases({ - owner: context.repo.owner, - repo: context.repo.repo, - per_page: 1 - }); - - let lastReleaseVersion = null; - let newVersion = null; - - if (releases.length > 0) { - lastReleaseVersion = releases[0].tag_name.replace(/^v/, ''); - console.log(`Latest release version: ${lastReleaseVersion}`); - // Calcular a nova versão baseada na última release - newVersion = calculateVersion(lastReleaseVersion, branchName); - } else { - console.log('No releases found, starting from 0.0.0'); - lastReleaseVersion = 'N/A'; - // Primeira release: começar com 0.0.0 independente do tipo de branch - newVersion = '0.0.0'; - } - console.log(`Calculated version: ${newVersion}`); - - // Exportar a versão como output - core.setOutput('version', newVersion); - - // Adicionar comentário informativo no PR - const prNumber = context.issue.number; - const infoMessage = `## ✅ Versão Calculada Automaticamente\n\n` + - `**Branch:** \`${branchName}\`\n` + - `**Última release:** \`${lastReleaseVersion}\`\n` + - `**Nova versão:** \`${newVersion}\`\n\n` + - `### 📝 Regras Aplicadas:\n` + - `- \`release/*\`: Aumenta major, zera minor e patch (ex: 2.0.0)\n` + - `- \`feature/*\`: Mantém major, aumenta minor, zera patch (ex: 1.2.0)\n` + - `- \`fix/*\`: Mantém major e minor, aumenta patch (ex: 1.1.3)\n` + - `- \`rc/*\`: Mantém versão base, aumenta RC (ex: 1.1.2-rc2)\n`; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: infoMessage - }); - - } catch (error) { - console.error('Error during version calculation:', error); - - const prNumber = context.issue.number; - const errorMessage = `## 🚨 Erro no Cálculo de Versão\n\n` + - `Ocorreu um erro durante o cálculo da versão:\n\n\`\`\`\n${error.message}\n\`\`\`\n\n` + - `Por favor, verifique se o nome do branch está correto e tente novamente.`; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: errorMessage - }); - - core.setFailed(`Version calculation error: ${error.message}`); - } - - - name: Generate App Token - id: generate-app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - owner: 'Aignosi' - repositories: 'sientia-dataops-library' - - - name: Prepare requirements.txt - id: prepare-requirements - run: | - sed -e "s|git+ssh://git@github.com/|git+https://github.com/|g" \ - -e "s|git@github.com:|git+https://github.com/|g" \ - requirements.txt > requirements_prepared.txt - echo "PROCESSED_REQUIREMENTS_FILE=requirements_prepared.txt" >> $GITHUB_OUTPUT - - - name: Configure Git to use App Token - env: - GH_APP_TOKEN: ${{ steps.generate-app-token.outputs.token }} - run: | - git config --global url."https://oauth2:${GH_APP_TOKEN}@github.com/".insteadOf "https://github.com/" - - - name: 🔧 Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: 💾 Cache pip packages - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements_prepared.txt', 'requirements-dev.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: 📦 Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r ${{ steps.prepare-requirements.outputs.PROCESSED_REQUIREMENTS_FILE }} - pip install -r requirements-dev.txt - - - name: 📝 Code Formatting Check (Ruff) - run: | - echo "Checking code formatting..." - ruff format --check model_manager/ tests/ - continue-on-error: false - - - name: 🔎 Code Linting (Ruff) - run: | - echo "Running linting checks..." - ruff check model_manager/ tests/ - continue-on-error: false - - - name: 🏷️ Type Checking (mypy) - run: | - echo "Running type checks..." - mypy model_manager/ - continue-on-error: true - - - name: 🔒 Security Analysis (Bandit) - run: | - echo "Running security analysis..." - bandit -r model_manager/ -ll -q - continue-on-error: true - - - name: 🧪 Run Tests with Pytest - run: | - pytest tests --junitxml=pytest.xml --cov=model_manager --cov-report=xml --cov-report=term - - - name: Run SonarQube Analysis - uses: SonarSource/sonarqube-scan-action@v6 - env: - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} - with: - args: > - -Dsonar.projectVersion=${{ steps.calculate-version.outputs.version || '0.0.0' }} + with: + project_name: 'model_manager' + repositories: 'sientia-dataops-library' + secrets: inherit diff --git a/.gitignore b/.gitignore index 2135453..571dd6e 100644 --- a/.gitignore +++ b/.gitignore @@ -244,3 +244,5 @@ git_key* git_log tmp/ sientia-module/ +.secrets +.event.json From 46df241f7d38c1f2d505c8557042898cf7faa3f0 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Tue, 2 Dec 2025 17:28:04 -0300 Subject: [PATCH 2/3] SIENTIAPDE-1352: Add documentation for local GitHub Actions testing with act. This commit adds a new section to the README.md file detailing how to use the 'act' tool for local testing of GitHub Actions workflows. It includes installation instructions, configuration details, usage examples, and troubleshooting tips. --- README.md | 150 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/README.md b/README.md index 65884e4..974ca62 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,12 @@ An enterprise-grade ML model training orchestration platform built on Temporal. - [Code Quality Standards](#code-quality-standards) - [License](#license) - [Support](#support) +- [Local GitHub Actions Testing (act)](#local-github-actions-testing-act) + - [What is act?](#what-is-act) + - [Installation](#installation) + - [Configuration](#configuration-2) + - [Usage](#usage) + - [Command Reference](#command-reference) - [Docker](#docker) - [Helm Chart](#helm-chart) @@ -1172,6 +1178,150 @@ For support and questions: - Open an issue in the project repository - Contact the development team +## Local GitHub Actions Testing (act) + +### What is act? + +[act](https://nektosact.com/) is a tool that allows you to run GitHub Actions locally using Docker. This is useful for: + +- **Testing workflows** before pushing to the repository +- **Debugging issues** in workflows without creating commits +- **Speeding up development** by avoiding push/wait/check cycles +- **Saving GitHub Actions minutes** during development + +### Installation + +#### Prerequisites + +- Docker installed and running +- Go (for installation via `go install`) + +#### Installation Steps + +```bash +# 1. Update packages +sudo apt-get update + +# 2. Install Go (if not already installed) +sudo apt-get install golang + +# 3. Install act +go install github.com/nektos/act@latest + +# 4. Add Go bin to PATH +echo 'export PATH="$PATH:$HOME/go/bin"' >> ~/.bashrc +source ~/.bashrc + +# 5. Verify installation +act --version +``` + +On first run, `act` will ask which Docker image to use: +- **Large** (~17GB): Full image, compatible with almost all actions +- **Medium** (~500MB): Balanced image, compatible with most actions ✅ Recommended +- **Micro** (<200MB): Minimal image, Node.js only + +### Configuration + +#### `.secrets` File + +Create a `.secrets` file in the project root to store tokens and credentials: + +```bash +# .secrets +GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxx +SONAR_TOKEN=sqp_xxxxxxxxxxxxxxxxxxxx +SONAR_HOST_URL=https://sonarqube.example.com +APP_ID=123456 +APP_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" +``` + +> ⚠️ **Important**: The `.secrets` file is already in `.gitignore`. Never commit this file! + +#### `.event.json` File + +Create a `.event.json` file to simulate GitHub events (e.g., pull request): + +```json +{ + "pull_request": { + "head": { + "ref": "feature/my-feature" + }, + "number": 1 + } +} +``` + +> ⚠️ **Important**: The `.event.json` file is already in `.gitignore`. Never commit this file! + +### Usage + +#### List Available Jobs + +```bash +act -l +``` + +This command lists all available workflows and jobs in the repository. + +#### Run Quality Gate Locally + +```bash +act pull_request -j quality-gate \ + --secret-file .secrets \ + --env SONAR_SCANNER_OPTS="-Dsonar.ci.autoconfig.disabled=true" \ + --eventpath .event.json +``` + +### Command Reference + +#### Command Parameters + +| Parameter | Description | +|-----------|-------------| +| `pull_request` | Event type to simulate (can be `push`, `pull_request`, `workflow_dispatch`, etc.) | +| `-j quality-gate` | Specific job name to execute (use `act -l` to see available jobs) | +| `--secret-file .secrets` | File containing secrets (tokens, credentials) | +| `--env VAR=value` | Sets environment variables for execution | +| `--eventpath .event.json` | JSON file with the simulated event payload | + +#### Special Parameter: `SONAR_SCANNER_OPTS` + +```bash +--env SONAR_SCANNER_OPTS="-Dsonar.ci.autoconfig.disabled=true" +``` + +This parameter is required because SonarQube tries to automatically detect the CI environment. When running locally with `act`, the complete GitHub Actions context is not available, causing errors. The `-Dsonar.ci.autoconfig.disabled=true` flag disables this automatic detection. + +#### Other Useful Commands + +```bash +# List all jobs +act -l + +# Run with verbose output +act pull_request -j quality-gate --secret-file .secrets -v + +# Run a push event +act push -j build --secret-file .secrets + +# Use a specific Docker image +act -P ubuntu-latest=catthehacker/ubuntu:act-latest + +# Dry-run (doesn't execute, only shows what would be done) +act -n +``` + +#### Troubleshooting + +| Problem | Solution | +|---------|----------| +| `SyntaxError: Unexpected end of JSON input` | Check if `.event.json` is properly formatted | +| `NullPointerException` in SonarQube | Add `--env SONAR_SCANNER_OPTS="-Dsonar.ci.autoconfig.disabled=true"` | +| `Not Found` when accessing GitHub API | Check if `GITHUB_TOKEN` in `.secrets` is valid | +| Job not found | Use `act -l` to see the correct job names | + ## Docker ### Create image From cdaafd12cce46d1333a31dbcb4e4968dc6bf68b6 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Thu, 4 Dec 2025 12:09:50 -0300 Subject: [PATCH 3/3] SIENTIAPDE-1352: Add deploy workflow and update README with local repository execution instructions. --- .github/workflows/deploy.yml | 52 ++++++++++++++++++++++++++++++++++++ README.md | 30 +++++++++++++++++++-- 2 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..3ce122d --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,52 @@ +name: Deploy Python Application + +on: + pull_request: + types: + - closed + branches: + - main + +jobs: + get-version: + name: Determine Next Version + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + outputs: + version: ${{ steps.version.outputs.next_version }} + steps: + - uses: actions/checkout@v4 + + - name: Determine next version + id: version + uses: Aignosi/github_workflow_templates/.github/actions/determine-next-version@main + with: + branch_name: ${{ github.event.pull_request.head.ref }} + + deploy: + name: Deploy + needs: [get-version] + uses: Aignosi/github_workflow_templates/.github/workflows/reusable-deploy.yml@feature/SIENTIAPDE-1352 + with: + version: ${{ needs.get-version.outputs.version }} + project_type: 'python' + image_name: 'sientia-dataops-model-manager' + helm_chart_path: 'sientia-module' + helm_release_name: 'sientia-dataops-model-manager' + helm_namespace: 'sientia' + app_owner: 'Aignosi' + private_repos: 'sientia-dataops-library' + helm_repo_name: 'sientia' + helm_repo_url: 'https://raw.githubusercontent.com/Aignosi/sientia-dataops-helm-repo/refs/heads/main/' + helm_chart_version: '0.6.0' + helm_values_file: './values.yaml' + secrets: + AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + REGISTRY_LOGIN_SERVER: ${{ secrets.REGISTRY_LOGIN_SERVER }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + SUSE_RKE2_KUBE_CONFIG: ${{ secrets.SUSE_RKE2_KUBE_CONFIG }} + CI_DEPS_APP_ID: ${{ secrets.CI_DEPS_APP_ID }} + CI_DEPS_PRIVATE_KEY: ${{ secrets.CI_DEPS_PRIVATE_KEY }} + HELM_REPO_USERNAME: ${{ secrets.HELM_REPO_USERNAME }} + HELM_REPO_PASSWORD: ${{ secrets.HELM_REPO_PASSWORD }} diff --git a/README.md b/README.md index 974ca62..977bfda 100644 --- a/README.md +++ b/README.md @@ -1232,8 +1232,8 @@ Create a `.secrets` file in the project root to store tokens and credentials: GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxx SONAR_TOKEN=sqp_xxxxxxxxxxxxxxxxxxxx SONAR_HOST_URL=https://sonarqube.example.com -APP_ID=123456 -APP_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" +CI_DEPS_APP_ID=123456 +CI_DEPS_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" ``` > ⚠️ **Important**: The `.secrets` file is already in `.gitignore`. Never commit this file! @@ -1274,6 +1274,32 @@ act pull_request -j quality-gate \ --eventpath .event.json ``` +#### Run Deploy Workflow with Local Repository + +If you have workflows that reference external repositories (e.g., reusable workflows), you can map them locally: + +```bash +act pull_request \ + -e .event.json \ + --secret-file .secrets \ + -W .github/workflows/deploy.yml \ + --local-repository Aignosi/github_workflow_templates=/path/to/local/github_workflow_templates \ + --container-daemon-socket /var/run/docker.sock \ + --container-options "--user $(id -u):$(id -g)" +``` + +#### Docker Socket Permissions + +If you encounter permission issues with Docker socket: + +```bash +# Grant temporary access to Docker socket +sudo chmod 666 /var/run/docker.sock + +# Fix file ownership after running act (if needed) +sudo chown -R $USER:$USER /path/to/project +``` + ### Command Reference #### Command Parameters