gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #16 from Aignosi/release/SIENTIAPDE-1352

Browse Source 
SIENTIAPDE-1352: Enhance CI/CD and Local Development Setup
This commit is contained in:
Bruno Domingues
2025-12-04 12:41:54 -03:00
committed by GitHub
parent 3a61633ec6 cdaafd12cc
commit 9871867b76
4 changed files with 236 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
.github/workflows/deploy.yml vendored Normal file
View File

@@ -0,0 +1,52 @@
name: Deploy Python Application
on:
pull_request:
types:
- closed
branches:
- main
jobs:
get-version:
name: Determine Next Version
runs-on: ubuntu-latest
if: github.event.pull_request.merged == true
outputs:
version: ${{ steps.version.outputs.next_version }}
steps:
- uses: actions/checkout@v4
- name: Determine next version
id: version
uses: Aignosi/github_workflow_templates/.github/actions/determine-next-version@main
with:
branch_name: ${{ github.event.pull_request.head.ref }}
deploy:
name: Deploy
needs: [get-version]
uses: Aignosi/github_workflow_templates/.github/workflows/reusable-deploy.yml@feature/SIENTIAPDE-1352
with:
version: ${{ needs.get-version.outputs.version }}
project_type: 'python'
image_name: 'sientia-dataops-model-manager'
helm_chart_path: 'sientia-module'
helm_release_name: 'sientia-dataops-model-manager'
helm_namespace: 'sientia'
app_owner: 'Aignosi'
private_repos: 'sientia-dataops-library'
helm_repo_name: 'sientia'
helm_repo_url: 'https://raw.githubusercontent.com/Aignosi/sientia-dataops-helm-repo/refs/heads/main/'
helm_chart_version: '0.6.0'
helm_values_file: './values.yaml'
secrets:
AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
REGISTRY_LOGIN_SERVER: ${{ secrets.REGISTRY_LOGIN_SERVER }}
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
SUSE_RKE2_KUBE_CONFIG: ${{ secrets.SUSE_RKE2_KUBE_CONFIG }}
CI_DEPS_APP_ID: ${{ secrets.CI_DEPS_APP_ID }}
CI_DEPS_PRIVATE_KEY: ${{ secrets.CI_DEPS_PRIVATE_KEY }}
HELM_REPO_USERNAME: ${{ secrets.HELM_REPO_USERNAME }}
HELM_REPO_PASSWORD: ${{ secrets.HELM_REPO_PASSWORD }}

233
.github/workflows/quality-gate.yml vendored
View File

@@ -1,237 +1,16 @@
name: Quality gate
on:
# push:
# branches:
# - main
pull_request:
branches:
- main
types: [ opened, synchronize, reopened ]
jobs:
sonar:
name: SonarQube Analysis
runs-on: ubuntu-latest
quality-gate:
uses: Aignosi/github_workflow_templates/.github/workflows/dataops-module-quality-gate.yml@feature/SIENTIAPDE-1352
permissions: write-all
steps:
- name: ⬇️ Checkout Code
uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
- name: Calculate Version
id: calculate-version
if: github.event_name == 'pull_request'
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
// Função para calcular nova versão baseada no branch
function calculateVersion(lastVersion, branchName) {
const parseVersion = (v) => {
const match = v.match(/^(\d+)\.(\d+)\.(\d+)(?:-rc(\d+))?$/);
if (!match) throw new Error(`Invalid version format: ${v}`);
return {
major: parseInt(match[1]),
minor: parseInt(match[2]),
patch: parseInt(match[3]),
rc: match[4] ? parseInt(match[4]) : null
};
};
const current = parseVersion(lastVersion);
if (branchName.startsWith('release/')) {
return `${current.major + 1}.0.0`;
} else if (branchName.startsWith('feature/')) {
return `${current.major}.${current.minor + 1}.0`;
} else if (branchName.startsWith('fix/')) {
return `${current.major}.${current.minor}.${current.patch + 1}`;
} else if (branchName.startsWith('rc/')) {
if (current.rc !== null) {
return `${current.major}.${current.minor}.${current.patch}-rc${current.rc + 1}`;
} else {
return `${current.major}.${current.minor}.${current.patch}-rc1`;
}
}
return null; // Não sugerir para outros tipos de branch
}
try {
const branchName = context.payload.pull_request.head.ref;
console.log(`Branch name: ${branchName}`);
// Validar se o branch segue os padrões aceitos
const validPrefixes = ['release/', 'feature/', 'fix/', 'rc/'];
const isValidBranch = validPrefixes.some(prefix => branchName.startsWith(prefix));
if (!isValidBranch) {
const errorMessage = `## 🚨 Erro: Nome do Branch Inválido\n\n` +
`O branch \`${branchName}\` não segue os padrões de nomenclatura aceitos.\n\n` +
`### 📝 Padrões Aceitos:\n` +
`- \`release/*\`: Para releases de major version (ex: release/v2.0.0)\n` +
`- \`feature/*\`: Para novas funcionalidades (ex: feature/nova-funcionalidade)\n` +
`- \`fix/*\`: Para correções de bugs (ex: fix/correcao-bug)\n` +
`- \`rc/*\`: Para release candidates (ex: rc/v1.2.0-rc1)\n\n` +
`### 🔧 Como corrigir:\n` +
`1. Renomeie o branch para seguir um dos padrões acima\n` +
`2. Ou crie um novo branch com o nome correto\n`;
const prNumber = context.issue.number;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body: errorMessage
});
core.setFailed(`Invalid branch name: ${branchName}. Must start with release/, feature/, fix/, or rc/`);
return;
}
// Obter a última tag de release
console.log('Fetching latest release tag...');
const { data: releases } = await github.rest.repos.listReleases({
owner: context.repo.owner,
repo: context.repo.repo,
per_page: 1
});
let lastReleaseVersion = null;
let newVersion = null;
if (releases.length > 0) {
lastReleaseVersion = releases[0].tag_name.replace(/^v/, '');
console.log(`Latest release version: ${lastReleaseVersion}`);
// Calcular a nova versão baseada na última release
newVersion = calculateVersion(lastReleaseVersion, branchName);
} else {
console.log('No releases found, starting from 0.0.0');
lastReleaseVersion = 'N/A';
// Primeira release: começar com 0.0.0 independente do tipo de branch
newVersion = '0.0.0';
}
console.log(`Calculated version: ${newVersion}`);
// Exportar a versão como output
core.setOutput('version', newVersion);
// Adicionar comentário informativo no PR
const prNumber = context.issue.number;
const infoMessage = `## ✅ Versão Calculada Automaticamente\n\n` +
`**Branch:** \`${branchName}\`\n` +
`**Última release:** \`${lastReleaseVersion}\`\n` +
`**Nova versão:** \`${newVersion}\`\n\n` +
`### 📝 Regras Aplicadas:\n` +
`- \`release/*\`: Aumenta major, zera minor e patch (ex: 2.0.0)\n` +
`- \`feature/*\`: Mantém major, aumenta minor, zera patch (ex: 1.2.0)\n` +
`- \`fix/*\`: Mantém major e minor, aumenta patch (ex: 1.1.3)\n` +
`- \`rc/*\`: Mantém versão base, aumenta RC (ex: 1.1.2-rc2)\n`;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body: infoMessage
});
} catch (error) {
console.error('Error during version calculation:', error);
const prNumber = context.issue.number;
const errorMessage = `## 🚨 Erro no Cálculo de Versão\n\n` +
`Ocorreu um erro durante o cálculo da versão:\n\n\`\`\`\n${error.message}\n\`\`\`\n\n` +
`Por favor, verifique se o nome do branch está correto e tente novamente.`;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body: errorMessage
});
core.setFailed(`Version calculation error: ${error.message}`);
}
- name: Generate App Token
id: generate-app-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
owner: 'Aignosi'
repositories: 'sientia-dataops-library'
- name: Prepare requirements.txt
id: prepare-requirements
run: |
sed -e "s|git+ssh://git@github.com/|git+https://github.com/|g" \
-e "s|git@github.com:|git+https://github.com/|g" \
requirements.txt > requirements_prepared.txt
echo "PROCESSED_REQUIREMENTS_FILE=requirements_prepared.txt" >> $GITHUB_OUTPUT
- name: Configure Git to use App Token
env:
GH_APP_TOKEN: ${{ steps.generate-app-token.outputs.token }}
run: |
git config --global url."https://oauth2:${GH_APP_TOKEN}@github.com/".insteadOf "https://github.com/"
- name: 🔧 Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: 💾 Cache pip packages
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('requirements_prepared.txt', 'requirements-dev.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: 📦 Install Dependencies
run: |
python -m pip install --upgrade pip
pip install -r ${{ steps.prepare-requirements.outputs.PROCESSED_REQUIREMENTS_FILE }}
pip install -r requirements-dev.txt
- name: 📝 Code Formatting Check (Ruff)
run: |
echo "Checking code formatting..."
ruff format --check model_manager/ tests/
continue-on-error: false
- name: 🔎 Code Linting (Ruff)
run: |
echo "Running linting checks..."
ruff check model_manager/ tests/
continue-on-error: false
- name: 🏷️ Type Checking (mypy)
run: |
echo "Running type checks..."
mypy model_manager/
continue-on-error: true
- name: 🔒 Security Analysis (Bandit)
run: |
echo "Running security analysis..."
bandit -r model_manager/ -ll -q
continue-on-error: true
- name: 🧪 Run Tests with Pytest
run: |
pytest tests --junitxml=pytest.xml --cov=model_manager --cov-report=xml --cov-report=term
- name: Run SonarQube Analysis
uses: SonarSource/sonarqube-scan-action@v6
env:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
with:
args: >
-Dsonar.projectVersion=${{ steps.calculate-version.outputs.version || '0.0.0' }}
with:
project_name: 'model_manager'
repositories: 'sientia-dataops-library'
secrets: inherit

2
.gitignore vendored
View File

@@ -244,3 +244,5 @@ git_key*
git_log
tmp/
sientia-module/
.secrets
.event.json

176
README.md
View File

@@ -60,6 +60,12 @@ An enterprise-grade ML model training orchestration platform built on Temporal.
- [Code Quality Standards](#code-quality-standards)
- [License](#license)
- [Support](#support)
- [Local GitHub Actions Testing (act)](#local-github-actions-testing-act)
- [What is act?](#what-is-act)
- [Installation](#installation)
- [Configuration](#configuration-2)
- [Usage](#usage)
- [Command Reference](#command-reference)
- [Docker](#docker)
- [Helm Chart](#helm-chart)
@@ -1172,6 +1178,176 @@ For support and questions:
- Open an issue in the project repository
- Contact the development team
## Local GitHub Actions Testing (act)
### What is act?
[act](https://nektosact.com/) is a tool that allows you to run GitHub Actions locally using Docker. This is useful for:
- **Testing workflows** before pushing to the repository
- **Debugging issues** in workflows without creating commits
- **Speeding up development** by avoiding push/wait/check cycles
- **Saving GitHub Actions minutes** during development
### Installation
#### Prerequisites
- Docker installed and running
- Go (for installation via `go install`)
#### Installation Steps
```bash
# 1. Update packages
sudo apt-get update
# 2. Install Go (if not already installed)
sudo apt-get install golang
# 3. Install act
go install github.com/nektos/act@latest
# 4. Add Go bin to PATH
echo 'export PATH="$PATH:$HOME/go/bin"' >> ~/.bashrc
source ~/.bashrc
# 5. Verify installation
act --version
```
On first run, `act` will ask which Docker image to use:
- **Large** (~17GB): Full image, compatible with almost all actions
- **Medium** (~500MB): Balanced image, compatible with most actions ✅ Recommended
- **Micro** (<200MB): Minimal image, Node.js only
### Configuration
#### `.secrets` File
Create a `.secrets` file in the project root to store tokens and credentials:
```bash
# .secrets
GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxx
SONAR_TOKEN=sqp_xxxxxxxxxxxxxxxxxxxx
SONAR_HOST_URL=https://sonarqube.example.com
CI_DEPS_APP_ID=123456
CI_DEPS_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----"
```
> ⚠️ **Important**: The `.secrets` file is already in `.gitignore`. Never commit this file!
#### `.event.json` File
Create a `.event.json` file to simulate GitHub events (e.g., pull request):
```json
{
"pull_request": {
"head": {
"ref": "feature/my-feature"
},
"number": 1
}
}
```
> ⚠️ **Important**: The `.event.json` file is already in `.gitignore`. Never commit this file!
### Usage
#### List Available Jobs
```bash
act -l
```
This command lists all available workflows and jobs in the repository.
#### Run Quality Gate Locally
```bash
act pull_request -j quality-gate \
--secret-file .secrets \
--env SONAR_SCANNER_OPTS="-Dsonar.ci.autoconfig.disabled=true" \
--eventpath .event.json
```
#### Run Deploy Workflow with Local Repository
If you have workflows that reference external repositories (e.g., reusable workflows), you can map them locally:
```bash
act pull_request \
-e .event.json \
--secret-file .secrets \
-W .github/workflows/deploy.yml \
--local-repository Aignosi/github_workflow_templates=/path/to/local/github_workflow_templates \
--container-daemon-socket /var/run/docker.sock \
--container-options "--user $(id -u):$(id -g)"
```
#### Docker Socket Permissions
If you encounter permission issues with Docker socket:
```bash
# Grant temporary access to Docker socket
sudo chmod 666 /var/run/docker.sock
# Fix file ownership after running act (if needed)
sudo chown -R $USER:$USER /path/to/project
```
### Command Reference
#### Command Parameters
| Parameter | Description |
|-----------|-------------|
| `pull_request` | Event type to simulate (can be `push`, `pull_request`, `workflow_dispatch`, etc.) |
| `-j quality-gate` | Specific job name to execute (use `act -l` to see available jobs) |
| `--secret-file .secrets` | File containing secrets (tokens, credentials) |
| `--env VAR=value` | Sets environment variables for execution |
| `--eventpath .event.json` | JSON file with the simulated event payload |
#### Special Parameter: `SONAR_SCANNER_OPTS`
```bash
--env SONAR_SCANNER_OPTS="-Dsonar.ci.autoconfig.disabled=true"
```
This parameter is required because SonarQube tries to automatically detect the CI environment. When running locally with `act`, the complete GitHub Actions context is not available, causing errors. The `-Dsonar.ci.autoconfig.disabled=true` flag disables this automatic detection.
#### Other Useful Commands
```bash
# List all jobs
act -l
# Run with verbose output
act pull_request -j quality-gate --secret-file .secrets -v
# Run a push event
act push -j build --secret-file .secrets
# Use a specific Docker image
act -P ubuntu-latest=catthehacker/ubuntu:act-latest
# Dry-run (doesn't execute, only shows what would be done)
act -n
```
#### Troubleshooting
| Problem | Solution |
|---------|----------|
| `SyntaxError: Unexpected end of JSON input` | Check if `.event.json` is properly formatted |
| `NullPointerException` in SonarQube | Add `--env SONAR_SCANNER_OPTS="-Dsonar.ci.autoconfig.disabled=true"` |
| `Not Found` when accessing GitHub API | Check if `GITHUB_TOKEN` in `.secrets` is valid |
| Job not found | Use `act -l` to see the correct job names |
## Docker
### Create image