Merge pull request #2 from Aignosi/SIENTIAPDE-1017-criar-helm-generico-para-os-modulos-do-sientia
Sientiapde 1017 criar helm generico para os modulos do sientia
This commit is contained in:
44
README.md
44
README.md
@@ -3,58 +3,58 @@ OPC gateway to manage Scouter pipelines
|
|||||||
|
|
||||||
## Local tests
|
## Local tests
|
||||||
### Generate your ssh key to Docker
|
### Generate your ssh key to Docker
|
||||||
'''
|
```
|
||||||
ssh-keygen -t ed25519 -C "docker-access" -f ~/.ssh/id_ed25519_docker
|
ssh-keygen -t ed25519 -C "docker-access" -f ~/.ssh/id_ed25519_docker
|
||||||
'''
|
```
|
||||||
Add the public key to yout Git SSH keys
|
Add the public key to yout Git SSH keys
|
||||||
|
|
||||||
### Enable Docker BuildKit
|
### Enable Docker BuildKit
|
||||||
'''
|
```
|
||||||
export DOCKER_BUILDKIT=1
|
export DOCKER_BUILDKIT=1
|
||||||
'''
|
```
|
||||||
or make it permanent:
|
or make it permanent:
|
||||||
'''
|
```
|
||||||
echo '{ "features": { "buildkit": true } }' | sudo tee /etc/docker/daemon.json
|
echo '{ "features": { "buildkit": true } }' | sudo tee /etc/docker/daemon.json
|
||||||
sudo systemctl restart docker
|
sudo systemctl restart docker
|
||||||
'''
|
```
|
||||||
|
|
||||||
### Run docker compose
|
### Run docker compose
|
||||||
'''
|
```
|
||||||
docker compose down -v
|
docker compose down -v
|
||||||
docker compose build --ssh default=$HOME/.ssh/id_ed25519_docker
|
docker compose build --ssh default=$HOME/.ssh/id_ed25519_docker
|
||||||
docker compose up -d
|
docker compose up -d
|
||||||
'''
|
```
|
||||||
|
|
||||||
### Populate redis server
|
### Populate redis server
|
||||||
Create venv with python3.11
|
Create venv with python3.11
|
||||||
'''
|
```
|
||||||
python3.11 -m venv venv
|
python3.11 -m venv venv
|
||||||
source ./venv/bin/activate
|
source ./venv/bin/activate
|
||||||
'''
|
```
|
||||||
Install requirements
|
Install requirements
|
||||||
'''
|
```
|
||||||
pip install -r requirements.txt
|
pip install -r requirements.txt
|
||||||
'''
|
```
|
||||||
Run feeder
|
Run feeder
|
||||||
'''
|
```
|
||||||
python simulator/redis-feeder.py
|
python simulator/redis-feeder.py
|
||||||
'''
|
```
|
||||||
|
|
||||||
## Unit tests
|
## Unit tests
|
||||||
### Install pytest
|
### Install pytest
|
||||||
'''
|
```
|
||||||
pip install pytest
|
pip install pytest
|
||||||
'''
|
```
|
||||||
### Run pytest
|
### Run pytest
|
||||||
'''
|
```
|
||||||
pytest
|
pytest
|
||||||
'''
|
```
|
||||||
### Get current coverage
|
### Get current coverage
|
||||||
'''
|
```
|
||||||
pip install pytest-cov
|
pip install pytest-cov
|
||||||
pytest --cov=ingestor
|
pytest --cov=ingestor
|
||||||
'''
|
```
|
||||||
### Generate complete report
|
### Generate complete report
|
||||||
'''
|
```
|
||||||
pytest --cov=ingestor --cov-report=html
|
pytest --cov=ingestor --cov-report=html
|
||||||
'''
|
```
|
||||||
23
helm/.helmignore
Normal file
23
helm/.helmignore
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
# Patterns to ignore when building packages.
|
||||||
|
# This supports shell glob matching, relative path matching, and
|
||||||
|
# negation (prefixed with !). Only one pattern per line.
|
||||||
|
.DS_Store
|
||||||
|
# Common VCS dirs
|
||||||
|
.git/
|
||||||
|
.gitignore
|
||||||
|
.bzr/
|
||||||
|
.bzrignore
|
||||||
|
.hg/
|
||||||
|
.hgignore
|
||||||
|
.svn/
|
||||||
|
# Common backup files
|
||||||
|
*.swp
|
||||||
|
*.bak
|
||||||
|
*.tmp
|
||||||
|
*.orig
|
||||||
|
*~
|
||||||
|
# Various IDEs
|
||||||
|
.project
|
||||||
|
.idea/
|
||||||
|
*.tmproj
|
||||||
|
.vscode/
|
||||||
26
helm/Chart.yaml
Normal file
26
helm/Chart.yaml
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
name: sientia-module
|
||||||
|
description: A generic application helm for Kubernetes. Contains a set of templates
|
||||||
|
that can be used to deploy a generic sientia python module to a Kubernetes cluster,
|
||||||
|
based in some git repository and a generic docker image.
|
||||||
|
|
||||||
|
# A chart can be either an 'application' or a 'library' chart.
|
||||||
|
#
|
||||||
|
# Application charts are a collection of templates that can be packaged into versioned archives
|
||||||
|
# to be deployed.
|
||||||
|
#
|
||||||
|
# Library charts provide useful utilities or functions for the chart developer. They're included as
|
||||||
|
# a dependency of application charts to inject those utilities and functions into the rendering
|
||||||
|
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
|
||||||
|
type: application
|
||||||
|
|
||||||
|
# This is the chart version. This version number should be incremented each time you make changes
|
||||||
|
# to the chart and its templates, including the app version.
|
||||||
|
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||||
|
version: 0.1.0
|
||||||
|
|
||||||
|
# This is the version number of the application being deployed. This version number should be
|
||||||
|
# incremented each time you make changes to the application. Versions are not expected to
|
||||||
|
# follow Semantic Versioning. They should reflect the version the application is using.
|
||||||
|
# It is recommended to use it with quotes.
|
||||||
|
appVersion: "1.16.0"
|
||||||
73
helm/README.md
Normal file
73
helm/README.md
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
# How to create an image and deploy this helm chart:
|
||||||
|
|
||||||
|
## Image creation:
|
||||||
|
### Generate your ssh key to Docker
|
||||||
|
```
|
||||||
|
ssh-keygen -t ed25519 -C "docker-access" -f ~/.ssh/id_ed25519_docker
|
||||||
|
```
|
||||||
|
|
||||||
|
Add the public key to yout Git SSH keys in your git platform
|
||||||
|
|
||||||
|
### Enable Docker BuildKit
|
||||||
|
```
|
||||||
|
export DOCKER_BUILDKIT=1
|
||||||
|
```
|
||||||
|
or make it permanent:
|
||||||
|
```
|
||||||
|
echo '{ "features": { "buildkit": true } }' | sudo tee /etc/docker/daemon.json
|
||||||
|
sudo systemctl restart docker
|
||||||
|
```
|
||||||
|
|
||||||
|
### Build and upload image
|
||||||
|
```
|
||||||
|
make -C docker
|
||||||
|
```
|
||||||
|
|
||||||
|
## SSH secrets:
|
||||||
|
|
||||||
|
### Create secret to your git ssh credentials
|
||||||
|
We need a dedicated keypair to access GitHub from a Kubernetes pod without exposing your user credentials or tokens.
|
||||||
|
```
|
||||||
|
ssh-keygen -t ed25519 -f git_key -C "k8s-deploy-key" -N ""
|
||||||
|
```
|
||||||
|
- ```-t ed25519```: modern, secure key type
|
||||||
|
|
||||||
|
- ```-f git_key```: saves to git_key (private) and git_key.pub (public)
|
||||||
|
|
||||||
|
- ```-C "..."```: comment to identify this key
|
||||||
|
|
||||||
|
- ```-N ""```: no passphrase (for non-interactive use in pods)
|
||||||
|
|
||||||
|
### Add the Public Key to GitHub (Read-Only Deploy Key)
|
||||||
|
Deploy keys allow read-only access to a specific repo. This avoids using user tokens or giving broad permissions.
|
||||||
|
|
||||||
|
1. Copy your public key:
|
||||||
|
```
|
||||||
|
cat git_key.pub
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Go to your GitHub repo → Settings → Deploy Keys
|
||||||
|
|
||||||
|
3. Click “Add deploy key”
|
||||||
|
|
||||||
|
- Title: K8s ReadOnly
|
||||||
|
|
||||||
|
- Key: paste your git_key.pub
|
||||||
|
|
||||||
|
- ✅ Check “Allow read access”
|
||||||
|
|
||||||
|
- ❌ Do NOT check “Allow write access”
|
||||||
|
|
||||||
|
### Create a Kubernetes Secret from the Private Key
|
||||||
|
Kubernetes Secrets securely store sensitive data like private keys. This secret will later be mounted into your pod for Git to use.
|
||||||
|
```
|
||||||
|
kubectl create secret generic git-ssh-key \
|
||||||
|
--namespace sientia-opc \
|
||||||
|
--from-file=ssh-privatekey=git_key \
|
||||||
|
--type=kubernetes.io/ssh-auth
|
||||||
|
```
|
||||||
|
- ```ssh-privatekey```: required key name for type kubernetes.io/ssh-auth
|
||||||
|
|
||||||
|
- Secret is base64-encoded and stored in Kubernetes (not encrypted unless you're using an external secret manager or encryption at rest is enabled)
|
||||||
|
|
||||||
|
|
||||||
3
helm/files/github_known_hosts
Normal file
3
helm/files/github_known_hosts
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
|
||||||
|
github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
|
||||||
|
github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
|
||||||
62
helm/templates/_helpers.tpl
Normal file
62
helm/templates/_helpers.tpl
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
{{/*
|
||||||
|
Expand the name of the chart.
|
||||||
|
*/}}
|
||||||
|
{{- define "sientia-module.name" -}}
|
||||||
|
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Create a default fully qualified app name.
|
||||||
|
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||||
|
If release name contains chart name it will be used as a full name.
|
||||||
|
*/}}
|
||||||
|
{{- define "sientia-module.fullname" -}}
|
||||||
|
{{- if .Values.fullnameOverride }}
|
||||||
|
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- else }}
|
||||||
|
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||||
|
{{- if contains $name .Release.Name }}
|
||||||
|
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- else }}
|
||||||
|
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Create chart name and version as used by the chart label.
|
||||||
|
*/}}
|
||||||
|
{{- define "sientia-module.chart" -}}
|
||||||
|
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Common labels
|
||||||
|
*/}}
|
||||||
|
{{- define "sientia-module.labels" -}}
|
||||||
|
helm.sh/chart: {{ include "sientia-module.chart" . }}
|
||||||
|
{{ include "sientia-module.selectorLabels" . }}
|
||||||
|
{{- if .Chart.AppVersion }}
|
||||||
|
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||||
|
{{- end }}
|
||||||
|
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Selector labels
|
||||||
|
*/}}
|
||||||
|
{{- define "sientia-module.selectorLabels" -}}
|
||||||
|
app.kubernetes.io/name: {{ include "sientia-module.name" . }}
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Create the name of the service account to use
|
||||||
|
*/}}
|
||||||
|
{{- define "sientia-module.serviceAccountName" -}}
|
||||||
|
{{- if .Values.serviceAccount.create }}
|
||||||
|
{{- default (include "sientia-module.fullname" .) .Values.serviceAccount.name }}
|
||||||
|
{{- else }}
|
||||||
|
{{- default "default" .Values.serviceAccount.name }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
96
helm/templates/deployment.yaml
Normal file
96
helm/templates/deployment.yaml
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: {{ .Values.nameOverride | default (include "sientia-module.fullname" .) }}
|
||||||
|
labels:
|
||||||
|
{{- include "sientia-module.labels" . | nindent 4 }}
|
||||||
|
spec:
|
||||||
|
{{- if not .Values.autoscaling.enabled }}
|
||||||
|
replicas: {{ .Values.replicaCount }}
|
||||||
|
{{- end }}
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
{{- include "sientia-module.selectorLabels" . | nindent 6 }}
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
{{- with .Values.podAnnotations }}
|
||||||
|
annotations:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
labels:
|
||||||
|
{{- include "sientia-module.labels" . | nindent 8 }}
|
||||||
|
{{- with .Values.podLabels }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
{{- with .Values.imagePullSecrets }}
|
||||||
|
imagePullSecrets:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
serviceAccountName: {{ include "sientia-module.serviceAccountName" . }}
|
||||||
|
{{- with .Values.podSecurityContext }}
|
||||||
|
securityContext:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
containers:
|
||||||
|
- name: {{ .Chart.Name }}
|
||||||
|
{{- with .Values.securityContext }}
|
||||||
|
securityContext:
|
||||||
|
{{- toYaml . | nindent 12 }}
|
||||||
|
{{- end }}
|
||||||
|
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
|
||||||
|
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: {{ .Values.service.port }}
|
||||||
|
protocol: TCP
|
||||||
|
env:
|
||||||
|
{{- toYaml .Values.env | nindent 12 }}
|
||||||
|
{{- with .Values.livenessProbe }}
|
||||||
|
livenessProbe:
|
||||||
|
{{- toYaml . | nindent 12 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.readinessProbe }}
|
||||||
|
readinessProbe:
|
||||||
|
{{- toYaml . | nindent 12 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.resources }}
|
||||||
|
resources:
|
||||||
|
{{- toYaml . | nindent 12 }}
|
||||||
|
{{- end }}
|
||||||
|
volumeMounts:
|
||||||
|
{{- if .Values.ssh.enabled }}
|
||||||
|
- name: ssh-key
|
||||||
|
mountPath: {{ .Values.ssh.sshPath }}
|
||||||
|
readOnly: true
|
||||||
|
- name: ssh-known-hosts
|
||||||
|
mountPath: {{ .Values.ssh.knownHostsPath }}
|
||||||
|
readOnly: true
|
||||||
|
{{- end }}
|
||||||
|
volumes:
|
||||||
|
{{- if .Values.ssh.enabled }}
|
||||||
|
- name: ssh-key
|
||||||
|
secret:
|
||||||
|
secretName: {{ .Values.ssh.secretName }}
|
||||||
|
items:
|
||||||
|
- key: ssh-privatekey
|
||||||
|
path: id_ed25519
|
||||||
|
- name: ssh-known-hosts
|
||||||
|
configMap:
|
||||||
|
name: {{ include "sientia-module.fullname" . }}-ssh-known-hosts
|
||||||
|
items:
|
||||||
|
- key: known_hosts
|
||||||
|
path: known_hosts
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.nodeSelector }}
|
||||||
|
nodeSelector:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.affinity }}
|
||||||
|
affinity:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.tolerations }}
|
||||||
|
tolerations:
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
32
helm/templates/hpa.yaml
Normal file
32
helm/templates/hpa.yaml
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
{{- if .Values.autoscaling.enabled }}
|
||||||
|
apiVersion: autoscaling/v2
|
||||||
|
kind: HorizontalPodAutoscaler
|
||||||
|
metadata:
|
||||||
|
name: {{ include "sientia-module.fullname" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "sientia-module.labels" . | nindent 4 }}
|
||||||
|
spec:
|
||||||
|
scaleTargetRef:
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
name: {{ include "sientia-module.fullname" . }}
|
||||||
|
minReplicas: {{ .Values.autoscaling.minReplicas }}
|
||||||
|
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
|
||||||
|
metrics:
|
||||||
|
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
|
||||||
|
- type: Resource
|
||||||
|
resource:
|
||||||
|
name: cpu
|
||||||
|
target:
|
||||||
|
type: Utilization
|
||||||
|
averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
|
||||||
|
- type: Resource
|
||||||
|
resource:
|
||||||
|
name: memory
|
||||||
|
target:
|
||||||
|
type: Utilization
|
||||||
|
averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
7
helm/templates/know-hosts-configmap.yaml
Normal file
7
helm/templates/know-hosts-configmap.yaml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: {{ include "sientia-module.fullname" . }}-ssh-known-hosts
|
||||||
|
data:
|
||||||
|
known_hosts: |
|
||||||
|
{{ .Files.Get "files/github_known_hosts" | indent 4 }}
|
||||||
15
helm/templates/service.yaml
Normal file
15
helm/templates/service.yaml
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: {{ include "sientia-module.fullname" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "sientia-module.labels" . | nindent 4 }}
|
||||||
|
spec:
|
||||||
|
type: {{ .Values.service.type }}
|
||||||
|
ports:
|
||||||
|
- port: {{ .Values.service.port }}
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
|
name: http
|
||||||
|
selector:
|
||||||
|
{{- include "sientia-module.selectorLabels" . | nindent 4 }}
|
||||||
13
helm/templates/serviceaccount.yaml
Normal file
13
helm/templates/serviceaccount.yaml
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
{{- if .Values.serviceAccount.create -}}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: {{ include "sientia-module.serviceAccountName" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "sientia-module.labels" . | nindent 4 }}
|
||||||
|
{{- with .Values.serviceAccount.annotations }}
|
||||||
|
annotations:
|
||||||
|
{{- toYaml . | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
|
||||||
|
{{- end }}
|
||||||
162
helm/values.yaml
Normal file
162
helm/values.yaml
Normal file
@@ -0,0 +1,162 @@
|
|||||||
|
# Default values for sientia-module.
|
||||||
|
# This is a YAML-formatted file.
|
||||||
|
# Declare variables to be passed into your templates.
|
||||||
|
|
||||||
|
# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
|
||||||
|
replicaCount: 1
|
||||||
|
|
||||||
|
# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
|
||||||
|
image:
|
||||||
|
repository: aignosi.azurecr.io/sientia-module
|
||||||
|
# This sets the pull policy for images.
|
||||||
|
pullPolicy: Always
|
||||||
|
# Overrides the image tag whose default is the chart appVersion.
|
||||||
|
tag: "0.0.1"
|
||||||
|
|
||||||
|
# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: docker-hub-secret
|
||||||
|
# This is to override the chart name.
|
||||||
|
nameOverride: "sientia-opc-ingestor"
|
||||||
|
fullnameOverride: "sientia-opc-ingestor"
|
||||||
|
namespace: sientia-opc
|
||||||
|
|
||||||
|
# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
|
||||||
|
serviceAccount:
|
||||||
|
# Specifies whether a service account should be created
|
||||||
|
create: true
|
||||||
|
# Automatically mount a ServiceAccount's API credentials?
|
||||||
|
automount: true
|
||||||
|
# Annotations to add to the service account
|
||||||
|
annotations: {}
|
||||||
|
# The name of the service account to use.
|
||||||
|
# If not set and create is true, a name is generated using the fullname template
|
||||||
|
name: "sientia-opc-ingestor"
|
||||||
|
|
||||||
|
# This is for setting Kubernetes Annotations to a Pod.
|
||||||
|
# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
|
||||||
|
podAnnotations: {}
|
||||||
|
# This is for setting Kubernetes Labels to a Pod.
|
||||||
|
# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
|
||||||
|
podLabels: {}
|
||||||
|
|
||||||
|
podSecurityContext: {}
|
||||||
|
# fsGroup: 2000
|
||||||
|
|
||||||
|
securityContext: {}
|
||||||
|
# capabilities:
|
||||||
|
# drop:
|
||||||
|
# - ALL
|
||||||
|
# readOnlyRootFilesystem: true
|
||||||
|
# runAsNonRoot: true
|
||||||
|
# runAsUser: 1000
|
||||||
|
|
||||||
|
|
||||||
|
resources: {}
|
||||||
|
# We usually recommend not to specify default resources and to leave this as a conscious
|
||||||
|
# choice for the user. This also increases chances charts run on environments with little
|
||||||
|
# resources, such as Minikube. If you do want to specify resources, uncomment the following
|
||||||
|
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
|
||||||
|
# limits:
|
||||||
|
# cpu: 100m
|
||||||
|
# memory: 128Mi
|
||||||
|
# requests:
|
||||||
|
# cpu: 100m
|
||||||
|
# memory: 128Mi
|
||||||
|
|
||||||
|
# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- -c
|
||||||
|
- pgrep -f "ingestor.app"
|
||||||
|
initialDelaySeconds: 20
|
||||||
|
periodSeconds: 30
|
||||||
|
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- -c
|
||||||
|
- pgrep -f "ingestor.app"
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 15
|
||||||
|
|
||||||
|
|
||||||
|
# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/
|
||||||
|
autoscaling:
|
||||||
|
enabled: false
|
||||||
|
minReplicas: 1
|
||||||
|
maxReplicas: 100
|
||||||
|
targetCPUUtilizationPercentage: 80
|
||||||
|
# targetMemoryUtilizationPercentage: 80
|
||||||
|
|
||||||
|
# Additional volumes on the output Deployment definition.
|
||||||
|
volumes: []
|
||||||
|
# - name: foo
|
||||||
|
# secret:
|
||||||
|
# secretName: mysecret
|
||||||
|
# optional: false
|
||||||
|
|
||||||
|
# Additional volumeMounts on the output Deployment definition.
|
||||||
|
volumeMounts: []
|
||||||
|
# - name: foo
|
||||||
|
# mountPath: "/etc/foo"
|
||||||
|
# readOnly: true
|
||||||
|
|
||||||
|
nodeSelector: {}
|
||||||
|
|
||||||
|
tolerations: []
|
||||||
|
|
||||||
|
affinity: {}
|
||||||
|
|
||||||
|
service:
|
||||||
|
enabled: false
|
||||||
|
type: ClusterIP
|
||||||
|
port: 4840
|
||||||
|
targetPort: 4840
|
||||||
|
|
||||||
|
|
||||||
|
env:
|
||||||
|
# Entrypoint variables
|
||||||
|
- name: GITHUB_REPO_URL
|
||||||
|
value: "git@github.com:Aignosi/sientia-dataops-opc-ingestor.git"
|
||||||
|
- name: GITHUB_BRANCH
|
||||||
|
value: "SIENTIAPDE-1017-criar-helm-generico-para-os-modulos-do-sientia"
|
||||||
|
- name: PYTHON_APP
|
||||||
|
value: "ingestor.app"
|
||||||
|
|
||||||
|
# Application variables
|
||||||
|
- name: KAFKA_SERVERS
|
||||||
|
value: "kafka.kafka.svc.cluster.local:9092"
|
||||||
|
- name: REDIS_HOST
|
||||||
|
value: "redis-master.redis.svc.cluster.local"
|
||||||
|
- name: REDIS_PORT
|
||||||
|
value: "6379"
|
||||||
|
- name: REDIS_USERNAME
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: redis
|
||||||
|
key: redis-username
|
||||||
|
- name: REDIS_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: redis
|
||||||
|
key: redis-password
|
||||||
|
- name: LEASE_TTL
|
||||||
|
value: "10"
|
||||||
|
- name: HEARTBEAT_TTL
|
||||||
|
value: "20"
|
||||||
|
- name: POLL_INTERVAL
|
||||||
|
value: "5"
|
||||||
|
|
||||||
|
|
||||||
|
ssh:
|
||||||
|
enabled: true
|
||||||
|
secretName: git-ssh-key-temp
|
||||||
|
sshPath: /mnt/.ssh
|
||||||
|
knownHostsPath: /mnt/known_hosts
|
||||||
|
|
||||||
|
# kubectl create secret docker-registry docker-hub-secret --namespace sientia-opc --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp
|
||||||
|
# helm upgrade --install sientia-dataops-opc-ingestor ./helm -n sientia-opc --create-namespace
|
||||||
@@ -29,6 +29,8 @@ class Ingestor:
|
|||||||
kafka_servers = getenv("KAFKA_SERVERS", "localhost:9092")
|
kafka_servers = getenv("KAFKA_SERVERS", "localhost:9092")
|
||||||
self.redis_host = getenv("REDIS_HOST", "localhost")
|
self.redis_host = getenv("REDIS_HOST", "localhost")
|
||||||
self.redis_port = int(getenv("REDIS_PORT", 6379))
|
self.redis_port = int(getenv("REDIS_PORT", 6379))
|
||||||
|
self.redis_username = getenv("REDIS_USERNAME", None)
|
||||||
|
self.redis_password = getenv("REDIS_PASSWORD", None)
|
||||||
self.lease_ttl = int(getenv("LEASE_TTL", 10))
|
self.lease_ttl = int(getenv("LEASE_TTL", 10))
|
||||||
self.heartbeat_ttl = int(getenv("HEARTBEAT_TTL", 20))
|
self.heartbeat_ttl = int(getenv("HEARTBEAT_TTL", 20))
|
||||||
self.pod_id = getenv("HOSTNAME", "localhost")
|
self.pod_id = getenv("HOSTNAME", "localhost")
|
||||||
@@ -103,7 +105,7 @@ class Ingestor:
|
|||||||
|
|
||||||
self.ingestor_manager = IngestorManager(
|
self.ingestor_manager = IngestorManager(
|
||||||
self.kafka_servers, self.redis_host, self.redis_port, self.lease_ttl,
|
self.kafka_servers, self.redis_host, self.redis_port, self.lease_ttl,
|
||||||
self.heartbeat_ttl, self.pod_id, self.poll_interval, self.logger
|
self.heartbeat_ttl, self.pod_id, self.poll_interval, self.logger, self.redis_username, self.redis_password
|
||||||
)
|
)
|
||||||
|
|
||||||
# Declare ingestor ative
|
# Declare ingestor ative
|
||||||
|
|||||||
@@ -11,12 +11,13 @@ class IngestorManager():
|
|||||||
def __init__(self,
|
def __init__(self,
|
||||||
kafka_servers: str, redis_host: str, redis_port: int,
|
kafka_servers: str, redis_host: str, redis_port: int,
|
||||||
lease_ttl: int, heartbeat_ttl: int, pod_id: str,
|
lease_ttl: int, heartbeat_ttl: int, pod_id: str,
|
||||||
poll_interval: int, logger: Logger):
|
poll_interval: int, logger: Logger,
|
||||||
|
redis_username: str = None, redis_password: str = None):
|
||||||
|
|
||||||
self.data_manager = DataManager(kafka_servers, logger)
|
self.data_manager = DataManager(kafka_servers, logger)
|
||||||
self.opc_managers = {}
|
self.opc_managers = {}
|
||||||
self.resource_manager = ResourceManager(
|
self.resource_manager = ResourceManager(
|
||||||
redis_host, redis_port, lease_ttl, heartbeat_ttl, pod_id
|
redis_host, redis_port, lease_ttl, heartbeat_ttl, pod_id, redis_username, redis_password
|
||||||
)
|
)
|
||||||
self.number_of_slots = 0
|
self.number_of_slots = 0
|
||||||
self.poll_interval = poll_interval
|
self.poll_interval = poll_interval
|
||||||
@@ -44,6 +45,8 @@ class IngestorManager():
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
self.logger.info(
|
||||||
|
f"Initializing OpcManager at {server_config['url']}")
|
||||||
manager = OpcManager(
|
manager = OpcManager(
|
||||||
server_config['name'], server_config['url'], data_manager, logger, server_config['server_uri'],
|
server_config['name'], server_config['url'], data_manager, logger, server_config['server_uri'],
|
||||||
server_config.get('cert_path'), server_config.get(
|
server_config.get('cert_path'), server_config.get(
|
||||||
@@ -54,6 +57,7 @@ class IngestorManager():
|
|||||||
manager.connect()
|
manager.connect()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Failed to initialize OpcManager: {e}")
|
logger.error(f"Failed to initialize OpcManager: {e}")
|
||||||
|
traceback.print_exc()
|
||||||
return None
|
return None
|
||||||
|
|
||||||
return manager
|
return manager
|
||||||
|
|||||||
@@ -5,8 +5,10 @@ from redis import Redis
|
|||||||
|
|
||||||
class ResourceManager:
|
class ResourceManager:
|
||||||
def __init__(self, host: str, port: int,
|
def __init__(self, host: str, port: int,
|
||||||
lease_ttl: int, heartbeat_ttl: int, pod_id: str) -> None:
|
lease_ttl: int, heartbeat_ttl: int, pod_id: str,
|
||||||
self.redis = Redis(host=host, port=port, decode_responses=True)
|
username: str = None, password: str = None) -> None:
|
||||||
|
self.redis = Redis(host=host, port=port, decode_responses=True,
|
||||||
|
username=username, password=password)
|
||||||
self.lease_ttl = lease_ttl
|
self.lease_ttl = lease_ttl
|
||||||
self.heartbeat_ttl = heartbeat_ttl
|
self.heartbeat_ttl = heartbeat_ttl
|
||||||
self.pod_id = pod_id
|
self.pod_id = pod_id
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ redis_host = "localhost"
|
|||||||
redis_port = 6379
|
redis_port = 6379
|
||||||
|
|
||||||
# Connect to Redis
|
# Connect to Redis
|
||||||
r = redis.Redis(host=redis_host, port=redis_port, decode_responses=True)
|
r = redis.Redis(host=redis_host, port=redis_port,
|
||||||
|
decode_responses=True, username='default', password='bdnZOpcyiL')
|
||||||
|
|
||||||
# Define the key pattern to target
|
# Define the key pattern to target
|
||||||
pattern = "slot:opc_tags:*"
|
pattern = "slot:opc_tags:*"
|
||||||
@@ -24,7 +25,7 @@ new_data = {
|
|||||||
"slot:opc_tags:1": {
|
"slot:opc_tags:1": {
|
||||||
"server1": {
|
"server1": {
|
||||||
"name": "server1",
|
"name": "server1",
|
||||||
"url": "opc.tcp://simulator:4840",
|
"url": "opc.tcp://sientia-opc-simulator-service.sientia-opc.svc.cluster.local:4840",
|
||||||
"server_uri": "http://opcua-server.simulator",
|
"server_uri": "http://opcua-server.simulator",
|
||||||
"tags": {
|
"tags": {
|
||||||
'ns=2;i=2': {
|
'ns=2;i=2': {
|
||||||
@@ -44,30 +45,6 @@ new_data = {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
"slot:opc_tags:2": {
|
|
||||||
"server2": {
|
|
||||||
"name": "server2",
|
|
||||||
"url": "opc.tcp://simulator:4840",
|
|
||||||
"server_uri": "http://opcua-server.simulator",
|
|
||||||
"tags": {
|
|
||||||
'ns=2;i=2': {
|
|
||||||
'tag_name': 'Counter',
|
|
||||||
'frequency': 1000,
|
|
||||||
'topics': ['opcua2', 'counter'],
|
|
||||||
},
|
|
||||||
'ns=2;i=3': {
|
|
||||||
'tag_name': 'Rollout',
|
|
||||||
'frequency': 1000,
|
|
||||||
"topics": ['opcua2', 'rollout'],
|
|
||||||
},
|
|
||||||
'ns=2;i=4': {
|
|
||||||
'tag_name': 'Square',
|
|
||||||
'frequency': 1000,
|
|
||||||
"topics": ['opcua2'],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user