SIENTIAPDE-1973

SIENTIAPDE-1973: Implement per-connector OPC client certificates and remove legacy read engine. This update overhauls certificate management, transitioning from a single platform certificate to individual client certificates defined in the slot configuration. The CertProvider now downloads certificates from MinIO based on certificate_id, applying dynamic security policies (mode, policy) from the slot. The legacy READ_ENGINE=legacy option and OpcManager have been entirely removed, standardizing on the ServerReader with strict slot v2 enforcement. Write command resolution (ServerConfigResolver) now consistently uses connectors joined with opc_certificates. The CERT_BUCKET environment variable is no longer used.
2026-10-29 12:39:21 -03:00
parent d68e842e6e
commit ecb055eefb
32 changed files with 1639 additions and 2284 deletions
--- a/scripts/generate-and-push-opc-cert.sh
+++ b/scripts/generate-and-push-opc-cert.sh
@@ -1,95 +0,0 @@
-#!/usr/bin/env bash
-# Gera o certificado X.509 de cliente OPC UA do ingestor e sobe pro Gitea
-# in-cluster (repo que o pod clona no boot — nada muda no values/deployment,
-# só rollout restart depois do push).
-#
-# Uso:
-#   GITEA_PASSWORD='<senha>' ./scripts/generate-and-push-opc-cert.sh
-#
-# Vars (defaults pro cluster dev/gcp):
-#   GITEA_URL      default https://git.sientia.ai
-#   GITEA_USER     default gitea_admin
-#   GITEA_PASSWORD obrigatória
-#   GITEA_REPO     default gitea_admin/sientia-dataops-opc-ingestor
-#   BRANCH         default main
-#   APP_URI        default urn:sientia:opc-ingestor  (SAN URI do cert — TEM que
-#                  ser igual ao campo `uri` do documento na collection
-#                  OPC_servers: o ingestor usa esse valor como application_uri
-#                  e o KEPServer rejeita com BadCertificateUriInvalid se divergir)
-#   DAYS           default 3650
-#   FORCE=1        sobrescreve cert já existente no repo
-set -euo pipefail
-
-GITEA_URL=${GITEA_URL:-https://git.sientia.ai}
-GITEA_USER=${GITEA_USER:-gitea_admin}
-GITEA_PASSWORD=${GITEA_PASSWORD:?export GITEA_PASSWORD com a senha do Gitea}
-GITEA_REPO=${GITEA_REPO:-gitea_admin/sientia-dataops-opc-ingestor}
-BRANCH=${BRANCH:-main}
-APP_URI=${APP_URI:-urn:sientia:opc-ingestor}
-CN=${CN:-sientia-opc-ingestor}
-DAYS=${DAYS:-3650}
-
-CERT_DIR_IN_REPO="certs"
-CERT_NAME="opc-ingestor-cert.pem"
-KEY_NAME="opc-ingestor-key.pem"
-DER_NAME="opc-ingestor-cert.der"
-
-workdir=$(mktemp -d)
-trap 'rm -rf "$workdir"' EXIT
-
-echo ">> Gerando certificado (CN=${CN}, URI=${APP_URI}, ${DAYS} dias)..."
-# Extensões exigidas por OPC UA (KEPServer valida keyUsage/EKU e o SAN URI).
-openssl req -x509 -newkey rsa:2048 -sha256 -nodes -days "$DAYS" \
-  -keyout "$workdir/$KEY_NAME" -out "$workdir/$CERT_NAME" \
-  -subj "/CN=${CN}/O=Aignosi/OU=Sientia" \
-  -addext "subjectAltName=URI:${APP_URI},DNS:${CN}" \
-  -addext "keyUsage=critical,digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment" \
-  -addext "extendedKeyUsage=critical,clientAuth,serverAuth" \
-  -addext "basicConstraints=critical,CA:FALSE"
-
-# Cópia DER — só referência p/ import manual no KEPServer (o trust normal é
-# aceitar o cert rejeitado na 1ª conexão).
-openssl x509 -in "$workdir/$CERT_NAME" -outform der -out "$workdir/$DER_NAME"
-
-echo ">> Clonando ${GITEA_REPO}@${BRANCH} do Gitea..."
-encoded_pass=$(python3 - <<EOF
-import urllib.parse; print(urllib.parse.quote('''${GITEA_PASSWORD}''', safe=''))
-EOF
-)
-clone_url="${GITEA_URL%%/}"
-clone_url="${clone_url/:\/\//:\/\/${GITEA_USER}:${encoded_pass}@}/${GITEA_REPO}.git"
-git clone --depth 1 --branch "$BRANCH" "$clone_url" "$workdir/repo" 2>&1 | sed "s#${encoded_pass}#***#g"
-
-if [[ -f "$workdir/repo/$CERT_DIR_IN_REPO/$CERT_NAME" && "${FORCE:-0}" != "1" ]]; then
-  echo "ERRO: $CERT_DIR_IN_REPO/$CERT_NAME já existe no repo. Re-rodar com FORCE=1 sobrescreve" >&2
-  echo "      (o cert antigo deixa de valer — retrust no KEPServer necessário)." >&2
-  exit 1
-fi
-
-mkdir -p "$workdir/repo/$CERT_DIR_IN_REPO"
-cp "$workdir/$CERT_NAME" "$workdir/$KEY_NAME" "$workdir/$DER_NAME" "$workdir/repo/$CERT_DIR_IN_REPO/"
-
-cd "$workdir/repo"
-git add "$CERT_DIR_IN_REPO"
-git -c user.name="opc-cert-script" -c user.email="platform@aignosi.com.br" \
-  commit -m "Add OPC UA client certificate for KEPServer connection (URI ${APP_URI})"
-git push origin "$BRANCH" 2>&1 | sed "s#${encoded_pass}#***#g"
-
-fingerprint=$(openssl x509 -in "$CERT_DIR_IN_REPO/$CERT_NAME" -noout -fingerprint -sha1)
-
-cat <<EOF
-
-== PRONTO ==
-Cert no Gitea: ${GITEA_REPO}@${BRANCH} -> ${CERT_DIR_IN_REPO}/{${CERT_NAME},${KEY_NAME},${DER_NAME}}
-${fingerprint}
-
-Paths dentro do pod (o entrypoint clona o repo em /app):
-  cert_path:        /app/${CERT_DIR_IN_REPO}/${CERT_NAME}
-  private_key_path: /app/${CERT_DIR_IN_REPO}/${KEY_NAME}
-
-Próximos passos:
-  1. Documento na collection OPC_servers com "uri": "${APP_URI}" (ver README/PR).
-  2. kubectl -n sientia rollout restart deployment/opc-ingestor   # re-clona o repo
-  3. Pod tenta conectar -> no KEPServer: OPC UA Configuration Manager ->
-     Trusted Clients -> cert '${CN}' rejeitado -> Trust -> restart runtime.
-EOF
--- a/scripts/seed_e2e.py
+++ b/scripts/seed_e2e.py
@@ -3,7 +3,7 @@
 Populates:
  1. Mongo `connectors` + `connector_tags` (each tag with `metadata: {}` — required
     for documentation, per the feature owner).
-  2. Redis slot v1 (`slot:opc_tags:1`) so the legacy subscription path collects them.
+  2. Redis slot v2 (`slot:opc_tags:1`, contract C1) so the subscription path collects them.
  3. Postgres test predictions table (with the `write_status` column) + one row the
     NATS write test can UPDATE (correlated by timestamp + model_id).

@@ -37,41 +37,53 @@ except ImportError:

 OPC_URL = os.getenv('OPC_SERVER_URL', 'opc.tcp://localhost:49320')

+# No client certificate: the gateway connects without security (SIENTIAPDE-1974 D5).
+# To test security, point client_certificate_id at an active client certificate of
+# opc_certificates and set mode/policy (e.g. SignAndEncrypt / Basic256Sha256).
+SECURITY = {'mode': 'None', 'policy': 'None', 'client_certificate_id': None}
+
 CONNECTOR = {
-    'connector_id': 9001,
+    'connector_id': 'e2e-kepserver',
    'type': 'opc_ua',
    'name': 'e2e-kepserver',
    'url': OPC_URL,
-    'server_uri': 'urn:e2e:gateway:test',
    'subscription_period_ms': 1000,
-    'use_security': False,  # round 1; flip after certs are in sientia-dataops
+    'security': SECURITY,
    'status': 'Active',
    'description': 'OPC Gateway e2e validation connector (SIENTIAPDE-1972)',
 }

 # metadata: {} on every tag — feature owner asked test tags to carry the field.
 TAGS = [
-    {'tag_id': 9101, 'connector_id': 9001, 'address': 'ns=2;s=Simulation Examples.Functions.Ramp1',
+    {'tag_id': 9101, 'connector_id': 'e2e-kepserver', 'address': 'ns=2;s=Simulation Examples.Functions.Ramp1',
     'tag_name': 'e2e_ramp', 'category': 'Read', 'data_type': 'float', 'unit': '-',
     'status': 'Active', 'description': 'e2e read tag', 'metadata': {}},
-    {'tag_id': 9102, 'connector_id': 9001, 'address': 'ns=2;s=Simulation Examples.Functions.User1',
+    {'tag_id': 9102, 'connector_id': 'e2e-kepserver', 'address': 'ns=2;s=Simulation Examples.Functions.User1',
     'tag_name': 'e2e_write_target', 'category': 'Write', 'data_type': 'float', 'unit': '-',
     'status': 'Active', 'description': 'e2e write tag', 'metadata': {}},
 ]

-# Slot v1 (legacy path the gateway still runs until SIENTIAPDE-1974 delivers slot v2).
-SLOT_V1 = {
+# Slot v2 (contract C1), as the Orchestrator writes it for the connector above.
+SLOT = {
    'e2e-kepserver': {
        'name': 'e2e-kepserver',
-        'server_id': '9001',
+        'server_id': 'e2e-kepserver',
        'url': OPC_URL,
-        'server_uri': 'urn:e2e:gateway:test',
        'subscription_period_ms': 1000,
+        'security': {'mode': SECURITY['mode'], 'policy': SECURITY['policy']},
+        'certificate': None,
        'tags': {
            TAGS[0]['address']: {
-                'tag_name': TAGS[0]['tag_name'],
-                'frequency': '5000',
-                'topics': ['raw_e2e-gateway-test'],
+                'frequency': 5000,
+                'active_read': {'enabled': False},
+                'names': [
+                    {
+                        'collection': 'raw_e2e-gateway-test',
+                        'tag_name': TAGS[0]['tag_name'],
+                        'model_id': '9001',
+                        'model_name': 'e2e-model',
+                    }
+                ],
            }
        },
    }
@@ -94,12 +106,11 @@ ON CONFLICT DO NOTHING;

 # The matching NATS write command for the row above (paste into `nats pub opc.write '...'`).
 NATS_CMD = {
-    'msg_id': 'e2e-0001', 'server_id': '9001', 'addr': TAGS[1]['address'],
+    'msg_id': 'e2e-0001', 'server_id': 'e2e-kepserver', 'addr': TAGS[1]['address'],
    'value': 42.5, 'data_type': 'float', 'tag_type': 'prediction',
    'model_id': '9001', 'model_name': 'e2e-model',
    'schema': 'sientia_data', 'table_name': 'e2e_predictions',
    'timestamp': '2026-07-15T12:00:00Z',
-    'url': OPC_URL, 'use_security': False, 'server_uri': 'urn:e2e:gateway:test',
 }


@@ -119,8 +130,8 @@ def seed_redis():
    import redis

    r = redis.Redis(host=os.getenv('REDIS_HOST', 'localhost'), port=int(os.getenv('REDIS_PORT', '6379')))
-    r.set('slot:opc_tags:1', json.dumps(SLOT_V1))
-    print('redis: wrote slot:opc_tags:1 (v1)')
+    r.set('slot:opc_tags:1', json.dumps(SLOT))
+    print('redis: wrote slot:opc_tags:1 (v2)')


 def seed_postgres():
@@ -144,7 +155,7 @@ def main() -> int:
    if args.print_only:
        print('connector:', json.dumps(CONNECTOR, indent=2))
        print('tags:', json.dumps(TAGS, indent=2))
-        print('slot v1:', json.dumps(SLOT_V1, indent=2))
+        print('slot v2:', json.dumps(SLOT, indent=2))
        print('sql:', PREDICTIONS_SQL)
        print('nats write cmd:', json.dumps(NATS_CMD, indent=2))
        return 0