From d7d61f27a446e13c094f7117ccdf5fb8aa618b2e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Dec 2025 16:42:51 -0300 Subject: [PATCH] SIENTIAPDE-1445 Update API configuration in values.yaml and connectors_config.py; integrate PIWebAPIScouter into worker setup --- scouter/utils/connectors_config.py | 6 +- scouter/worker/worker.py | 17 +++- tests.ipynb | 138 +++++++++++++++++++++++++++++ values.yaml | 18 +++- 4 files changed, 173 insertions(+), 6 deletions(-) create mode 100644 tests.ipynb diff --git a/scouter/utils/connectors_config.py b/scouter/utils/connectors_config.py index abb1879..da712d9 100644 --- a/scouter/utils/connectors_config.py +++ b/scouter/utils/connectors_config.py @@ -94,9 +94,9 @@ def build_api_config() -> dict[str, Any]: - auth_token: API authentication token (default: None) """ return { - 'base_url': getenv('API_BASE_URL', 'https://pi.example.com'), - 'auth_type': getenv('API_AUTH_TYPE', 'basic'), - 'auth_token': getenv('API_AUTH_TOKEN', None), + 'base_url': getenv('PI_WEB_API_BASE_URL', 'https://pi.example.com'), + 'auth_type': getenv('PI_WEB_API_AUTH_TYPE', 'basic'), + 'auth_token': getenv('PI_WEB_API_AUTH_TOKEN', None), } diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 9329bbf..e7804b7 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -22,6 +22,7 @@ with workflow.unsafe.imports_passed_through(): ) from scouter.workflow.scouter import Scouter from scouter.workflow.sub_workflows.core_scouter import CoreScouter + from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter # Environment configuration POD_ID = os.getenv('HOSTNAME', 'localhost') @@ -139,7 +140,21 @@ async def main(): activities.write_metrics, activities.store_data_package, ], - ) + ), + prepare_worker( + temporal_client=temporal_client, + main_workflow=PIWebAPIScouter, + other_workflows=[CoreScouter], + activities=[ + activities.get_tag_values, + activities.data_quality_gate, + activities.aggregate_data, + activities.group_and_hold_data, + activities.export_data_to_postgres, + activities.write_metrics, + activities.store_data_package, + ], + ), ] handlers = [] diff --git a/tests.ipynb b/tests.ipynb new file mode 100644 index 0000000..607f5ad --- /dev/null +++ b/tests.ipynb @@ -0,0 +1,138 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "9d16b24a", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import timedelta\n", + "from typing import Any\n", + "from temporalio import client\n", + "from temporalio.client import WorkflowHandle\n", + "\n", + "\n", + "async def start_workflow_advanced(\n", + " temporal_client: client.Client,\n", + " workflow_name: str,\n", + " workflow_input: dict[str, Any],\n", + " workflow_id: str | None = None,\n", + " task_queue: str = \"scouter-queue\",\n", + " execution_timeout: timedelta | None = None,\n", + " run_timeout: timedelta | None = None,\n", + " task_timeout: timedelta | None = None,\n", + ") -> WorkflowHandle:\n", + " handle = await temporal_client.start_workflow(\n", + " workflow=workflow_name,\n", + " arg=workflow_input,\n", + " id=workflow_id or f\"{workflow_name}-{id(workflow_input)}\",\n", + " task_queue=task_queue,\n", + " execution_timeout=execution_timeout,\n", + " run_timeout=run_timeout,\n", + " task_timeout=task_timeout,\n", + " )\n", + " \n", + " return handle" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e344fb0", + "metadata": {}, + "outputs": [], + "source": [ + "input_data = {\n", + " \"debug_data_package\": False,\n", + " \"execution_timeout_seconds\": 300,\n", + " \"fill_missing_tags\": False,\n", + " \"filters\": {\n", + " \"NULL_VALUES_FILTER\": {\n", + " \"policy\": \"DISCARD\"\n", + " },\n", + " \"OUT_OF_BOUNDS_FILTER\": {\n", + " \"policy\": \"DISCARD\"\n", + " }\n", + " },\n", + " \"frequency\": \"30s\",\n", + " \"max_retry_policy\": 1,\n", + " \"model_config\": {\n", + " \"predict_flavor\": \"sklearn\",\n", + " \"retention_minutes\": 0,\n", + " \"target\": \"CI-W3A05F1\",\n", + " \"transform_flavor\": \"sklearn\"\n", + " },\n", + " \"model_id\": \"10\",\n", + " \"model_name\": \"Pi Web API Test Model\",\n", + " \"model_tags\": {\n", + " \"CI-W3W03S1\": {\n", + " \"aggr_func\": \"avg\",\n", + " \"data_range\": [\n", + " -100,\n", + " 100\n", + " ],\n", + " \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujATFUAAAUElIQVZDXENJLVczVzAzUzE\"\n", + " },\n", + " \"CI-W3A05F1\": {\n", + " \"aggr_func\": \"lts\",\n", + " \"data_range\": [\n", + " -100,\n", + " 100\n", + " ],\n", + " \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE\"\n", + " }\n", + " },\n", + " \"offset\": \"0m\",\n", + " \"retention_time\": 3600,\n", + " \"schedule_name\": \"pi-web-api-scouter-test\",\n", + " \"schema\": \"sientia_data\",\n", + " \"table_name\": \"laborious_data\",\n", + " \"task_timeout_seconds\": 300,\n", + " \"trigger_laborious\": False,\n", + " \"updated_at\": \"2025-08-13 18:35:01.600000+0000\",\n", + " \"workflow_type\": \"pi_web_api_scouter\"\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9350bff3", + "metadata": {}, + "outputs": [], + "source": [ + "from temporalio import client\n", + "import datetime\n", + "\n", + "temporal_client = await client.Client.connect(\n", + " target_host=\"localhost:7233\",\n", + " namespace=\"scouter\"\n", + ")\n", + "\n", + "now = datetime.datetime.now()\n", + "\n", + "handle = await start_workflow_advanced(\n", + " temporal_client=temporal_client,\n", + " workflow_name='pi_web_api_scouter',\n", + " workflow_input=input_data,\n", + " workflow_id='test_workflow_id_' + now.strftime('%Y%m%d%H%M%S')\n", + ")\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/values.yaml b/values.yaml index 56d9b71..8c690ae 100644 --- a/values.yaml +++ b/values.yaml @@ -163,7 +163,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git" - name: GITHUB_BRANCH - value: "release/SIENTIAPDE-1441" + value: "feature/SIENTIAPDE-1445" - name: PYTHON_APP value: "scouter.worker.worker" @@ -219,6 +219,16 @@ env: - name: MONGODB_DATABASE value: "sientia" + - name: PI_WEB_API_BASE_URL + value: "https://pivision.votorantimcimentos.com/piwebapi" + - name: PI_WEB_API_AUTH_TYPE + value: "basic" + - name: PI_WEB_API_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: pi-web-api-auth-token + key: token + - name: PYPI_SERVER value: "http://library-distribution-server.library.svc.cluster.local:5000" @@ -260,4 +270,8 @@ ssh: # kubectl create secret generic git-ssh-key-sientia-scouter-worker \ # --namespace sientia \ # --from-file=ssh-privatekey=git_key \ -# --type=kubernetes.io/ssh-auth \ No newline at end of file +# --type=kubernetes.io/ssh-auth + +# kubectl create secret generic pi-web-api-auth-token \ +# --namespace sientia \ +# --from-literal=token=your-token-here \ No newline at end of file