SIENTIAPDE-1445

Update API configuration in values.yaml and connectors_config.py; integrate PIWebAPIScouter into worker setup

scouter/utils/connectors_config.py

@@ -94,9 +94,9 @@ def build_api_config() -> dict[str, Any]:
             - auth_token: API authentication token (default: None)
     """
     return {
-        'base_url': getenv('API_BASE_URL', 'https://pi.example.com'),
+        'base_url': getenv('PI_WEB_API_BASE_URL', 'https://pi.example.com'),
-        'auth_type': getenv('API_AUTH_TYPE', 'basic'),
+        'auth_type': getenv('PI_WEB_API_AUTH_TYPE', 'basic'),
-        'auth_token': getenv('API_AUTH_TOKEN', None),
+        'auth_token': getenv('PI_WEB_API_AUTH_TOKEN', None),
     }
 
 

scouter/worker/worker.py

@@ -22,6 +22,7 @@ with workflow.unsafe.imports_passed_through():
     )
     from scouter.workflow.scouter import Scouter
     from scouter.workflow.sub_workflows.core_scouter import CoreScouter
+    from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter
 
 # Environment configuration
 POD_ID = os.getenv('HOSTNAME', 'localhost')
@@ -139,7 +140,21 @@ async def main():
                 activities.write_metrics,
                 activities.store_data_package,
             ],
-        )
+        ),
+        prepare_worker(
+            temporal_client=temporal_client,
+            main_workflow=PIWebAPIScouter,
+            other_workflows=[CoreScouter],
+            activities=[
+                activities.get_tag_values,
+                activities.data_quality_gate,
+                activities.aggregate_data,
+                activities.group_and_hold_data,
+                activities.export_data_to_postgres,
+                activities.write_metrics,
+                activities.store_data_package,
+            ],
+        ),
     ]
 
     handlers = []

tests.ipynb

@@ -0,0 +1,138 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9d16b24a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datetime import timedelta\n",
+    "from typing import Any\n",
+    "from temporalio import client\n",
+    "from temporalio.client import WorkflowHandle\n",
+    "\n",
+    "\n",
+    "async def start_workflow_advanced(\n",
+    "    temporal_client: client.Client,\n",
+    "    workflow_name: str,\n",
+    "    workflow_input: dict[str, Any],\n",
+    "    workflow_id: str | None = None,\n",
+    "    task_queue: str = \"scouter-queue\",\n",
+    "    execution_timeout: timedelta | None = None,\n",
+    "    run_timeout: timedelta | None = None,\n",
+    "    task_timeout: timedelta | None = None,\n",
+    ") -> WorkflowHandle:\n",
+    "    handle = await temporal_client.start_workflow(\n",
+    "        workflow=workflow_name,\n",
+    "        arg=workflow_input,\n",
+    "        id=workflow_id or f\"{workflow_name}-{id(workflow_input)}\",\n",
+    "        task_queue=task_queue,\n",
+    "        execution_timeout=execution_timeout,\n",
+    "        run_timeout=run_timeout,\n",
+    "        task_timeout=task_timeout,\n",
+    "    )\n",
+    "    \n",
+    "    return handle"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e344fb0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "input_data = {\n",
+    "  \"debug_data_package\": False,\n",
+    "  \"execution_timeout_seconds\": 300,\n",
+    "  \"fill_missing_tags\": False,\n",
+    "  \"filters\": {\n",
+    "    \"NULL_VALUES_FILTER\": {\n",
+    "      \"policy\": \"DISCARD\"\n",
+    "    },\n",
+    "    \"OUT_OF_BOUNDS_FILTER\": {\n",
+    "      \"policy\": \"DISCARD\"\n",
+    "    }\n",
+    "  },\n",
+    "  \"frequency\": \"30s\",\n",
+    "  \"max_retry_policy\": 1,\n",
+    "  \"model_config\": {\n",
+    "    \"predict_flavor\": \"sklearn\",\n",
+    "    \"retention_minutes\": 0,\n",
+    "    \"target\": \"CI-W3A05F1\",\n",
+    "    \"transform_flavor\": \"sklearn\"\n",
+    "  },\n",
+    "  \"model_id\": \"10\",\n",
+    "  \"model_name\": \"Pi Web API Test Model\",\n",
+    "  \"model_tags\": {\n",
+    "    \"CI-W3W03S1\": {\n",
+    "      \"aggr_func\": \"avg\",\n",
+    "      \"data_range\": [\n",
+    "        -100,\n",
+    "        100\n",
+    "      ],\n",
+    "      \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujATFUAAAUElIQVZDXENJLVczVzAzUzE\"\n",
+    "    },\n",
+    "    \"CI-W3A05F1\": {\n",
+    "      \"aggr_func\": \"lts\",\n",
+    "      \"data_range\": [\n",
+    "        -100,\n",
+    "        100\n",
+    "      ],\n",
+    "      \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE\"\n",
+    "    }\n",
+    "  },\n",
+    "  \"offset\": \"0m\",\n",
+    "  \"retention_time\": 3600,\n",
+    "  \"schedule_name\": \"pi-web-api-scouter-test\",\n",
+    "  \"schema\": \"sientia_data\",\n",
+    "  \"table_name\": \"laborious_data\",\n",
+    "  \"task_timeout_seconds\": 300,\n",
+    "  \"trigger_laborious\": False,\n",
+    "  \"updated_at\": \"2025-08-13 18:35:01.600000+0000\",\n",
+    "  \"workflow_type\": \"pi_web_api_scouter\"\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9350bff3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from temporalio import client\n",
+    "import datetime\n",
+    "\n",
+    "temporal_client = await client.Client.connect(\n",
+    "    target_host=\"localhost:7233\",\n",
+    "    namespace=\"scouter\"\n",
+    ")\n",
+    "\n",
+    "now = datetime.datetime.now()\n",
+    "\n",
+    "handle = await start_workflow_advanced(\n",
+    "    temporal_client=temporal_client,\n",
+    "    workflow_name='pi_web_api_scouter',\n",
+    "    workflow_input=input_data,\n",
+    "    workflow_id='test_workflow_id_' + now.strftime('%Y%m%d%H%M%S')\n",
+    ")\n",
+    "\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

values.yaml

@@ -163,7 +163,7 @@ env:
   - name: GITHUB_REPO_URL
     value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git"
   - name: GITHUB_BRANCH
-    value: "release/SIENTIAPDE-1441"
+    value: "feature/SIENTIAPDE-1445"
   - name: PYTHON_APP
     value: "scouter.worker.worker"
 
@@ -219,6 +219,16 @@ env:
   - name: MONGODB_DATABASE
     value: "sientia"
 
+  - name: PI_WEB_API_BASE_URL
+    value: "https://pivision.votorantimcimentos.com/piwebapi"
+  - name: PI_WEB_API_AUTH_TYPE
+    value: "basic"
+  - name: PI_WEB_API_AUTH_TOKEN
+    valueFrom:
+      secretKeyRef:
+        name: pi-web-api-auth-token
+        key: token
+
   - name: PYPI_SERVER
     value: "http://library-distribution-server.library.svc.cluster.local:5000"
 
@@ -261,3 +271,7 @@ ssh:
 #   --namespace sientia \
 #   --from-file=ssh-privatekey=git_key \
 #   --type=kubernetes.io/ssh-auth
+
+# kubectl create secret generic pi-web-api-auth-token \
+#   --namespace sientia \
+#   --from-literal=token=your-token-here