gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
dc38309348365adfcc0f4d60b5402b0402fdda6f
sientia-dataops-scouter_tem…/tests.ipynb
vitor-aignosi dc38309348 SIENTIAPDE-1478 
Update tests.ipynb and prepare_worker.py

- Reset execution counts in tests.ipynb for reproducibility.
- Removed error outputs and added new tags in TAG_NAMES for enhanced data retrieval.
- Introduced JSON export of web_ids to 'web_ids.json' for better data management.
- Added comments in prepare_worker.py to clarify worker configuration parameters.
2026-01-13 15:53:25 -03:00

1000 lines
38 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"cells": [
{
"cell_type": "code",
"execution_count": 12,
"id": "9d16b24a",
"metadata": {},
"outputs": [],
"source": [
"from datetime import timedelta\n",
"from typing import Any\n",
"from temporalio import client\n",
"from temporalio.client import WorkflowHandle\n",
"\n",
"\n",
"async def start_workflow_advanced(\n",
" temporal_client: client.Client,\n",
" workflow_name: str,\n",
" workflow_input: dict[str, Any],\n",
" workflow_id: str,\n",
" task_queue: str,\n",
" execution_timeout: timedelta | None = None,\n",
" run_timeout: timedelta | None = None,\n",
" task_timeout: timedelta | None = None,\n",
") -> WorkflowHandle:\n",
" handle = await temporal_client.start_workflow(\n",
" workflow=workflow_name,\n",
" arg=workflow_input,\n",
" id=workflow_id or f\"{workflow_name}-{id(workflow_input)}\",\n",
" task_queue=task_queue,\n",
" execution_timeout=execution_timeout,\n",
" run_timeout=run_timeout,\n",
" task_timeout=task_timeout,\n",
" )\n",
" \n",
" return handle"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "5e344fb0",
"metadata": {},
"outputs": [],
"source": [
"input_data = {\n",
" \"debug_data_package\": False,\n",
" \"execution_timeout_seconds\": 300,\n",
" \"fill_missing_tags\": False,\n",
" \"filters\": {\n",
" \"NULL_VALUES_FILTER\": {\n",
" \"policy\": \"DISCARD\"\n",
" },\n",
" \"OUT_OF_BOUNDS_FILTER\": {\n",
" \"policy\": \"DISCARD\"\n",
" }\n",
" },\n",
" \"frequency\": \"30s\",\n",
" \"max_retry_policy\": 1,\n",
" \"model_config\": {\n",
" \"predict_flavor\": \"sklearn\",\n",
" \"retention_minutes\": 0,\n",
" \"target\": \"CI-W3A05F1\",\n",
" \"transform_flavor\": \"sklearn\"\n",
" },\n",
" \"model_id\": \"10\",\n",
" \"model_name\": \"Pi Web API Test Model\",\n",
" \"model_tags\": {\n",
" \"CI-W3W03S1\": {\n",
" \"aggr_func\": \"avg\",\n",
" \"data_range\": [\n",
" -100000,\n",
" 100000\n",
" ],\n",
" \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujATFUAAAUElIQVZDXENJLVczVzAzUzE\"\n",
" },\n",
" \"CI-W3A05F1\": {\n",
" \"aggr_func\": \"lts\",\n",
" \"data_range\": [\n",
" -100000,\n",
" 100000\n",
" ],\n",
" \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE\"\n",
" }\n",
" },\n",
" \"pi_web_api_query\": {\n",
" \"endpoint\": \"/streamsets/recorded\",\n",
" \"period\": \"*-1d\",\n",
" \"max_count\": 1,\n",
" \"api_timeout\": 5\n",
" },\n",
" \"offset\": \"0m\",\n",
" \"retention_time\": 3600,\n",
" \"schedule_name\": \"pi-web-api-scouter-test\",\n",
" \"schema\": \"sientia_data\",\n",
" \"table_name\": \"laborious_data\",\n",
" \"task_timeout_seconds\": 300,\n",
" \"trigger_laborious\": False,\n",
" \"updated_at\": \"2025-08-13 18:35:01.600000+0000\",\n",
" \"workflow_type\": \"pi_web_api_scouter\"\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "9350bff3",
"metadata": {},
"outputs": [],
"source": [
"from temporalio import client\n",
"\n",
"temporal_client = await client.Client.connect(\n",
" target_host=\"localhost:7233\",\n",
" namespace=\"scouter\"\n",
")\n"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "45712d7a",
"metadata": {},
"outputs": [],
"source": [
"\n",
"import datetime\n",
"\n",
"now = datetime.datetime.now()\n",
"\n",
"handle = await start_workflow_advanced(\n",
" temporal_client=temporal_client,\n",
" workflow_name='pi_web_api_scouter',\n",
" workflow_input=input_data,\n",
" workflow_id='test_workflow_id_' + now.strftime('%Y%m%d%H%M%S'),\n",
" task_queue='pi-web-api-scouter-queue',\n",
" execution_timeout=timedelta(seconds=30),\n",
" run_timeout=timedelta(seconds=30),\n",
" task_timeout=timedelta(seconds=30),\n",
")\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "d065d0de",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"pi-web-api-scouter\n"
]
}
],
"source": [
"import re\n",
"def camel_to_kebab(text: str) -> str:\n",
" \"\"\"Convert camelCase or PascalCase to kebab-case.\"\"\"\n",
" text = re.sub('(.)([A-Z][a-z]+)', r'\\1-\\2', text)\n",
" text = re.sub('([a-z0-9])([A-Z])', r'\\1-\\2', text)\n",
" return text.lower()\n",
"\n",
"print(camel_to_kebab('PiWebApiScouter'))\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "72af4236",
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"from time import sleep\n",
"\n",
"# Obter web id das seguintes tags:\n",
"TAG_NAMES = [\n",
" \"CI-W3A05F1\",\n",
" \"CI-W3W03S1\",\n",
" \"CI-W3W03I1\",\n",
" \"CI-W3K01T1\",\n",
" \"CI-W3W01A3\",\n",
" \"CI-W3W01A2\",\n",
" \"CI-W3W01A1\",\n",
" \"CI-J3P01T1A\",\n",
" \"CI-W3A50T1\",\n",
" \"CI-W3A55T1\",\n",
" \"CI-W3A55P1\",\n",
" \"CI-W3V33P1\",\n",
" \"CI-W3E01F1\",\n",
" \"CI-W3A50A3\",\n",
" \"CI-W3A50A2\",\n",
" \"CI-W3A50A1\",\n",
" \"CI-W3A50P1\",\n",
" \"CI-W3W01P1\",\n",
" \"CI-W3A71P1\",\n",
" \"CI-W3W01P2\",\n",
" \"CI-W3A71P2\",\n",
" \"CI-W3A71P3\",\n",
" \"CI-J3J01S1\",\n",
" \"CI-W3P17S1\",\n",
" \"CI-J3P03S1\",\n",
" \"CI-W3K01S1\",\n",
" \"CI-W3K14P1\",\n",
" \"CI-W3K01T4\",\n",
" \"CI-W3K01T2\",\n",
" \"CI-W3A65_SO3\",\n",
" \"CI-W3A65_CL\",\n",
" \"CI-W3_C3S\",\n",
" \"CI-W3_MS\",\n",
" \"CI-W3_MA\",\n",
" \"CI-W3_PL\",\n",
" \"CI-W3_CAO\",\n",
" \"CI-W3V04P3\",\n",
" \"CI-W3V04P1\",\n",
" \"CI-W3W01G1\",\n",
" \"CI-W3V21F1\",\n",
" \"CI-W3V21P1\",\n",
" \"CI-W3V30F1\",\n",
" \"CI-W3V33P1\"\n",
"]\n",
"url = 'https://pivision.votorantimcimentos.com/piwebapi/dataservers/F1DS-7fYgsRTtUOa7V9NIwSujAUElIQVZD/points?namefilter={tag}'\n",
"\n",
"headers = {\n",
" 'Content-Type': 'application/json',\n",
" 'Accept': 'application/json',\n",
" 'X-Requested-With': 'piwebapistreams', # Header recomendado pelo PI Web API\n",
" 'Authorization': \"Basic dmlkX3ZjbmV0XHN2Yy5waW9zaS5wcmQud2ViYXBpOlN2Y1ByRFdlQkBQaQ==\"\n",
"}\n",
"\n",
"web_ids = {}\n",
"\n",
"for tag in TAG_NAMES:\n",
" response = requests.get(url.replace('{tag}', tag), headers=headers).json()\n",
" web_ids[tag] = {\n",
" 'webid': response['Items'][0]['WebId'],\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000],\n",
" }\n",
" sleep(0.5)\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "55793801",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'CI-W3A05F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W03S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujATFUAAAUElIQVZDXENJLVczVzAzUzE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W03I1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAS1UAAAUElIQVZDXENJLVczVzAzSTE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3K01T1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAklQAAAUElIQVZDXENJLVczSzAxVDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W01A3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAOFUAAAUElIQVZDXENJLVczVzAxQTM',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W01A2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAN1UAAAUElIQVZDXENJLVczVzAxQTI',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W01A1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujANlUAAAUElIQVZDXENJLVczVzAxQTE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-J3P01T1A': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAYUUAAAUElIQVZDXENJLUozUDAxVDFB',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A50T1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujArFMAAAUElIQVZDXENJLVczQTUwVDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A55T1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAxFMAAAUElIQVZDXENJLVczQTU1VDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A55P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAwVMAAAUElIQVZDXENJLVczQTU1UDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3V33P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAJVUAAAUElIQVZDXENJLVczVjMzUDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3E01F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAzY0CAAUElIQVZDXENJLVczRTAxRjE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A50A3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAqVMAAAUElIQVZDXENJLVczQTUwQTM',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A50A2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAqFMAAAUElIQVZDXENJLVczQTUwQTI',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A50A1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAp1MAAAUElIQVZDXENJLVczQTUwQTE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A50P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAq1MAAAUElIQVZDXENJLVczQTUwUDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W01P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAPlUAAAUElIQVZDXENJLVczVzAxUDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A71P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAy1MAAAUElIQVZDXENJLVczQTcxUDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W01P2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAP1UAAAUElIQVZDXENJLVczVzAxUDI',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A71P2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAzFMAAAUElIQVZDXENJLVczQTcxUDI',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A71P3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAzVMAAAUElIQVZDXENJLVczQTcxUDM',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-J3J01S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAMUUAAAUElIQVZDXENJLUozSjAxUzE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3P17S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA7VQAAAUElIQVZDXENJLVczUDE3UzE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-J3P03S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAa0UAAAUElIQVZDXENJLUozUDAzUzE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3K01S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAkFQAAAUElIQVZDXENJLVczSzAxUzE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3K14P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAuFQAAAUElIQVZDXENJLVczSzE0UDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3K01T4': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAl1QAAAUElIQVZDXENJLVczSzAxVDQ',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3K01T2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAlVQAAAUElIQVZDXENJLVczSzAxVDI',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A65_SO3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAjFQAAAUElIQVZDXENJLVczQTY1X1NPMw',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A65_CL': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAf1QAAAUElIQVZDXENJLVczQTY1X0NM',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3_C3S': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA5FMAAAUElIQVZDXENJLVczX0MzUw',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3_MS': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujABlQAAAUElIQVZDXENJLVczX01T',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3_MA': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA_VMAAAUElIQVZDXENJLVczX01B',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3_PL': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAFVQAAAUElIQVZDXENJLVczX1BM',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3_CAO': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA61MAAAUElIQVZDXENJLVczX0NBTw',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3V04P3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAElUAAAUElIQVZDXENJLVczVjA0UDM',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3V04P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAEFUAAAUElIQVZDXENJLVczVjA0UDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W01G1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAOlUAAAUElIQVZDXENJLVczVzAxRzE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3V21F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAQnYEAAUElIQVZDXENJLVczVjIxRjE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3V21P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAQHYEAAUElIQVZDXENJLVczVjIxUDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3V30F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAIFUAAAUElIQVZDXENJLVczVjMwRjE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]}}"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import json\n",
"json.dump(web_ids, open('web_ids.json', 'w'), indent=4)\n",
"web_ids"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2e0e3d5a",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'Links': {},\n",
" 'Items': [{'WebId': 'F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE',\n",
" 'Name': 'CI-W3A05F1',\n",
" 'Path': '\\\\\\\\PIHAVC\\\\CI-W3A05F1',\n",
" 'Links': {'Source': 'https://pivision.votorantimcimentos.com/piwebapi/points/F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE'},\n",
" 'Items': [{'Timestamp': '2025-12-17T18:12:49.2170104Z',\n",
" 'Value': 273.3339,\n",
" 'UnitsAbbreviation': '',\n",
" 'Good': True,\n",
" 'Questionable': False,\n",
" 'Substituted': False,\n",
" 'Annotated': False}],\n",
" 'UnitsAbbreviation': ''}]}"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"item = web_ids['CI-W3A05F1']['webid']\n",
"\n",
"requests.get(\n",
" f'https://pivision.votorantimcimentos.com/piwebapi/streamsets/recorded',\n",
" params={\n",
" 'webid': item,\n",
" 'startTime': '*-1d',\n",
" 'endtime': '*',\n",
" \"maxCount\": 1,\n",
" },\n",
" headers=headers\n",
").json()"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "f8c425e2",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>variable</th>\n",
" <th>CI-J3J01S1</th>\n",
" <th>CI-J3P01T1A</th>\n",
" <th>CI-J3P03S1</th>\n",
" <th>CI-W3A05F1</th>\n",
" <th>CI-W3A50A1</th>\n",
" <th>CI-W3A50A2</th>\n",
" <th>CI-W3A50A3</th>\n",
" <th>CI-W3A50P1</th>\n",
" <th>CI-W3A50T1</th>\n",
" <th>CI-W3A55P1</th>\n",
" <th>...</th>\n",
" <th>CI-W3V33P1</th>\n",
" <th>CI-W3W01A1</th>\n",
" <th>CI-W3W01A2</th>\n",
" <th>CI-W3W01A3</th>\n",
" <th>CI-W3W01G1</th>\n",
" <th>CI-W3W01P1</th>\n",
" <th>CI-W3W01P2</th>\n",
" <th>CI-W3W03I1</th>\n",
" <th>CI-W3W03S1</th>\n",
" <th>timestamp</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>87.999020</td>\n",
" <td>238.000000</td>\n",
" <td>100.442688</td>\n",
" <td>272.869100</td>\n",
" <td>0.083338</td>\n",
" <td>2.636171</td>\n",
" <td>589.246400</td>\n",
" <td>-0.923784</td>\n",
" <td>383.722400</td>\n",
" <td>-18.167961</td>\n",
" <td>...</td>\n",
" <td>259.565500</td>\n",
" <td>0.008220</td>\n",
" <td>4.190174</td>\n",
" <td>376.848267</td>\n",
" <td>600.466900</td>\n",
" <td>-3.109598</td>\n",
" <td>0.221798</td>\n",
" <td>63.605830</td>\n",
" <td>1545.50293</td>\n",
" <td>2025-12-17 22:12:42</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>87.999020</td>\n",
" <td>238.000000</td>\n",
" <td>100.442688</td>\n",
" <td>272.649200</td>\n",
" <td>0.083338</td>\n",
" <td>2.267340</td>\n",
" <td>618.462100</td>\n",
" <td>-0.771080</td>\n",
" <td>381.468872</td>\n",
" <td>-17.495136</td>\n",
" <td>...</td>\n",
" <td>263.099854</td>\n",
" <td>0.008220</td>\n",
" <td>2.611867</td>\n",
" <td>435.797668</td>\n",
" <td>600.466900</td>\n",
" <td>-3.030156</td>\n",
" <td>0.153527</td>\n",
" <td>65.785706</td>\n",
" <td>1545.50293</td>\n",
" <td>2025-12-17 22:41:54</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>86.997925</td>\n",
" <td>228.200012</td>\n",
" <td>100.442688</td>\n",
" <td>264.225372</td>\n",
" <td>0.082358</td>\n",
" <td>3.254068</td>\n",
" <td>444.800200</td>\n",
" <td>-0.789923</td>\n",
" <td>389.007800</td>\n",
" <td>-16.590466</td>\n",
" <td>...</td>\n",
" <td>223.346313</td>\n",
" <td>0.085927</td>\n",
" <td>9.447197</td>\n",
" <td>343.596000</td>\n",
" <td>600.000000</td>\n",
" <td>-2.436767</td>\n",
" <td>0.350289</td>\n",
" <td>69.886520</td>\n",
" <td>1616.15491</td>\n",
" <td>2025-12-18 19:08:08</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>86.997925</td>\n",
" <td>228.200012</td>\n",
" <td>100.442688</td>\n",
" <td>266.714172</td>\n",
" <td>0.082361</td>\n",
" <td>2.444070</td>\n",
" <td>481.586060</td>\n",
" <td>-0.551620</td>\n",
" <td>390.612854</td>\n",
" <td>-16.791473</td>\n",
" <td>...</td>\n",
" <td>240.661469</td>\n",
" <td>0.085927</td>\n",
" <td>9.324739</td>\n",
" <td>462.062256</td>\n",
" <td>600.000000</td>\n",
" <td>-3.030156</td>\n",
" <td>0.208616</td>\n",
" <td>64.659730</td>\n",
" <td>1628.22876</td>\n",
" <td>2025-12-18 19:23:08</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>91.991210</td>\n",
" <td>139.299988</td>\n",
" <td>100.442688</td>\n",
" <td>280.392334</td>\n",
" <td>0.082361</td>\n",
" <td>1.835279</td>\n",
" <td>436.144100</td>\n",
" <td>-0.486019</td>\n",
" <td>387.727000</td>\n",
" <td>-17.495136</td>\n",
" <td>...</td>\n",
" <td>249.935165</td>\n",
" <td>0.059602</td>\n",
" <td>5.359922</td>\n",
" <td>367.250300</td>\n",
" <td>600.000000</td>\n",
" <td>-3.286316</td>\n",
" <td>0.276372</td>\n",
" <td>59.949337</td>\n",
" <td>1691.23462</td>\n",
" <td>2025-12-18 19:31:54</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>90.001220</td>\n",
" <td>230.000000</td>\n",
" <td>100.442688</td>\n",
" <td>268.893158</td>\n",
" <td>0.083623</td>\n",
" <td>2.572147</td>\n",
" <td>410.169100</td>\n",
" <td>-0.929219</td>\n",
" <td>395.898200</td>\n",
" <td>-16.390797</td>\n",
" <td>...</td>\n",
" <td>241.309967</td>\n",
" <td>0.002000</td>\n",
" <td>4.262301</td>\n",
" <td>340.296700</td>\n",
" <td>880.778200</td>\n",
" <td>-3.671064</td>\n",
" <td>0.248075</td>\n",
" <td>65.195390</td>\n",
" <td>1609.20154</td>\n",
" <td>2025-12-18 19:34:28</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>90.001220</td>\n",
" <td>230.000000</td>\n",
" <td>100.442688</td>\n",
" <td>269.359000</td>\n",
" <td>0.083623</td>\n",
" <td>2.636187</td>\n",
" <td>417.679138</td>\n",
" <td>-0.837243</td>\n",
" <td>396.060272</td>\n",
" <td>-16.390797</td>\n",
" <td>...</td>\n",
" <td>239.364624</td>\n",
" <td>0.002000</td>\n",
" <td>3.964616</td>\n",
" <td>318.547300</td>\n",
" <td>935.914368</td>\n",
" <td>-3.477627</td>\n",
" <td>0.248075</td>\n",
" <td>63.284638</td>\n",
" <td>1609.20154</td>\n",
" <td>2025-12-18 19:35:19</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>90.001220</td>\n",
" <td>230.000000</td>\n",
" <td>100.442688</td>\n",
" <td>268.651978</td>\n",
" <td>0.083623</td>\n",
" <td>2.483914</td>\n",
" <td>404.910522</td>\n",
" <td>-0.906091</td>\n",
" <td>396.222473</td>\n",
" <td>-16.390797</td>\n",
" <td>...</td>\n",
" <td>239.364624</td>\n",
" <td>0.002000</td>\n",
" <td>4.766942</td>\n",
" <td>314.721130</td>\n",
" <td>903.813232</td>\n",
" <td>-3.797625</td>\n",
" <td>0.248075</td>\n",
" <td>64.601974</td>\n",
" <td>1645.86389</td>\n",
" <td>2025-12-18 19:39:04</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>90.001220</td>\n",
" <td>230.000000</td>\n",
" <td>100.442688</td>\n",
" <td>270.836426</td>\n",
" <td>0.083623</td>\n",
" <td>2.748308</td>\n",
" <td>403.442100</td>\n",
" <td>-0.939606</td>\n",
" <td>396.222473</td>\n",
" <td>-16.390797</td>\n",
" <td>...</td>\n",
" <td>239.364624</td>\n",
" <td>0.001362</td>\n",
" <td>4.637836</td>\n",
" <td>325.616100</td>\n",
" <td>909.961060</td>\n",
" <td>-3.605707</td>\n",
" <td>0.248075</td>\n",
" <td>62.869644</td>\n",
" <td>1646.51428</td>\n",
" <td>2025-12-18 19:40:58</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>90.001220</td>\n",
" <td>227.700012</td>\n",
" <td>100.442688</td>\n",
" <td>270.715637</td>\n",
" <td>0.089754</td>\n",
" <td>2.644230</td>\n",
" <td>425.803000</td>\n",
" <td>-1.007370</td>\n",
" <td>396.384521</td>\n",
" <td>-16.725641</td>\n",
" <td>...</td>\n",
" <td>239.040222</td>\n",
" <td>0.001361</td>\n",
" <td>5.083293</td>\n",
" <td>349.286682</td>\n",
" <td>949.923500</td>\n",
" <td>-3.702983</td>\n",
" <td>0.208413</td>\n",
" <td>65.850296</td>\n",
" <td>1647.25537</td>\n",
" <td>2025-12-18 20:12:23</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>90.001220</td>\n",
" <td>227.700012</td>\n",
" <td>100.442688</td>\n",
" <td>270.661438</td>\n",
" <td>0.078859</td>\n",
" <td>2.628080</td>\n",
" <td>435.462860</td>\n",
" <td>-0.962243</td>\n",
" <td>396.384521</td>\n",
" <td>-17.031452</td>\n",
" <td>...</td>\n",
" <td>239.364441</td>\n",
" <td>0.108690</td>\n",
" <td>5.151842</td>\n",
" <td>353.761353</td>\n",
" <td>949.961060</td>\n",
" <td>-3.733788</td>\n",
" <td>0.207413</td>\n",
" <td>64.788740</td>\n",
" <td>1644.68042</td>\n",
" <td>2025-12-19 03:26:44</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>90.001220</td>\n",
" <td>227.700012</td>\n",
" <td>100.442688</td>\n",
" <td>270.725159</td>\n",
" <td>0.078859</td>\n",
" <td>2.628080</td>\n",
" <td>424.227722</td>\n",
" <td>-0.827545</td>\n",
" <td>396.384521</td>\n",
" <td>-17.031452</td>\n",
" <td>...</td>\n",
" <td>239.364441</td>\n",
" <td>0.108690</td>\n",
" <td>4.830516</td>\n",
" <td>353.761353</td>\n",
" <td>893.074000</td>\n",
" <td>-3.733788</td>\n",
" <td>0.207413</td>\n",
" <td>64.788740</td>\n",
" <td>1644.68042</td>\n",
" <td>2025-12-19 03:32:37</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>90.001220</td>\n",
" <td>227.700012</td>\n",
" <td>100.442688</td>\n",
" <td>269.977020</td>\n",
" <td>0.078859</td>\n",
" <td>3.799854</td>\n",
" <td>350.998047</td>\n",
" <td>-0.901527</td>\n",
" <td>395.614441</td>\n",
" <td>-18.777557</td>\n",
" <td>...</td>\n",
" <td>226.556274</td>\n",
" <td>0.108690</td>\n",
" <td>5.567447</td>\n",
" <td>312.856700</td>\n",
" <td>887.664734</td>\n",
" <td>-3.445201</td>\n",
" <td>0.188967</td>\n",
" <td>65.376144</td>\n",
" <td>1644.68042</td>\n",
" <td>2025-12-19 03:59:20</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>90.001220</td>\n",
" <td>227.700012</td>\n",
" <td>100.442688</td>\n",
" <td>269.098000</td>\n",
" <td>0.078859</td>\n",
" <td>3.325234</td>\n",
" <td>401.663940</td>\n",
" <td>-0.960138</td>\n",
" <td>392.380127</td>\n",
" <td>-16.966602</td>\n",
" <td>...</td>\n",
" <td>235.862518</td>\n",
" <td>0.025195</td>\n",
" <td>2.443661</td>\n",
" <td>305.058400</td>\n",
" <td>963.424100</td>\n",
" <td>-3.654345</td>\n",
" <td>0.188967</td>\n",
" <td>64.824660</td>\n",
" <td>1392.28394</td>\n",
" <td>2025-12-19 04:07:07</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>89.000120</td>\n",
" <td>133.000000</td>\n",
" <td>100.442688</td>\n",
" <td>280.595900</td>\n",
" <td>0.078859</td>\n",
" <td>2.347593</td>\n",
" <td>453.182526</td>\n",
" <td>-0.409587</td>\n",
" <td>389.169952</td>\n",
" <td>-15.958172</td>\n",
" <td>...</td>\n",
" <td>238.099854</td>\n",
" <td>0.025195</td>\n",
" <td>5.168004</td>\n",
" <td>417.250244</td>\n",
" <td>852.529200</td>\n",
" <td>-3.702983</td>\n",
" <td>0.268095</td>\n",
" <td>62.195198</td>\n",
" <td>1645.46326</td>\n",
" <td>2025-12-19 04:27:35</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>91.002320</td>\n",
" <td>130.000000</td>\n",
" <td>100.442688</td>\n",
" <td>283.037842</td>\n",
" <td>0.109597</td>\n",
" <td>2.547736</td>\n",
" <td>364.069400</td>\n",
" <td>-0.737562</td>\n",
" <td>394.961900</td>\n",
" <td>-18.296043</td>\n",
" <td>...</td>\n",
" <td>226.230900</td>\n",
" <td>0.073010</td>\n",
" <td>4.334429</td>\n",
" <td>338.391663</td>\n",
" <td>793.501953</td>\n",
" <td>-4.727626</td>\n",
" <td>0.003959</td>\n",
" <td>72.804596</td>\n",
" <td>1718.75586</td>\n",
" <td>2025-12-21 14:12:47</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>16 rows × 33 columns</p>\n",
"</div>"
],
"text/plain": [
"variable CI-J3J01S1 CI-J3P01T1A CI-J3P03S1 CI-W3A05F1 CI-W3A50A1 \\\n",
"0 87.999020 238.000000 100.442688 272.869100 0.083338 \n",
"1 87.999020 238.000000 100.442688 272.649200 0.083338 \n",
"2 86.997925 228.200012 100.442688 264.225372 0.082358 \n",
"3 86.997925 228.200012 100.442688 266.714172 0.082361 \n",
"4 91.991210 139.299988 100.442688 280.392334 0.082361 \n",
"5 90.001220 230.000000 100.442688 268.893158 0.083623 \n",
"6 90.001220 230.000000 100.442688 269.359000 0.083623 \n",
"7 90.001220 230.000000 100.442688 268.651978 0.083623 \n",
"8 90.001220 230.000000 100.442688 270.836426 0.083623 \n",
"9 90.001220 227.700012 100.442688 270.715637 0.089754 \n",
"10 90.001220 227.700012 100.442688 270.661438 0.078859 \n",
"11 90.001220 227.700012 100.442688 270.725159 0.078859 \n",
"12 90.001220 227.700012 100.442688 269.977020 0.078859 \n",
"13 90.001220 227.700012 100.442688 269.098000 0.078859 \n",
"14 89.000120 133.000000 100.442688 280.595900 0.078859 \n",
"15 91.002320 130.000000 100.442688 283.037842 0.109597 \n",
"\n",
"variable CI-W3A50A2 CI-W3A50A3 CI-W3A50P1 CI-W3A50T1 CI-W3A55P1 ... \\\n",
"0 2.636171 589.246400 -0.923784 383.722400 -18.167961 ... \n",
"1 2.267340 618.462100 -0.771080 381.468872 -17.495136 ... \n",
"2 3.254068 444.800200 -0.789923 389.007800 -16.590466 ... \n",
"3 2.444070 481.586060 -0.551620 390.612854 -16.791473 ... \n",
"4 1.835279 436.144100 -0.486019 387.727000 -17.495136 ... \n",
"5 2.572147 410.169100 -0.929219 395.898200 -16.390797 ... \n",
"6 2.636187 417.679138 -0.837243 396.060272 -16.390797 ... \n",
"7 2.483914 404.910522 -0.906091 396.222473 -16.390797 ... \n",
"8 2.748308 403.442100 -0.939606 396.222473 -16.390797 ... \n",
"9 2.644230 425.803000 -1.007370 396.384521 -16.725641 ... \n",
"10 2.628080 435.462860 -0.962243 396.384521 -17.031452 ... \n",
"11 2.628080 424.227722 -0.827545 396.384521 -17.031452 ... \n",
"12 3.799854 350.998047 -0.901527 395.614441 -18.777557 ... \n",
"13 3.325234 401.663940 -0.960138 392.380127 -16.966602 ... \n",
"14 2.347593 453.182526 -0.409587 389.169952 -15.958172 ... \n",
"15 2.547736 364.069400 -0.737562 394.961900 -18.296043 ... \n",
"\n",
"variable CI-W3V33P1 CI-W3W01A1 CI-W3W01A2 CI-W3W01A3 CI-W3W01G1 \\\n",
"0 259.565500 0.008220 4.190174 376.848267 600.466900 \n",
"1 263.099854 0.008220 2.611867 435.797668 600.466900 \n",
"2 223.346313 0.085927 9.447197 343.596000 600.000000 \n",
"3 240.661469 0.085927 9.324739 462.062256 600.000000 \n",
"4 249.935165 0.059602 5.359922 367.250300 600.000000 \n",
"5 241.309967 0.002000 4.262301 340.296700 880.778200 \n",
"6 239.364624 0.002000 3.964616 318.547300 935.914368 \n",
"7 239.364624 0.002000 4.766942 314.721130 903.813232 \n",
"8 239.364624 0.001362 4.637836 325.616100 909.961060 \n",
"9 239.040222 0.001361 5.083293 349.286682 949.923500 \n",
"10 239.364441 0.108690 5.151842 353.761353 949.961060 \n",
"11 239.364441 0.108690 4.830516 353.761353 893.074000 \n",
"12 226.556274 0.108690 5.567447 312.856700 887.664734 \n",
"13 235.862518 0.025195 2.443661 305.058400 963.424100 \n",
"14 238.099854 0.025195 5.168004 417.250244 852.529200 \n",
"15 226.230900 0.073010 4.334429 338.391663 793.501953 \n",
"\n",
"variable CI-W3W01P1 CI-W3W01P2 CI-W3W03I1 CI-W3W03S1 timestamp \n",
"0 -3.109598 0.221798 63.605830 1545.50293 2025-12-17 22:12:42 \n",
"1 -3.030156 0.153527 65.785706 1545.50293 2025-12-17 22:41:54 \n",
"2 -2.436767 0.350289 69.886520 1616.15491 2025-12-18 19:08:08 \n",
"3 -3.030156 0.208616 64.659730 1628.22876 2025-12-18 19:23:08 \n",
"4 -3.286316 0.276372 59.949337 1691.23462 2025-12-18 19:31:54 \n",
"5 -3.671064 0.248075 65.195390 1609.20154 2025-12-18 19:34:28 \n",
"6 -3.477627 0.248075 63.284638 1609.20154 2025-12-18 19:35:19 \n",
"7 -3.797625 0.248075 64.601974 1645.86389 2025-12-18 19:39:04 \n",
"8 -3.605707 0.248075 62.869644 1646.51428 2025-12-18 19:40:58 \n",
"9 -3.702983 0.208413 65.850296 1647.25537 2025-12-18 20:12:23 \n",
"10 -3.733788 0.207413 64.788740 1644.68042 2025-12-19 03:26:44 \n",
"11 -3.733788 0.207413 64.788740 1644.68042 2025-12-19 03:32:37 \n",
"12 -3.445201 0.188967 65.376144 1644.68042 2025-12-19 03:59:20 \n",
"13 -3.654345 0.188967 64.824660 1392.28394 2025-12-19 04:07:07 \n",
"14 -3.702983 0.268095 62.195198 1645.46326 2025-12-19 04:27:35 \n",
"15 -4.727626 0.003959 72.804596 1718.75586 2025-12-21 14:12:47 \n",
"\n",
"[16 rows x 33 columns]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from pandas import read_csv, to_datetime\n",
"\n",
"df = read_csv('/home/grezewave/Downloads/laborious_data_202512220821.csv')\n",
"data = df.pivot(index='timestamp', columns='variable', values='value')\n",
"data['timestamp'] = data.index\n",
"\n",
"#Remove tz from timestamp\n",
"data['timestamp'] = to_datetime(data['timestamp'])\n",
"data['timestamp'] = data['timestamp'].dt.tz_localize(None)\n",
"\n",
"# Back to string and add \"\"\n",
"data['timestamp'] = data['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')\n",
"data.reset_index(drop=True, inplace=True)\n",
"data.to_csv('VC-model-data.csv', index=False)\n",
"\n",
"display(data)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.14"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink