gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
256d3c909a7f242f197ea782ea1f05bbca717450
sientia-dataops-scouter_tem…/tests.ipynb
vitor-aignosi d9820cf3d7 SIENTIAPDE-1445 
Update requirements and enhance tests.ipynb functionality

- Changed the dependency path for sientia-dataops-library in requirements.txt to a local directory.
- Updated execution counts in tests.ipynb for better state management and added new code cells for retrieving web IDs from the PI Web API, improving data handling capabilities.
2025-12-19 11:15:19 -03:00

467 lines
26 KiB
Plaintext
Raw Blame History

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "9d16b24a",
"metadata": {},
"outputs": [],
"source": [
"from datetime import timedelta\n",
"from typing import Any\n",
"from temporalio import client\n",
"from temporalio.client import WorkflowHandle\n",
"\n",
"\n",
"async def start_workflow_advanced(\n",
" temporal_client: client.Client,\n",
" workflow_name: str,\n",
" workflow_input: dict[str, Any],\n",
" workflow_id: str,\n",
" task_queue: str,\n",
" execution_timeout: timedelta | None = None,\n",
" run_timeout: timedelta | None = None,\n",
" task_timeout: timedelta | None = None,\n",
") -> WorkflowHandle:\n",
" handle = await temporal_client.start_workflow(\n",
" workflow=workflow_name,\n",
" arg=workflow_input,\n",
" id=workflow_id or f\"{workflow_name}-{id(workflow_input)}\",\n",
" task_queue=task_queue,\n",
" execution_timeout=execution_timeout,\n",
" run_timeout=run_timeout,\n",
" task_timeout=task_timeout,\n",
" )\n",
" \n",
" return handle"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "5e344fb0",
"metadata": {},
"outputs": [],
"source": [
"input_data = {\n",
" \"debug_data_package\": False,\n",
" \"execution_timeout_seconds\": 300,\n",
" \"fill_missing_tags\": False,\n",
" \"filters\": {\n",
" \"NULL_VALUES_FILTER\": {\n",
" \"policy\": \"DISCARD\"\n",
" },\n",
" \"OUT_OF_BOUNDS_FILTER\": {\n",
" \"policy\": \"DISCARD\"\n",
" }\n",
" },\n",
" \"frequency\": \"30s\",\n",
" \"max_retry_policy\": 1,\n",
" \"model_config\": {\n",
" \"predict_flavor\": \"sklearn\",\n",
" \"retention_minutes\": 0,\n",
" \"target\": \"CI-W3A05F1\",\n",
" \"transform_flavor\": \"sklearn\"\n",
" },\n",
" \"model_id\": \"10\",\n",
" \"model_name\": \"Pi Web API Test Model\",\n",
" \"model_tags\": {\n",
" \"CI-W3W03S1\": {\n",
" \"aggr_func\": \"avg\",\n",
" \"data_range\": [\n",
" -100000,\n",
" 100000\n",
" ],\n",
" \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujATFUAAAUElIQVZDXENJLVczVzAzUzE\"\n",
" },\n",
" \"CI-W3A05F1\": {\n",
" \"aggr_func\": \"lts\",\n",
" \"data_range\": [\n",
" -100000,\n",
" 100000\n",
" ],\n",
" \"webid\": \"F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE\"\n",
" }\n",
" },\n",
" \"pi_web_api_query\": {\n",
" \"endpoint\": \"/streamsets/recorded\",\n",
" \"period\": \"*-1d\",\n",
" \"max_count\": 1,\n",
" \"api_timeout\": 5\n",
" },\n",
" \"offset\": \"0m\",\n",
" \"retention_time\": 3600,\n",
" \"schedule_name\": \"pi-web-api-scouter-test\",\n",
" \"schema\": \"sientia_data\",\n",
" \"table_name\": \"laborious_data\",\n",
" \"task_timeout_seconds\": 300,\n",
" \"trigger_laborious\": False,\n",
" \"updated_at\": \"2025-08-13 18:35:01.600000+0000\",\n",
" \"workflow_type\": \"pi_web_api_scouter\"\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "9350bff3",
"metadata": {},
"outputs": [],
"source": [
"from temporalio import client\n",
"\n",
"temporal_client = await client.Client.connect(\n",
" target_host=\"localhost:7233\",\n",
" namespace=\"scouter\"\n",
")\n"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "45712d7a",
"metadata": {},
"outputs": [],
"source": [
"\n",
"import datetime\n",
"\n",
"now = datetime.datetime.now()\n",
"\n",
"handle = await start_workflow_advanced(\n",
" temporal_client=temporal_client,\n",
" workflow_name='pi_web_api_scouter',\n",
" workflow_input=input_data,\n",
" workflow_id='test_workflow_id_' + now.strftime('%Y%m%d%H%M%S'),\n",
" task_queue='pi-web-api-scouter-queue',\n",
" execution_timeout=timedelta(seconds=30),\n",
" run_timeout=timedelta(seconds=30),\n",
" task_timeout=timedelta(seconds=30),\n",
")\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d065d0de",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"pi-web-api-scouter\n"
]
}
],
"source": [
"import re\n",
"def camel_to_kebab(text: str) -> str:\n",
" \"\"\"Convert camelCase or PascalCase to kebab-case.\"\"\"\n",
" text = re.sub('(.)([A-Z][a-z]+)', r'\\1-\\2', text)\n",
" text = re.sub('([a-z0-9])([A-Z])', r'\\1-\\2', text)\n",
" return text.lower()\n",
"\n",
"print(camel_to_kebab('PiWebApiScouter'))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "72af4236",
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"from time import sleep\n",
"\n",
"# Obter web id das seguintes tags:\n",
"TAG_NAMES = [\n",
" \"CI-W3A05F1\",\n",
" \"CI-W3W03S1\", \"CI-W3W03I1\", \"CI-W3K01T1\", \"CI-W3W01A3\",\n",
" \"CI-W3W01A2\", \"CI-W3W01A1\", \"CI-J3P01T1A\", \"CI-W3A50T1\", \"CI-W3A55T1\",\n",
" \"CI-W3A55P1\", \"CI-W3V33P1\", \"CI-W3E01F1\", \"CI-W3A50A3\", \"CI-W3A50A2\",\n",
" \"CI-W3A50A1\", \"CI-W3A50P1\", \"CI-W3W01P1\", \"CI-W3A71P1\", \"CI-W3W01P2\",\n",
" \"CI-W3A71P2\", \"CI-W3A71P3\", \"CI-J3J01S1\", \"CI-W3P17S1\", \"CI-J3P03S1\",\n",
" \"CI-W3K01S1\", \"CI-W3K14P1\", \"CI-W3K01T4\", \"CI-W3K01T2\", \n",
"\n",
" \"CI-W3FARCI_FSC\",\n",
" \"CI-W3FARCI_MA\",\n",
" \"CI-W3FARCI_MS\",\n",
" \"CI-W3FARCI_P100\",\n",
" \"CI-W3FARCI_p170\",\n",
"\n",
" \"CI-W3CLK_C3S\",\n",
" \"CI-W3CLK_C3S_EXP\",\n",
" \"CI-W3CLK_C3S_MD_EXP\",\n",
" \"CI-W3CLK_C3S_MD_PETRO\",\n",
" \"CI-W3V04P3\", \"CI-W3V04P1\",\n",
" \"CI-W3W01G1\"\n",
"]\n",
"\n",
"url = 'https://pivision.votorantimcimentos.com/piwebapi/dataservers/F1DS-7fYgsRTtUOa7V9NIwSujAUElIQVZD/points?namefilter={tag}'\n",
"\n",
"headers = {\n",
" 'Content-Type': 'application/json',\n",
" 'Accept': 'application/json',\n",
" 'X-Requested-With': 'piwebapistreams', # Header recomendado pelo PI Web API\n",
" 'Authorization': \"\"\n",
"}\n",
"\n",
"web_ids = {}\n",
"\n",
"for tag in TAG_NAMES:\n",
" response = requests.get(url.replace('{tag}', tag), headers=headers).json()\n",
" web_ids[tag] = {\n",
" 'webid': response['Items'][0]['WebId'],\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000],\n",
" }\n",
" sleep(0.5)\n"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "55793801",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'CI-W3A05F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W03S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujATFUAAAUElIQVZDXENJLVczVzAzUzE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W03I1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAS1UAAAUElIQVZDXENJLVczVzAzSTE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3K01T1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAklQAAAUElIQVZDXENJLVczSzAxVDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W01A3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAOFUAAAUElIQVZDXENJLVczVzAxQTM',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W01A2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAN1UAAAUElIQVZDXENJLVczVzAxQTI',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W01A1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujANlUAAAUElIQVZDXENJLVczVzAxQTE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-J3P01T1A': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAYUUAAAUElIQVZDXENJLUozUDAxVDFB',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A50T1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujArFMAAAUElIQVZDXENJLVczQTUwVDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A55T1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAxFMAAAUElIQVZDXENJLVczQTU1VDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A55P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAwVMAAAUElIQVZDXENJLVczQTU1UDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3V33P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAJVUAAAUElIQVZDXENJLVczVjMzUDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3E01F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAzY0CAAUElIQVZDXENJLVczRTAxRjE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A50A3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAqVMAAAUElIQVZDXENJLVczQTUwQTM',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A50A2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAqFMAAAUElIQVZDXENJLVczQTUwQTI',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A50A1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAp1MAAAUElIQVZDXENJLVczQTUwQTE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A50P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAq1MAAAUElIQVZDXENJLVczQTUwUDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W01P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAPlUAAAUElIQVZDXENJLVczVzAxUDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A71P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAy1MAAAUElIQVZDXENJLVczQTcxUDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W01P2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAP1UAAAUElIQVZDXENJLVczVzAxUDI',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A71P2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAzFMAAAUElIQVZDXENJLVczQTcxUDI',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3A71P3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAzVMAAAUElIQVZDXENJLVczQTcxUDM',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-J3J01S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAMUUAAAUElIQVZDXENJLUozSjAxUzE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3P17S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA7VQAAAUElIQVZDXENJLVczUDE3UzE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-J3P03S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAa0UAAAUElIQVZDXENJLUozUDAzUzE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3K01S1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAkFQAAAUElIQVZDXENJLVczSzAxUzE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3K14P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAuFQAAAUElIQVZDXENJLVczSzE0UDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3K01T4': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAl1QAAAUElIQVZDXENJLVczSzAxVDQ',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3K01T2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAlVQAAAUElIQVZDXENJLVczSzAxVDI',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3FARCI_FSC': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAgVQAAAUElIQVZDXENJLVczRkFSQ0lfRlND',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3FARCI_MA': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAg1QAAAUElIQVZDXENJLVczRkFSQ0lfTUE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3FARCI_MS': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAhVQAAAUElIQVZDXENJLVczRkFSQ0lfTVM',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3FARCI_P100': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAiFQAAAUElIQVZDXENJLVczRkFSQ0lfUDEwMA',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3FARCI_p170': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAiVQAAAUElIQVZDXENJLVczRkFSQ0lfUDE3MA',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3CLK_C3S': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA4lMAAAUElIQVZDXENJLVczQ0xLX0MzUw',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3CLK_C3S_EXP': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA41MAAAUElIQVZDXENJLVczQ0xLX0MzU19FWFA',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3CLK_C3S_MD_EXP': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA5VMAAAUElIQVZDXENJLVczQ0xLX0MzU19NRF9FWFA',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3CLK_C3S_MD_PETRO': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA5lMAAAUElIQVZDXENJLVczQ0xLX0MzU19NRF9QRVRSTw',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3V04P3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAElUAAAUElIQVZDXENJLVczVjA0UDM',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3V04P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAEFUAAAUElIQVZDXENJLVczVjA0UDE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]},\n",
" 'CI-W3W01G1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAOlUAAAUElIQVZDXENJLVczVzAxRzE',\n",
" 'aggr_func': 'lts',\n",
" 'data_range': [-100000, 100000]}}"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"web_ids"
]
},
{
"cell_type": "code",
"execution_count": 35,
"id": "2e0e3d5a",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'Links': {},\n",
" 'Items': [{'WebId': 'F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE',\n",
" 'Name': 'CI-W3A05F1',\n",
" 'Path': '\\\\\\\\PIHAVC\\\\CI-W3A05F1',\n",
" 'Links': {'Source': 'https://pivision.votorantimcimentos.com/piwebapi/points/F1DP-7fYgsRTtUOa7V9NIwSujAkVMAAAUElIQVZDXENJLVczQTA1RjE'},\n",
" 'Items': [{'Timestamp': '2025-12-17T18:12:49.2170104Z',\n",
" 'Value': 273.3339,\n",
" 'UnitsAbbreviation': '',\n",
" 'Good': True,\n",
" 'Questionable': False,\n",
" 'Substituted': False,\n",
" 'Annotated': False}],\n",
" 'UnitsAbbreviation': ''}]}"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"item = web_ids['CI-W3A05F1']['webid']\n",
"\n",
"requests.get(\n",
" f'https://pivision.votorantimcimentos.com/piwebapi/streamsets/recorded',\n",
" params={\n",
" 'webid': item,\n",
" 'startTime': '*-1d',\n",
" 'endtime': '*',\n",
" \"maxCount\": 1,\n",
" },\n",
" headers=headers\n",
").json()"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "f8c425e2",
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "Index contains duplicate entries, cannot reshape",
"output_type": "error",
"traceback": [
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
"\u001b[31mValueError\u001b[39m Traceback (most recent call last)",
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[26]\u001b[39m\u001b[32m, line 4\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m read_csv\n\u001b[32m 3\u001b[39m df = read_csv(\u001b[33m'\u001b[39m\u001b[33m/home/grezewave/Downloads/laborious_data_202512181402.csv\u001b[39m\u001b[33m'\u001b[39m)\n\u001b[32m----> \u001b[39m\u001b[32m4\u001b[39m \u001b[43mdf\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpivot\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mtimestamp\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mvariable\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mvalue\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m 6\u001b[39m display(df)\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pandas/core/frame.py:9339\u001b[39m, in \u001b[36mDataFrame.pivot\u001b[39m\u001b[34m(self, columns, index, values)\u001b[39m\n\u001b[32m 9332\u001b[39m \u001b[38;5;129m@Substitution\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 9333\u001b[39m \u001b[38;5;129m@Appender\u001b[39m(_shared_docs[\u001b[33m\"\u001b[39m\u001b[33mpivot\u001b[39m\u001b[33m\"\u001b[39m])\n\u001b[32m 9334\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpivot\u001b[39m(\n\u001b[32m 9335\u001b[39m \u001b[38;5;28mself\u001b[39m, *, columns, index=lib.no_default, values=lib.no_default\n\u001b[32m 9336\u001b[39m ) -> DataFrame:\n\u001b[32m 9337\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mreshape\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mpivot\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m pivot\n\u001b[32m-> \u001b[39m\u001b[32m9339\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mpivot\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindex\u001b[49m\u001b[43m=\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m=\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pandas/core/reshape/pivot.py:570\u001b[39m, in \u001b[36mpivot\u001b[39m\u001b[34m(data, columns, index, values)\u001b[39m\n\u001b[32m 566\u001b[39m indexed = data._constructor_sliced(data[values]._values, index=multiindex)\n\u001b[32m 567\u001b[39m \u001b[38;5;66;03m# error: Argument 1 to \"unstack\" of \"DataFrame\" has incompatible type \"Union\u001b[39;00m\n\u001b[32m 568\u001b[39m \u001b[38;5;66;03m# [List[Any], ExtensionArray, ndarray[Any, Any], Index, Series]\"; expected\u001b[39;00m\n\u001b[32m 569\u001b[39m \u001b[38;5;66;03m# \"Hashable\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m570\u001b[39m result = \u001b[43mindexed\u001b[49m\u001b[43m.\u001b[49m\u001b[43munstack\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcolumns_listlike\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[32m 571\u001b[39m result.index.names = [\n\u001b[32m 572\u001b[39m name \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m lib.no_default \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01mfor\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m result.index.names\n\u001b[32m 573\u001b[39m ]\n\u001b[32m 575\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pandas/core/series.py:4615\u001b[39m, in \u001b[36mSeries.unstack\u001b[39m\u001b[34m(self, level, fill_value, sort)\u001b[39m\n\u001b[32m 4570\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 4571\u001b[39m \u001b[33;03mUnstack, also known as pivot, Series with MultiIndex to produce DataFrame.\u001b[39;00m\n\u001b[32m 4572\u001b[39m \n\u001b[32m (...)\u001b[39m\u001b[32m 4611\u001b[39m \u001b[33;03mb 2 4\u001b[39;00m\n\u001b[32m 4612\u001b[39m \u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 4613\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mreshape\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mreshape\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m unstack\n\u001b[32m-> \u001b[39m\u001b[32m4615\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43munstack\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msort\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pandas/core/reshape/reshape.py:517\u001b[39m, in \u001b[36munstack\u001b[39m\u001b[34m(obj, level, fill_value, sort)\u001b[39m\n\u001b[32m 515\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m is_1d_only_ea_dtype(obj.dtype):\n\u001b[32m 516\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m _unstack_extension_series(obj, level, fill_value, sort=sort)\n\u001b[32m--> \u001b[39m\u001b[32m517\u001b[39m unstacker = \u001b[43m_Unstacker\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 518\u001b[39m \u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\u001b[43m.\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconstructor\u001b[49m\u001b[43m=\u001b[49m\u001b[43mobj\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_constructor_expanddim\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msort\u001b[49m\u001b[43m=\u001b[49m\u001b[43msort\u001b[49m\n\u001b[32m 519\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 520\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m unstacker.get_result(\n\u001b[32m 521\u001b[39m obj._values, value_columns=\u001b[38;5;28;01mNone\u001b[39;00m, fill_value=fill_value\n\u001b[32m 522\u001b[39m )\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pandas/core/reshape/reshape.py:154\u001b[39m, in \u001b[36m_Unstacker.__init__\u001b[39m\u001b[34m(self, index, level, constructor, sort)\u001b[39m\n\u001b[32m 146\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m num_cells > np.iinfo(np.int32).max:\n\u001b[32m 147\u001b[39m warnings.warn(\n\u001b[32m 148\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mThe following operation may generate \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mnum_cells\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m cells \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 149\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33min the resulting pandas object.\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 150\u001b[39m PerformanceWarning,\n\u001b[32m 151\u001b[39m stacklevel=find_stack_level(),\n\u001b[32m 152\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m154\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_selectors\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pandas/core/reshape/reshape.py:210\u001b[39m, in \u001b[36m_Unstacker._make_selectors\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 207\u001b[39m mask.put(selector, \u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 209\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m mask.sum() < \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m.index):\n\u001b[32m--> \u001b[39m\u001b[32m210\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[33m\"\u001b[39m\u001b[33mIndex contains duplicate entries, cannot reshape\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 212\u001b[39m \u001b[38;5;28mself\u001b[39m.group_index = comp_index\n\u001b[32m 213\u001b[39m \u001b[38;5;28mself\u001b[39m.mask = mask\n",
"\u001b[31mValueError\u001b[39m: Index contains duplicate entries, cannot reshape"
]
}
],
"source": [
"from pandas import read_csv\n",
"\n",
"df = read_csv('/home/grezewave/Downloads/laborious_data_202512181402.csv')\n",
"df.pivot(index='timestamp', columns='variable', values='value')\n",
"\n",
"display(df)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.14"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink