gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1445

Browse Source 
Update tests.ipynb to improve execution count management and enhance data handling

- Adjusted execution counts for various code cells to maintain consistent state.
- Added error handling for KeyError in API response processing.
- Updated data processing logic to include timestamp formatting and CSV export functionality.
This commit is contained in:
vitor-aignosi
2025-12-22 14:10:58 -03:00
parent bcc4e6de6d
commit 90d543f810
1 changed files with 547 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

576
tests.ipynb
View File

@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 12,
"id": "9d16b24a",
"metadata": {},
"outputs": [],
@@ -38,7 +38,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 13,
"id": "5e344fb0",
"metadata": {},
"outputs": [],
@@ -103,7 +103,7 @@
},
{
"cell_type": "code",
"execution_count": 27,
"execution_count": 14,
"id": "9350bff3",
"metadata": {},
"outputs": [],
@@ -118,7 +118,7 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 15,
"id": "45712d7a",
"metadata": {},
"outputs": [],
@@ -143,7 +143,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 16,
"id": "d065d0de",
"metadata": {},
"outputs": [
@@ -168,10 +168,22 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 17,
"id": "72af4236",
"metadata": {},
"outputs": [],
"outputs": [
{
"ename": "KeyError",
"evalue": "'Items'",
"output_type": "error",
"traceback": [
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
"\u001b[31mKeyError\u001b[39m Traceback (most recent call last)",
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[17]\u001b[39m\u001b[32m, line 42\u001b[39m\n\u001b[32m 39\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m tag \u001b[38;5;129;01min\u001b[39;00m TAG_NAMES:\n\u001b[32m 40\u001b[39m response = requests.get(url.replace(\u001b[33m'\u001b[39m\u001b[38;5;132;01m{tag}\u001b[39;00m\u001b[33m'\u001b[39m, tag), headers=headers).json()\n\u001b[32m 41\u001b[39m web_ids[tag] = {\n\u001b[32m---> \u001b[39m\u001b[32m42\u001b[39m \u001b[33m'\u001b[39m\u001b[33mwebid\u001b[39m\u001b[33m'\u001b[39m: \u001b[43mresponse\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mItems\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m]\u001b[49m[\u001b[32m0\u001b[39m][\u001b[33m'\u001b[39m\u001b[33mWebId\u001b[39m\u001b[33m'\u001b[39m],\n\u001b[32m 43\u001b[39m \u001b[33m'\u001b[39m\u001b[33maggr_func\u001b[39m\u001b[33m'\u001b[39m: \u001b[33m'\u001b[39m\u001b[33mlts\u001b[39m\u001b[33m'\u001b[39m,\n\u001b[32m 44\u001b[39m \u001b[33m'\u001b[39m\u001b[33mdata_range\u001b[39m\u001b[33m'\u001b[39m: [-\u001b[32m100000\u001b[39m, \u001b[32m100000\u001b[39m],\n\u001b[32m 45\u001b[39m }\n\u001b[32m 46\u001b[39m sleep(\u001b[32m0.5\u001b[39m)\n",
"\u001b[31mKeyError\u001b[39m: 'Items'"
]
}
],
"source": [
"import requests\n",
"from time import sleep\n",
@@ -223,7 +235,7 @@
},
{
"cell_type": "code",
"execution_count": 23,
"execution_count": null,
"id": "55793801",
"metadata": {},
"outputs": [
@@ -366,7 +378,7 @@
},
{
"cell_type": "code",
"execution_count": 35,
"execution_count": null,
"id": "2e0e3d5a",
"metadata": {},
"outputs": [
@@ -410,35 +422,541 @@
},
{
"cell_type": "code",
"execution_count": 26,
"execution_count": 18,
"id": "f8c425e2",
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "Index contains duplicate entries, cannot reshape",
"output_type": "error",
"traceback": [
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
"\u001b[31mValueError\u001b[39m Traceback (most recent call last)",
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[26]\u001b[39m\u001b[32m, line 4\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m read_csv\n\u001b[32m 3\u001b[39m df = read_csv(\u001b[33m'\u001b[39m\u001b[33m/home/grezewave/Downloads/laborious_data_202512181402.csv\u001b[39m\u001b[33m'\u001b[39m)\n\u001b[32m----> \u001b[39m\u001b[32m4\u001b[39m \u001b[43mdf\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpivot\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mtimestamp\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mvariable\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mvalue\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m 6\u001b[39m display(df)\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pandas/core/frame.py:9339\u001b[39m, in \u001b[36mDataFrame.pivot\u001b[39m\u001b[34m(self, columns, index, values)\u001b[39m\n\u001b[32m 9332\u001b[39m \u001b[38;5;129m@Substitution\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 9333\u001b[39m \u001b[38;5;129m@Appender\u001b[39m(_shared_docs[\u001b[33m\"\u001b[39m\u001b[33mpivot\u001b[39m\u001b[33m\"\u001b[39m])\n\u001b[32m 9334\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpivot\u001b[39m(\n\u001b[32m 9335\u001b[39m \u001b[38;5;28mself\u001b[39m, *, columns, index=lib.no_default, values=lib.no_default\n\u001b[32m 9336\u001b[39m ) -> DataFrame:\n\u001b[32m 9337\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mreshape\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mpivot\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m pivot\n\u001b[32m-> \u001b[39m\u001b[32m9339\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mpivot\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindex\u001b[49m\u001b[43m=\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m=\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pandas/core/reshape/pivot.py:570\u001b[39m, in \u001b[36mpivot\u001b[39m\u001b[34m(data, columns, index, values)\u001b[39m\n\u001b[32m 566\u001b[39m indexed = data._constructor_sliced(data[values]._values, index=multiindex)\n\u001b[32m 567\u001b[39m \u001b[38;5;66;03m# error: Argument 1 to \"unstack\" of \"DataFrame\" has incompatible type \"Union\u001b[39;00m\n\u001b[32m 568\u001b[39m \u001b[38;5;66;03m# [List[Any], ExtensionArray, ndarray[Any, Any], Index, Series]\"; expected\u001b[39;00m\n\u001b[32m 569\u001b[39m \u001b[38;5;66;03m# \"Hashable\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m570\u001b[39m result = \u001b[43mindexed\u001b[49m\u001b[43m.\u001b[49m\u001b[43munstack\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcolumns_listlike\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[32m 571\u001b[39m result.index.names = [\n\u001b[32m 572\u001b[39m name \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m lib.no_default \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01mfor\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m result.index.names\n\u001b[32m 573\u001b[39m ]\n\u001b[32m 575\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pandas/core/series.py:4615\u001b[39m, in \u001b[36mSeries.unstack\u001b[39m\u001b[34m(self, level, fill_value, sort)\u001b[39m\n\u001b[32m 4570\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 4571\u001b[39m \u001b[33;03mUnstack, also known as pivot, Series with MultiIndex to produce DataFrame.\u001b[39;00m\n\u001b[32m 4572\u001b[39m \n\u001b[32m (...)\u001b[39m\u001b[32m 4611\u001b[39m \u001b[33;03mb 2 4\u001b[39;00m\n\u001b[32m 4612\u001b[39m \u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 4613\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mreshape\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mreshape\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m unstack\n\u001b[32m-> \u001b[39m\u001b[32m4615\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43munstack\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msort\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pandas/core/reshape/reshape.py:517\u001b[39m, in \u001b[36munstack\u001b[39m\u001b[34m(obj, level, fill_value, sort)\u001b[39m\n\u001b[32m 515\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m is_1d_only_ea_dtype(obj.dtype):\n\u001b[32m 516\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m _unstack_extension_series(obj, level, fill_value, sort=sort)\n\u001b[32m--> \u001b[39m\u001b[32m517\u001b[39m unstacker = \u001b[43m_Unstacker\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 518\u001b[39m \u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\u001b[43m.\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconstructor\u001b[49m\u001b[43m=\u001b[49m\u001b[43mobj\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_constructor_expanddim\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msort\u001b[49m\u001b[43m=\u001b[49m\u001b[43msort\u001b[49m\n\u001b[32m 519\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 520\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m unstacker.get_result(\n\u001b[32m 521\u001b[39m obj._values, value_columns=\u001b[38;5;28;01mNone\u001b[39;00m, fill_value=fill_value\n\u001b[32m 522\u001b[39m )\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pandas/core/reshape/reshape.py:154\u001b[39m, in \u001b[36m_Unstacker.__init__\u001b[39m\u001b[34m(self, index, level, constructor, sort)\u001b[39m\n\u001b[32m 146\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m num_cells > np.iinfo(np.int32).max:\n\u001b[32m 147\u001b[39m warnings.warn(\n\u001b[32m 148\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mThe following operation may generate \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mnum_cells\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m cells \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 149\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33min the resulting pandas object.\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 150\u001b[39m PerformanceWarning,\n\u001b[32m 151\u001b[39m stacklevel=find_stack_level(),\n\u001b[32m 152\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m154\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_selectors\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pandas/core/reshape/reshape.py:210\u001b[39m, in \u001b[36m_Unstacker._make_selectors\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 207\u001b[39m mask.put(selector, \u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 209\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m mask.sum() < \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m.index):\n\u001b[32m--> \u001b[39m\u001b[32m210\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[33m\"\u001b[39m\u001b[33mIndex contains duplicate entries, cannot reshape\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 212\u001b[39m \u001b[38;5;28mself\u001b[39m.group_index = comp_index\n\u001b[32m 213\u001b[39m \u001b[38;5;28mself\u001b[39m.mask = mask\n",
"\u001b[31mValueError\u001b[39m: Index contains duplicate entries, cannot reshape"
]
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th>variable</th>\n",
" <th>CI-J3J01S1</th>\n",
" <th>CI-J3P01T1A</th>\n",
" <th>CI-J3P03S1</th>\n",
" <th>CI-W3A05F1</th>\n",
" <th>CI-W3A50A1</th>\n",
" <th>CI-W3A50A2</th>\n",
" <th>CI-W3A50A3</th>\n",
" <th>CI-W3A50P1</th>\n",
" <th>CI-W3A50T1</th>\n",
" <th>CI-W3A55P1</th>\n",
" <th>...</th>\n",
" <th>CI-W3V33P1</th>\n",
" <th>CI-W3W01A1</th>\n",
" <th>CI-W3W01A2</th>\n",
" <th>CI-W3W01A3</th>\n",
" <th>CI-W3W01G1</th>\n",
" <th>CI-W3W01P1</th>\n",
" <th>CI-W3W01P2</th>\n",
" <th>CI-W3W03I1</th>\n",
" <th>CI-W3W03S1</th>\n",
" <th>timestamp</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>87.999020</td>\n",
" <td>238.000000</td>\n",
" <td>100.442688</td>\n",
" <td>272.869100</td>\n",
" <td>0.083338</td>\n",
" <td>2.636171</td>\n",
" <td>589.246400</td>\n",
" <td>-0.923784</td>\n",
" <td>383.722400</td>\n",
" <td>-18.167961</td>\n",
" <td>...</td>\n",
" <td>259.565500</td>\n",
" <td>0.008220</td>\n",
" <td>4.190174</td>\n",
" <td>376.848267</td>\n",
" <td>600.466900</td>\n",
" <td>-3.109598</td>\n",
" <td>0.221798</td>\n",
" <td>63.605830</td>\n",
" <td>1545.50293</td>\n",
" <td>2025-12-17 22:12:42</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>87.999020</td>\n",
" <td>238.000000</td>\n",
" <td>100.442688</td>\n",
" <td>272.649200</td>\n",
" <td>0.083338</td>\n",
" <td>2.267340</td>\n",
" <td>618.462100</td>\n",
" <td>-0.771080</td>\n",
" <td>381.468872</td>\n",
" <td>-17.495136</td>\n",
" <td>...</td>\n",
" <td>263.099854</td>\n",
" <td>0.008220</td>\n",
" <td>2.611867</td>\n",
" <td>435.797668</td>\n",
" <td>600.466900</td>\n",
" <td>-3.030156</td>\n",
" <td>0.153527</td>\n",
" <td>65.785706</td>\n",
" <td>1545.50293</td>\n",
" <td>2025-12-17 22:41:54</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>86.997925</td>\n",
" <td>228.200012</td>\n",
" <td>100.442688</td>\n",
" <td>264.225372</td>\n",
" <td>0.082358</td>\n",
" <td>3.254068</td>\n",
" <td>444.800200</td>\n",
" <td>-0.789923</td>\n",
" <td>389.007800</td>\n",
" <td>-16.590466</td>\n",
" <td>...</td>\n",
" <td>223.346313</td>\n",
" <td>0.085927</td>\n",
" <td>9.447197</td>\n",
" <td>343.596000</td>\n",
" <td>600.000000</td>\n",
" <td>-2.436767</td>\n",
" <td>0.350289</td>\n",
" <td>69.886520</td>\n",
" <td>1616.15491</td>\n",
" <td>2025-12-18 19:08:08</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>86.997925</td>\n",
" <td>228.200012</td>\n",
" <td>100.442688</td>\n",
" <td>266.714172</td>\n",
" <td>0.082361</td>\n",
" <td>2.444070</td>\n",
" <td>481.586060</td>\n",
" <td>-0.551620</td>\n",
" <td>390.612854</td>\n",
" <td>-16.791473</td>\n",
" <td>...</td>\n",
" <td>240.661469</td>\n",
" <td>0.085927</td>\n",
" <td>9.324739</td>\n",
" <td>462.062256</td>\n",
" <td>600.000000</td>\n",
" <td>-3.030156</td>\n",
" <td>0.208616</td>\n",
" <td>64.659730</td>\n",
" <td>1628.22876</td>\n",
" <td>2025-12-18 19:23:08</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>91.991210</td>\n",
" <td>139.299988</td>\n",
" <td>100.442688</td>\n",
" <td>280.392334</td>\n",
" <td>0.082361</td>\n",
" <td>1.835279</td>\n",
" <td>436.144100</td>\n",
" <td>-0.486019</td>\n",
" <td>387.727000</td>\n",
" <td>-17.495136</td>\n",
" <td>...</td>\n",
" <td>249.935165</td>\n",
" <td>0.059602</td>\n",
" <td>5.359922</td>\n",
" <td>367.250300</td>\n",
" <td>600.000000</td>\n",
" <td>-3.286316</td>\n",
" <td>0.276372</td>\n",
" <td>59.949337</td>\n",
" <td>1691.23462</td>\n",
" <td>2025-12-18 19:31:54</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>90.001220</td>\n",
" <td>230.000000</td>\n",
" <td>100.442688</td>\n",
" <td>268.893158</td>\n",
" <td>0.083623</td>\n",
" <td>2.572147</td>\n",
" <td>410.169100</td>\n",
" <td>-0.929219</td>\n",
" <td>395.898200</td>\n",
" <td>-16.390797</td>\n",
" <td>...</td>\n",
" <td>241.309967</td>\n",
" <td>0.002000</td>\n",
" <td>4.262301</td>\n",
" <td>340.296700</td>\n",
" <td>880.778200</td>\n",
" <td>-3.671064</td>\n",
" <td>0.248075</td>\n",
" <td>65.195390</td>\n",
" <td>1609.20154</td>\n",
" <td>2025-12-18 19:34:28</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>90.001220</td>\n",
" <td>230.000000</td>\n",
" <td>100.442688</td>\n",
" <td>269.359000</td>\n",
" <td>0.083623</td>\n",
" <td>2.636187</td>\n",
" <td>417.679138</td>\n",
" <td>-0.837243</td>\n",
" <td>396.060272</td>\n",
" <td>-16.390797</td>\n",
" <td>...</td>\n",
" <td>239.364624</td>\n",
" <td>0.002000</td>\n",
" <td>3.964616</td>\n",
" <td>318.547300</td>\n",
" <td>935.914368</td>\n",
" <td>-3.477627</td>\n",
" <td>0.248075</td>\n",
" <td>63.284638</td>\n",
" <td>1609.20154</td>\n",
" <td>2025-12-18 19:35:19</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>90.001220</td>\n",
" <td>230.000000</td>\n",
" <td>100.442688</td>\n",
" <td>268.651978</td>\n",
" <td>0.083623</td>\n",
" <td>2.483914</td>\n",
" <td>404.910522</td>\n",
" <td>-0.906091</td>\n",
" <td>396.222473</td>\n",
" <td>-16.390797</td>\n",
" <td>...</td>\n",
" <td>239.364624</td>\n",
" <td>0.002000</td>\n",
" <td>4.766942</td>\n",
" <td>314.721130</td>\n",
" <td>903.813232</td>\n",
" <td>-3.797625</td>\n",
" <td>0.248075</td>\n",
" <td>64.601974</td>\n",
" <td>1645.86389</td>\n",
" <td>2025-12-18 19:39:04</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>90.001220</td>\n",
" <td>230.000000</td>\n",
" <td>100.442688</td>\n",
" <td>270.836426</td>\n",
" <td>0.083623</td>\n",
" <td>2.748308</td>\n",
" <td>403.442100</td>\n",
" <td>-0.939606</td>\n",
" <td>396.222473</td>\n",
" <td>-16.390797</td>\n",
" <td>...</td>\n",
" <td>239.364624</td>\n",
" <td>0.001362</td>\n",
" <td>4.637836</td>\n",
" <td>325.616100</td>\n",
" <td>909.961060</td>\n",
" <td>-3.605707</td>\n",
" <td>0.248075</td>\n",
" <td>62.869644</td>\n",
" <td>1646.51428</td>\n",
" <td>2025-12-18 19:40:58</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>90.001220</td>\n",
" <td>227.700012</td>\n",
" <td>100.442688</td>\n",
" <td>270.715637</td>\n",
" <td>0.089754</td>\n",
" <td>2.644230</td>\n",
" <td>425.803000</td>\n",
" <td>-1.007370</td>\n",
" <td>396.384521</td>\n",
" <td>-16.725641</td>\n",
" <td>...</td>\n",
" <td>239.040222</td>\n",
" <td>0.001361</td>\n",
" <td>5.083293</td>\n",
" <td>349.286682</td>\n",
" <td>949.923500</td>\n",
" <td>-3.702983</td>\n",
" <td>0.208413</td>\n",
" <td>65.850296</td>\n",
" <td>1647.25537</td>\n",
" <td>2025-12-18 20:12:23</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>90.001220</td>\n",
" <td>227.700012</td>\n",
" <td>100.442688</td>\n",
" <td>270.661438</td>\n",
" <td>0.078859</td>\n",
" <td>2.628080</td>\n",
" <td>435.462860</td>\n",
" <td>-0.962243</td>\n",
" <td>396.384521</td>\n",
" <td>-17.031452</td>\n",
" <td>...</td>\n",
" <td>239.364441</td>\n",
" <td>0.108690</td>\n",
" <td>5.151842</td>\n",
" <td>353.761353</td>\n",
" <td>949.961060</td>\n",
" <td>-3.733788</td>\n",
" <td>0.207413</td>\n",
" <td>64.788740</td>\n",
" <td>1644.68042</td>\n",
" <td>2025-12-19 03:26:44</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>90.001220</td>\n",
" <td>227.700012</td>\n",
" <td>100.442688</td>\n",
" <td>270.725159</td>\n",
" <td>0.078859</td>\n",
" <td>2.628080</td>\n",
" <td>424.227722</td>\n",
" <td>-0.827545</td>\n",
" <td>396.384521</td>\n",
" <td>-17.031452</td>\n",
" <td>...</td>\n",
" <td>239.364441</td>\n",
" <td>0.108690</td>\n",
" <td>4.830516</td>\n",
" <td>353.761353</td>\n",
" <td>893.074000</td>\n",
" <td>-3.733788</td>\n",
" <td>0.207413</td>\n",
" <td>64.788740</td>\n",
" <td>1644.68042</td>\n",
" <td>2025-12-19 03:32:37</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>90.001220</td>\n",
" <td>227.700012</td>\n",
" <td>100.442688</td>\n",
" <td>269.977020</td>\n",
" <td>0.078859</td>\n",
" <td>3.799854</td>\n",
" <td>350.998047</td>\n",
" <td>-0.901527</td>\n",
" <td>395.614441</td>\n",
" <td>-18.777557</td>\n",
" <td>...</td>\n",
" <td>226.556274</td>\n",
" <td>0.108690</td>\n",
" <td>5.567447</td>\n",
" <td>312.856700</td>\n",
" <td>887.664734</td>\n",
" <td>-3.445201</td>\n",
" <td>0.188967</td>\n",
" <td>65.376144</td>\n",
" <td>1644.68042</td>\n",
" <td>2025-12-19 03:59:20</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>90.001220</td>\n",
" <td>227.700012</td>\n",
" <td>100.442688</td>\n",
" <td>269.098000</td>\n",
" <td>0.078859</td>\n",
" <td>3.325234</td>\n",
" <td>401.663940</td>\n",
" <td>-0.960138</td>\n",
" <td>392.380127</td>\n",
" <td>-16.966602</td>\n",
" <td>...</td>\n",
" <td>235.862518</td>\n",
" <td>0.025195</td>\n",
" <td>2.443661</td>\n",
" <td>305.058400</td>\n",
" <td>963.424100</td>\n",
" <td>-3.654345</td>\n",
" <td>0.188967</td>\n",
" <td>64.824660</td>\n",
" <td>1392.28394</td>\n",
" <td>2025-12-19 04:07:07</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>89.000120</td>\n",
" <td>133.000000</td>\n",
" <td>100.442688</td>\n",
" <td>280.595900</td>\n",
" <td>0.078859</td>\n",
" <td>2.347593</td>\n",
" <td>453.182526</td>\n",
" <td>-0.409587</td>\n",
" <td>389.169952</td>\n",
" <td>-15.958172</td>\n",
" <td>...</td>\n",
" <td>238.099854</td>\n",
" <td>0.025195</td>\n",
" <td>5.168004</td>\n",
" <td>417.250244</td>\n",
" <td>852.529200</td>\n",
" <td>-3.702983</td>\n",
" <td>0.268095</td>\n",
" <td>62.195198</td>\n",
" <td>1645.46326</td>\n",
" <td>2025-12-19 04:27:35</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>91.002320</td>\n",
" <td>130.000000</td>\n",
" <td>100.442688</td>\n",
" <td>283.037842</td>\n",
" <td>0.109597</td>\n",
" <td>2.547736</td>\n",
" <td>364.069400</td>\n",
" <td>-0.737562</td>\n",
" <td>394.961900</td>\n",
" <td>-18.296043</td>\n",
" <td>...</td>\n",
" <td>226.230900</td>\n",
" <td>0.073010</td>\n",
" <td>4.334429</td>\n",
" <td>338.391663</td>\n",
" <td>793.501953</td>\n",
" <td>-4.727626</td>\n",
" <td>0.003959</td>\n",
" <td>72.804596</td>\n",
" <td>1718.75586</td>\n",
" <td>2025-12-21 14:12:47</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>16 rows × 33 columns</p>\n",
"</div>"
],
"text/plain": [
"variable CI-J3J01S1 CI-J3P01T1A CI-J3P03S1 CI-W3A05F1 CI-W3A50A1 \\\n",
"0 87.999020 238.000000 100.442688 272.869100 0.083338 \n",
"1 87.999020 238.000000 100.442688 272.649200 0.083338 \n",
"2 86.997925 228.200012 100.442688 264.225372 0.082358 \n",
"3 86.997925 228.200012 100.442688 266.714172 0.082361 \n",
"4 91.991210 139.299988 100.442688 280.392334 0.082361 \n",
"5 90.001220 230.000000 100.442688 268.893158 0.083623 \n",
"6 90.001220 230.000000 100.442688 269.359000 0.083623 \n",
"7 90.001220 230.000000 100.442688 268.651978 0.083623 \n",
"8 90.001220 230.000000 100.442688 270.836426 0.083623 \n",
"9 90.001220 227.700012 100.442688 270.715637 0.089754 \n",
"10 90.001220 227.700012 100.442688 270.661438 0.078859 \n",
"11 90.001220 227.700012 100.442688 270.725159 0.078859 \n",
"12 90.001220 227.700012 100.442688 269.977020 0.078859 \n",
"13 90.001220 227.700012 100.442688 269.098000 0.078859 \n",
"14 89.000120 133.000000 100.442688 280.595900 0.078859 \n",
"15 91.002320 130.000000 100.442688 283.037842 0.109597 \n",
"\n",
"variable CI-W3A50A2 CI-W3A50A3 CI-W3A50P1 CI-W3A50T1 CI-W3A55P1 ... \\\n",
"0 2.636171 589.246400 -0.923784 383.722400 -18.167961 ... \n",
"1 2.267340 618.462100 -0.771080 381.468872 -17.495136 ... \n",
"2 3.254068 444.800200 -0.789923 389.007800 -16.590466 ... \n",
"3 2.444070 481.586060 -0.551620 390.612854 -16.791473 ... \n",
"4 1.835279 436.144100 -0.486019 387.727000 -17.495136 ... \n",
"5 2.572147 410.169100 -0.929219 395.898200 -16.390797 ... \n",
"6 2.636187 417.679138 -0.837243 396.060272 -16.390797 ... \n",
"7 2.483914 404.910522 -0.906091 396.222473 -16.390797 ... \n",
"8 2.748308 403.442100 -0.939606 396.222473 -16.390797 ... \n",
"9 2.644230 425.803000 -1.007370 396.384521 -16.725641 ... \n",
"10 2.628080 435.462860 -0.962243 396.384521 -17.031452 ... \n",
"11 2.628080 424.227722 -0.827545 396.384521 -17.031452 ... \n",
"12 3.799854 350.998047 -0.901527 395.614441 -18.777557 ... \n",
"13 3.325234 401.663940 -0.960138 392.380127 -16.966602 ... \n",
"14 2.347593 453.182526 -0.409587 389.169952 -15.958172 ... \n",
"15 2.547736 364.069400 -0.737562 394.961900 -18.296043 ... \n",
"\n",
"variable CI-W3V33P1 CI-W3W01A1 CI-W3W01A2 CI-W3W01A3 CI-W3W01G1 \\\n",
"0 259.565500 0.008220 4.190174 376.848267 600.466900 \n",
"1 263.099854 0.008220 2.611867 435.797668 600.466900 \n",
"2 223.346313 0.085927 9.447197 343.596000 600.000000 \n",
"3 240.661469 0.085927 9.324739 462.062256 600.000000 \n",
"4 249.935165 0.059602 5.359922 367.250300 600.000000 \n",
"5 241.309967 0.002000 4.262301 340.296700 880.778200 \n",
"6 239.364624 0.002000 3.964616 318.547300 935.914368 \n",
"7 239.364624 0.002000 4.766942 314.721130 903.813232 \n",
"8 239.364624 0.001362 4.637836 325.616100 909.961060 \n",
"9 239.040222 0.001361 5.083293 349.286682 949.923500 \n",
"10 239.364441 0.108690 5.151842 353.761353 949.961060 \n",
"11 239.364441 0.108690 4.830516 353.761353 893.074000 \n",
"12 226.556274 0.108690 5.567447 312.856700 887.664734 \n",
"13 235.862518 0.025195 2.443661 305.058400 963.424100 \n",
"14 238.099854 0.025195 5.168004 417.250244 852.529200 \n",
"15 226.230900 0.073010 4.334429 338.391663 793.501953 \n",
"\n",
"variable CI-W3W01P1 CI-W3W01P2 CI-W3W03I1 CI-W3W03S1 timestamp \n",
"0 -3.109598 0.221798 63.605830 1545.50293 2025-12-17 22:12:42 \n",
"1 -3.030156 0.153527 65.785706 1545.50293 2025-12-17 22:41:54 \n",
"2 -2.436767 0.350289 69.886520 1616.15491 2025-12-18 19:08:08 \n",
"3 -3.030156 0.208616 64.659730 1628.22876 2025-12-18 19:23:08 \n",
"4 -3.286316 0.276372 59.949337 1691.23462 2025-12-18 19:31:54 \n",
"5 -3.671064 0.248075 65.195390 1609.20154 2025-12-18 19:34:28 \n",
"6 -3.477627 0.248075 63.284638 1609.20154 2025-12-18 19:35:19 \n",
"7 -3.797625 0.248075 64.601974 1645.86389 2025-12-18 19:39:04 \n",
"8 -3.605707 0.248075 62.869644 1646.51428 2025-12-18 19:40:58 \n",
"9 -3.702983 0.208413 65.850296 1647.25537 2025-12-18 20:12:23 \n",
"10 -3.733788 0.207413 64.788740 1644.68042 2025-12-19 03:26:44 \n",
"11 -3.733788 0.207413 64.788740 1644.68042 2025-12-19 03:32:37 \n",
"12 -3.445201 0.188967 65.376144 1644.68042 2025-12-19 03:59:20 \n",
"13 -3.654345 0.188967 64.824660 1392.28394 2025-12-19 04:07:07 \n",
"14 -3.702983 0.268095 62.195198 1645.46326 2025-12-19 04:27:35 \n",
"15 -4.727626 0.003959 72.804596 1718.75586 2025-12-21 14:12:47 \n",
"\n",
"[16 rows x 33 columns]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from pandas import read_csv\n",
"from pandas import read_csv, to_datetime\n",
"\n",
"df = read_csv('/home/grezewave/Downloads/laborious_data_202512181402.csv')\n",
"df.pivot(index='timestamp', columns='variable', values='value')\n",
"df = read_csv('/home/grezewave/Downloads/laborious_data_202512220821.csv')\n",
"data = df.pivot(index='timestamp', columns='variable', values='value')\n",
"data['timestamp'] = data.index\n",
"\n",
"display(df)"
"#Remove tz from timestamp\n",
"data['timestamp'] = to_datetime(data['timestamp'])\n",
"data['timestamp'] = data['timestamp'].dt.tz_localize(None)\n",
"\n",
"# Back to string and add \"\"\n",
"data['timestamp'] = data['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')\n",
"data.reset_index(drop=True, inplace=True)\n",
"data.to_csv('VC-model-data.csv', index=False)\n",
"\n",
"display(data)"
]
}
],