{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "b10e5c25", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "# Set random seed for reproducibility\n", "rng = np.random.default_rng(42)\n", "\n", "# Generate random walks starting at 0\n", "counter = np.zeros(300)\n", "rollout = np.zeros(300)\n", "\n", "# Generate random steps between -1 and 1\n", "counter_steps = rng.uniform(-1, 1, 299)\n", "rollout_steps = rng.uniform(-1, 1, 299)\n", "\n", "print(counter_steps)\n", "print(rollout_steps)\n", "\n", "# Calculate cumulative sum and scale to -100 to 100 range\n", "for i in range(1, 300):\n", " counter[i] = counter[i-1] + counter_steps[i-1]\n", " rollout[i] = rollout[i-1] + rollout_steps[i-1]\n", "\n", "# normalize values between -100 and 100, lowest value is -100, highest value is 100\n", "counter = (counter - min(counter)) / (max(counter) - min(counter)) * 200 - 100\n", "rollout = (rollout - min(rollout)) / (max(rollout) - min(rollout)) * 200 - 100\n", "\n", "# Create DataFrame\n", "df = pd.DataFrame({\n", " 'Counter': counter,\n", " 'Rollout': rollout,\n", " 'CounterPlusRollout': counter + rollout\n", "})\n", "\n", "# add a timestamp column\n", "df['Timestamp'] = pd.date_range(start='2025-01-01', periods=300, freq='1s')\n", "\n", "# Save to CSV\n", "df.to_csv('random_walks.csv', index=False)\n", "\n", "# Display first few rows\n", "print(df.head())\n" ] }, { "cell_type": "code", "execution_count": null, "id": "c61be7ab", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "# Set random seed for reproducibility\n", "rng = np.random.default_rng(42)\n", "\n", "size = 50\n", "\n", "# Generate random walks starting at 0\n", "counter = np.zeros(size)\n", "rollout = np.zeros(size)\n", "square = np.zeros(size)\n", "\n", "# Generate random steps between -1 and 1\n", "counter_steps = rng.uniform(-1, 1, size-1)\n", "rollout_steps = rng.uniform(-1, 1, size-1)\n", "square_steps = rng.uniform(-1, 1, size-1)\n", "\n", "print(counter_steps)\n", "print(rollout_steps)\n", "\n", "# Calculate cumulative sum and scale to -100 to 100 range\n", "for i in range(1, size):\n", " counter[i] = counter[i-1] + counter_steps[i-1]\n", " rollout[i] = rollout[i-1] + rollout_steps[i-1]\n", " square[i] = square[i-1] + square_steps[i-1]\n", "\n", "# normalize values between -100 and 100, lowest value is -100, highest value is 100\n", "counter = (counter - min(counter)) / (max(counter) - min(counter)) * 200 - 100\n", "rollout = (rollout - min(rollout)) / (max(rollout) - min(rollout)) * 200 - 100\n", "square = (square - min(square)) / (max(square) - min(square)) * 200 - 100\n", "\n", "# Create DataFrame\n", "df = pd.DataFrame({\n", " 'Counter': counter,\n", " 'Rollout': rollout,\n", " 'Square': square\n", "})\n", "\n", "# add a timestamp column\n", "df['Timestamp'] = pd.date_range(start='2025-01-01', periods=size, freq='5s')\n", "\n", "# Save to CSV\n", "df.to_csv('random_walks_demo.csv', index=False)\n", "\n", "# Display first few rows\n", "print(df.head())\n" ] }, { "cell_type": "code", "execution_count": null, "id": "e7c8eeb1", "metadata": {}, "outputs": [], "source": [ "from pandas import DataFrame\n", "\n", "a = DataFrame({\n", " \"a\": {\"2025-01-01\": 1, \"2025-01-02\": 2, \"2025-01-03\": 3},\n", " \"b\": {\"2025-01-01\": 4, \"2025-01-02\": 5, \"2025-01-03\": 6},\n", "})\n", "\n", "a.index.name = \"timestamp\"\n", "\n", "display(a)\n", "\n", "print(\"a\" in a.columns)\n", "\n", "b = a.tail(1)\n", "\n", "display(b)\n", "\n", "print(len(a))\n", "print(len(b))\n", "print(b.size)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "f3374174", "metadata": {}, "outputs": [], "source": [ "from pandas import DataFrame\n", "\n", "data = DataFrame({\n", " \"a\": {\"2025-01-01\": 1, \"2025-01-02\": 2, \"2025-01-03\": 3},\n", " \"b\": {\"2025-01-01\": 4, \"2025-01-02\": 5, \"2025-01-03\": 6},\n", "})\n", "\n", "index = data.index\n", "\n", "# Get type of first element of index\n", "index_type = type(index[0])\n", "\n", "print(index_type)\n", "\n", "# Check if all in index are of the same type\n", "if all(isinstance(i, index_type) for i in index):\n", " print(\"All elements in index are of the same type\")\n", "else:\n", " print(\"Elements in index are of different types\")\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "40e72c60", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1fbb3788", "metadata": {}, "outputs": [], "source": [ "from unittest.mock import MagicMock\n", "from asyncua.ua.uaerrors import BadAlreadyExists\n", "\n", "mock1 = MagicMock(\n", " side_effect = Exception(\"test\")\n", ")\n", "\n", "mock2 = MagicMock(\n", " side_effect = BadAlreadyExists(\"test\")\n", ")\n", "\n", "try:\n", " mock1()\n", "except ValueError as e:\n", " try:\n", " mock2()\n", " except BadAlreadyExists as e:\n", " print(e)\n", "except Exception as e:\n", " print(e)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "771ab4ee", "metadata": {}, "outputs": [], "source": [ "from pandas import DataFrame, merge\n", "\n", "retrain_dataset = DataFrame({\n", " 'a': {'2025-01-01': 1, '2025-01-02': 2, '2025-01-03': 3},\n", " 'b': {'2025-01-01': 4, '2025-01-02': 5, '2025-01-03': 6},\n", " 'c': {'2025-01-01': 7, '2025-01-02': 8, '2025-01-03': 9},\n", "})\n", "\n", "prediction_data = DataFrame({\n", " 'prediction': {'1': 1, '2': 2, '3': 3},\n", "})\n", "\n", "prediction_data.index = retrain_dataset.index\n", "\n", "prediction_data = merge(\n", " retrain_dataset, prediction_data, left_index=True, right_index=True, how='left')\n", "\n", "prediction_data.rename(columns={'c': 'target'}, inplace=True)\n", "\n", "prediction_data['timestamp'] = prediction_data.index\n", "\n", "prediction_data.reset_index(drop=True, inplace=True)\n", "\n", "display(prediction_data)" ] }, { "cell_type": "code", "execution_count": null, "id": "486b95b3", "metadata": {}, "outputs": [], "source": [ "from pandas import DataFrame\n", "\n", "data = DataFrame()\n", "\n", "display(data.to_dict(orient='records'))\n", "\n", "data = DataFrame({\n", " \"a\": {\"2025-01-01\": 1, \"2025-01-02\": 2, \"2025-01-03\": 3},\n", " \"b\": {\"2025-01-01\": 4, \"2025-01-02\": 5, \"2025-01-03\": 6},\n", "})\n", "\n", "display(data)\n", "\n", "data_list = data.to_dict('split')\n", "\n", "display(data_list)\n", "\n", "data_rec = DataFrame.from_dict(data_list, orient='index')\n", "\n", "display(data_rec)\n", "\n", "data.shape[0]" ] }, { "cell_type": "code", "execution_count": null, "id": "d67d551f", "metadata": {}, "outputs": [], "source": [ "import os\n", "import shutil\n", "import mlflow\n", "from mlflow.tracking import MlflowClient\n", "from rich.console import Console\n", "import sys\n", "\n", "if \"src\" not in sys.path:\n", " sys.path.insert(0, \"src\")\n", "\n", "console = Console()\n", "\n", "os.environ[\"MLFLOW_TRACKING_URI\"] = \"http://localhost:35785/\"\n", "os.environ[\"MLFLOW_TRACKING_USERNAME\"] = \"aignosi\"\n", "os.environ[\"MLFLOW_TRACKING_PASSWORD\"] = \"1L0FP50j3ncp123\"\n", "\n", "client = MlflowClient()" ] }, { "cell_type": "code", "execution_count": null, "id": "98bdd6af", "metadata": {}, "outputs": [], "source": [ "from pandas import DataFrame, read_json\n", "\n", "# Load from data file (json)\n", "data = read_json('data.json')\n", "\n", "column_order = [\n", " 'CI-J3J01S1', 'CI-J3P01T1A', 'CI-J3P03S1', 'CI-W3A05F1',\n", " 'CI-W3A50A1', 'CI-W3A50A2', 'CI-W3A50A3', 'CI-W3A50P1', 'CI-W3A50T1',\n", " 'CI-W3A55P1', 'CI-W3A55T1', 'CI-W3A65_Cl', 'CI-W3A65_SO3', 'CI-W3A71P1',\n", " 'CI-W3A71P2', 'CI-W3A71P3', 'CI-W3E01F1', 'CI-W3K01S1', 'CI-W3K01T1',\n", " 'CI-W3K01T2', 'CI-W3K01T4', 'CI-W3K14P1', 'CI-W3P17S1', 'CI-W3V04P1',\n", " 'CI-W3V04P3', 'CI-W3V21F1', 'CI-W3V21P1', 'CI-W3V30F1', 'CI-W3V33P1',\n", " 'CI-W3W01G1', 'CI-W3W01P1', 'CI-W3W01P2', 'CI-W3W03I1', 'CI-W3W03S1',\n", " 'CI-W3_C3S', 'CI-W3_CAO', 'CI-W3_MA', 'CI-W3_MS', 'CI-W3_PL']\n", "\n", "ordered_data = data[column_order]\n", "\n", "display(ordered_data.head(3))\n", "display(data.head(3))\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "d7e73c30", "metadata": {}, "outputs": [], "source": [ "import mlflow\n", "\n", "model_name = \"vcm-o2-vanilla-ice\"\n", "\n", "model = mlflow.pyfunc.load_model(f'models:/{model_name}/production')" ] }, { "cell_type": "code", "execution_count": null, "id": "4e5bae06", "metadata": {}, "outputs": [], "source": [ "model.predict(ordered_data)" ] }, { "cell_type": "code", "execution_count": null, "id": "cd1bfaa0", "metadata": {}, "outputs": [], "source": [ "# Download pkl file from mlflow\n", "model_uri = f'models:/{model_name}/production'\n", "model_path = mlflow.artifacts.download_artifacts(model_uri, dst_path='./backup-model')\n", "\n", "print(model_path)" ] }, { "cell_type": "code", "execution_count": null, "id": "0ef9c913", "metadata": {}, "outputs": [], "source": [ "file_path = f'{model_path}/artifacts/xgboost_model.pkl'\n", "\n", "# Verificar os primeiros bytes do arquivo\n", "with open(file_path, 'rb') as f:\n", " first_bytes = f.read(10)\n", " print(first_bytes)" ] }, { "cell_type": "code", "execution_count": 12, "id": "8ae302f3", "metadata": {}, "outputs": [], "source": [ "from pandas import DataFrame, to_datetime\n", "from sientia_do.temporal.activities.postgres import Postgres\n", "from unittest.mock import MagicMock, AsyncMock\n", "\n", "postgres = Postgres(\n", " host=\"localhost\",\n", " port=5432,\n", " dbname=\"sientia\",\n", " user=\"sientia\",\n", " password=\"sientia\",\n", " min_connections=1,\n", " max_connections=10,\n", " logger=MagicMock(),\n", " notification_handler=MagicMock(),\n", " metrics_controller=AsyncMock()\n", ")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "51dd2dbf", "metadata": {}, "outputs": [], "source": [ "\n", "period_hours = 1\n", "\n", "samples = 2*60*period_hours\n", "retrain_samples = 2*period_hours\n", "\n", "query_nox = f\"\"\"\n", "select p.prediction, ld.value, p.\"timestamp\" from sientia_data.predictions p\n", "join sientia_data.laborious_data ld on p.\"timestamp\" = ld.\"timestamp\"\n", "where p.model_id = '4' and variable='CI-W3W01A3' order by p.created_at desc limit {samples};\n", "\"\"\"\n", "\n", "retrain_query = f\"\"\"\n", "select \"timestamp\" from sientia_data.log_retrain lr where model_id = '4' order by lr.\"timestamp\" desc limit {retrain_samples};\n", "\"\"\"\n", "\n", "retrain_data_nox = DataFrame(await postgres.load_custom_query(\n", " {\n", " \"query\": retrain_query,\n", " \"metadata\": {},\n", " }\n", "))\n", "\n", "retrain_data_nox['timestamp'] = to_datetime(retrain_data_nox['timestamp'])\n", "\n", "display(retrain_data_nox)\n", "\n", "data_nox = DataFrame(await postgres.load_custom_query(\n", " {\n", " \"query\": query_nox,\n", " \"metadata\": {},\n", " \"datetime_columns\": [\"timestamp\"],\n", " }\n", "))\n", "\n", "data_nox.drop_duplicates(subset=['timestamp'], inplace=True, keep='first')\n", "data_nox.sort_values(by='timestamp', inplace=True)\n", "data_nox['timestamp'] = to_datetime(data_nox['timestamp'])\n", "\n", "display(data_nox.head(3))\n", "\n", "query_o2 = f\"\"\"\n", "select p.prediction, ld.value, p.\"timestamp\" from sientia_data.predictions p\n", "join sientia_data.laborious_data ld on p.\"timestamp\" = ld.\"timestamp\"\n", "where p.model_id = '5' and variable='CI-W3W01A2'\n", "and p.\"timestamp\" >= NOW() - INTERVAL {period_hours} HOUR\n", "order by p.created_at;\n", "\"\"\"\n", "\n", "retrain_query_o2 = f\"\"\"\n", "select \"timestamp\" from sientia_data.log_retrain lr where model_id = '5' order by lr.\"timestamp\" desc limit {retrain_samples};\n", "\"\"\"\n", "\n", "retrain_data_o2 = DataFrame(await postgres.load_custom_query(\n", " {\n", " \"query\": retrain_query_o2,\n", " \"metadata\": {},\n", " }\n", "))\n", "\n", "retrain_data_o2['timestamp'] = to_datetime(retrain_data_o2['timestamp'])\n", "\n", "display(retrain_data_o2)\n", "\n", "data_o2 = DataFrame(await postgres.load_custom_query(\n", " {\n", " \"query\": query_o2,\n", " \"metadata\": {},\n", " \"datetime_columns\": [\"timestamp\"],\n", " }\n", "))\n", "\n", "data_o2.drop_duplicates(subset=['timestamp'], inplace=True, keep='first')\n", "data_o2.sort_values(by='timestamp', inplace=True)\n", "data_o2['timestamp'] = to_datetime(data_o2['timestamp'])\n", "\n", "display(data_o2.head(3))\n" ] }, { "cell_type": "code", "execution_count": null, "id": "7b82e5a7", "metadata": {}, "outputs": [], "source": [ "\n", "import matplotlib.pyplot as plt\n", "\n", "plt.figure(figsize=(10, 10))\n", "plt.subplot(2, 1, 1)\n", "plt.plot(data_nox['timestamp'], data_nox['value'])\n", "plt.plot(data_nox['timestamp'], data_nox['prediction'])\n", "plt.vlines(\n", " x=retrain_data_nox['timestamp'],\n", " ymin=plt.ylim()[0],\n", " ymax=plt.ylim()[1],\n", " colors='k',\n", " linestyles='--'\n", ")\n", "\n", "\n", "plt.legend(['real', 'prediction', 'retrain'])\n", "plt.title('NOx')\n", "plt.xlim(\n", " data_nox['timestamp'].min(),\n", " data_nox['timestamp'].max()\n", ")\n", "\n", "plt.subplot(2, 1, 2)\n", "plt.plot(data_o2['timestamp'], data_o2['value'])\n", "plt.plot(data_o2['timestamp'], data_o2['prediction'])\n", "plt.vlines(\n", " x=retrain_data_o2['timestamp'],\n", " ymin=plt.ylim()[0],\n", " ymax=plt.ylim()[1],\n", " colors='k',\n", " linestyles='--'\n", ")\n", "\n", "plt.xlim(\n", " data_o2['timestamp'].min(),\n", " data_o2['timestamp'].max()\n", ")\n", "\n", "plt.legend(['real', 'prediction', 'retrain'])\n", "plt.title('O2')\n", "plt.show()\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 13, "id": "6bbb8cf7", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/grezewave/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", " self.metrics_controller.start()\n", "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" ] }, { "data": { "text/html": [ "
| \n", " | prediction | \n", "value | \n", "timestamp | \n", "
|---|---|---|---|
| 0 | \n", "265.533539 | \n", "260.24646 | \n", "2026-01-22 15:42:29+0000 | \n", "
| 1 | \n", "262.758423 | \n", "258.94940 | \n", "2026-01-22 15:42:59+0000 | \n", "
| 2 | \n", "263.060638 | \n", "257.65370 | \n", "2026-01-22 15:43:29+0000 | \n", "