{ "cells": [ { "cell_type": "code", "execution_count": 4, "id": "a287fa45", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initializing Couchbase connection...\n", "Awaiting Couchbase connection...\n", "Couchbase connection ready\n" ] } ], "source": [ "from orchestrator.activities.couchbase import Couchbase\n", "from unittest.mock import MagicMock\n", "logger = MagicMock(info=MagicMock(side_effect=print), debug=MagicMock(side_effect=print))\n", "couchbase = Couchbase(\n", " connection_string=\"couchbase://localhost\",\n", " username=\"sientia\",\n", " password=\"sientia\",\n", " logger=logger,\n", " notification_handler=MagicMock()\n", ")\n" ] }, { "cell_type": "code", "execution_count": 3, "id": "d9a0f9c4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Executing couchbase query: %s \n", "SELECT\n", " pipelines.*,\n", " models as model\n", "FROM\n", " `pipelines`\n", "JOIN\n", " `models` ON KEYS pipelines.model_id;\n", "\n", "Fetched %d rows from couchbase 1\n", "Rows: \n", " %s [\n", " {\n", " \"filters\": [\n", " {\n", " \"filter_name\": \"OUT_OF_BOUNDS_FILTER\",\n", " \"policy\": \"DISCARD\"\n", " },\n", " {\n", " \"filter_name\": \"NULL_VALUES_FILTER\",\n", " \"policy\": \"DISCARD\"\n", " }\n", " ],\n", " \"frequency\": \"5s\",\n", " \"max_retry_policy\": 1,\n", " \"model\": {\n", " \"name\": \"Demo Model-Demo2\"\n", " },\n", " \"model_id\": \"1\",\n", " \"name\": \"scouter-opcua-pipeline\",\n", " \"read_tags\": [\n", " {\n", " \"aggr_func\": \"avg\",\n", " \"data_range\": [\n", " -100,\n", " 100\n", " ],\n", " \"tag_name\": \"Counter\"\n", " }\n", " ],\n", " \"tag_retention_minutes\": 60,\n", " \"workflow_type\": \"scouter\"\n", " }\n", "]\n", "[{'filters': [{'filter_name': 'OUT_OF_BOUNDS_FILTER', 'policy': 'DISCARD'}, {'filter_name': 'NULL_VALUES_FILTER', 'policy': 'DISCARD'}], 'frequency': '5s', 'max_retry_policy': 1, 'model': {'name': 'Demo Model-Demo2'}, 'model_id': '1', 'name': 'scouter-opcua-pipeline', 'read_tags': [{'aggr_func': 'avg', 'data_range': [-100, 100], 'tag_name': 'Counter'}], 'tag_retention_minutes': 60, 'workflow_type': 'scouter'}]\n" ] } ], "source": [ "query = \"\"\"\n", "SELECT\n", " pipelines.*,\n", " models as model\n", "FROM\n", " `pipelines`\n", "JOIN\n", " `models` ON KEYS pipelines.model_id;\n", "\"\"\"\n", "if __name__ == \"__main__\":\n", "\n", "\n", " result = await couchbase.load_query_from_couchbase({\n", " \"query\": query\n", " })\n", "\n", " print(result)\n", " " ] }, { "cell_type": "code", "execution_count": 3, "id": "7d01f160", "metadata": {}, "outputs": [], "source": [ "from temporalio import client\n", "from orchestrator.activities.temporal_manager import TemporalManager\n", "import os\n", "from unittest.mock import MagicMock\n", "\n", "host = \"localhost:7233\"\n", "logger = MagicMock(info=MagicMock(side_effect=print), debug=MagicMock(side_effect=print))\n", "\n", "temporal_client = await client.Client.connect(\n", " target_host=host,\n", " namespace=os.getenv('TEMPORAL_NAMESPACE', 'default')\n", ")\n", "\n", " " ] }, { "cell_type": "code", "execution_count": null, "id": "bb750ae6", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import asyncio\n", "from datetime import timedelta\n", "from temporalio.client import (\n", " Client,\n", " Schedule,\n", " ScheduleActionStartWorkflow,\n", " ScheduleIntervalSpec,\n", " ScheduleSpec,\n", ")\n", "from temporalio.common import TypedSearchAttributes, SearchAttributeKey, SearchAttributePair\n", "\n", "# temporal operator search-attribute create --namespace scouter --name model_id --type Text && temporal operator search-attribute create --namespace scouter --name orchestrated --type Text && temporal operator search-attribute create --namespace scouter --name model_name --type Text && temporal operator search-attribute create --namespace laborious --name model_id --type Text && temporal operator search-attribute create --namespace laborious --name orchestrated --type Text && temporal operator search-attribute create --namespace laborious --name model_name --type Text\n", "\n", "\n", "\n", "await temporal_client.create_schedule(\n", " \"orchestrator\",\n", " Schedule(\n", " action=ScheduleActionStartWorkflow(\n", " 'orchestrator',\n", " {\n", " \"schedule_name\": \"orchestrator-test\",\n", " \"pipelines_query\": {\n", " \"collection\": \"pipelines\",\n", " \"aggregation\": [\n", " {\n", " \"$lookup\": {\n", " \"from\": \"models\",\n", " \"localField\": \"model_id\",\n", " \"foreignField\": \"id\",\n", " \"as\": \"model_docs\"\n", " }\n", " },\n", " {\n", " \"$match\": {\n", " \"active\": True\n", " }\n", " },\n", " {\n", " \"$addFields\": {\n", " \"models\": {\n", " \"$arrayElemAt\": [\n", " \"$model_docs\",\n", " 0\n", " ]\n", " }\n", " }\n", " },\n", " {\n", " \"$match\": {\n", " \"models.active\": True\n", " }\n", " },\n", " {\n", " \"$project\": {\n", " \"model_docs\": 0\n", " }\n", " }\n", " ]\n", " },\n", " \"opc_servers_query\": {\n", " \"collection\": \"opc-servers\",\n", " \"filters\": {\n", "\n", " }\n", " }\n", " },\n", " id=\"orchestrator\",\n", " task_queue=\"orchestrator-queue\",\n", " execution_timeout=timedelta(minutes=600)\n", " ),\n", " spec=ScheduleSpec(\n", " intervals=[ScheduleIntervalSpec(every=timedelta(minutes=60))]\n", " )\n", " )\n", ")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "b840347d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Error deleting schedule scouter-load-test-1: workflow execution already completed\n", "Error deleting schedule scouter-load-test-2: workflow execution already completed\n", "Error deleting schedule scouter-load-test-3: workflow execution already completed\n", "Error deleting schedule scouter-load-test-4: workflow execution already completed\n", "Error deleting schedule scouter-load-test-5: workflow execution already completed\n", "Error deleting schedule scouter-load-test-6: workflow execution already completed\n", "Error deleting schedule scouter-load-test-7: workflow execution already completed\n", "Error deleting schedule scouter-load-test-8: workflow execution already completed\n", "Error deleting schedule scouter-load-test-9: workflow execution already completed\n", "Error deleting schedule scouter-load-test-10: workflow execution already completed\n", "Error deleting schedule scouter-load-test-11: workflow execution already completed\n", "Error deleting schedule scouter-load-test-12: workflow execution already completed\n", "Error deleting schedule scouter-load-test-13: workflow execution already completed\n", "Error deleting schedule scouter-load-test-14: workflow execution already completed\n", "Error deleting schedule scouter-load-test-15: workflow execution already completed\n", "Error deleting schedule scouter-load-test-16: workflow execution already completed\n", "Error deleting schedule scouter-load-test-17: workflow execution already completed\n", "Error deleting schedule scouter-load-test-18: workflow execution already completed\n", "Error deleting schedule scouter-load-test-19: workflow execution already completed\n" ] } ], "source": [ "# Delete schedule by ID\n", "schedule_id = \"scouter-load-test-num\"\n", "\n", "for i in range(0, 70):\n", "\n", " try:\n", " handle = temporal_client.get_schedule_handle(\n", " schedule_id.replace(\"num\", str(i)))\n", " \n", " await handle.delete()\n", " except Exception as e:\n", " print(f\"Error deleting schedule {schedule_id.replace('num', str(i))}: {e}\")\n", "\n", "# Delete schedule by ID\n", "schedule_id = \"laborious-load-test-num\"\n", "\n", "for i in range(0, 70):\n", "\n", " try:\n", " handle = temporal_client.get_schedule_handle(\n", " schedule_id.replace(\"num\", str(i)))\n", " \n", " await handle.delete()\n", " except Exception as e:\n", " print(f\"Error deleting schedule {schedule_id.replace('num', str(i))}: {e}\")\n" ] }, { "cell_type": "code", "execution_count": 7, "id": "374b5b0e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "scouter-load-test-num\n" ] } ], "source": [ "schedule_id = \"scouter-load-test-num\"\n", "schedule_id.replace(\"-\", \"_\")\n", "\n", "print(schedule_id)" ] }, { "cell_type": "code", "execution_count": 2, "id": "1bd82225", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Getting orchestrated schedules...\n" ] } ], "source": [ "schedules = await manager.load_schedule()" ] }, { "cell_type": "code", "execution_count": 24, "id": "1948670e", "metadata": {}, "outputs": [], "source": [ "from google.protobuf.json_format import MessageToDict\n", "import base64\n", "import json\n", "for arg in schedules.schedule.action.args:\n", " data = MessageToDict(arg)\n", " data = base64.b64decode(data['data']).decode('utf-8')" ] }, { "cell_type": "code", "execution_count": 12, "id": "a4c777dd", "metadata": {}, "outputs": [], "source": [ "from google.protobuf.json_format import MessageToDict\n", "import base64\n", "import json\n", "\n", "schedules_config = {}\n", "async for schedule in await temporal_client.list_schedules():\n", " id = schedule.id\n", "\n", " handle = temporal_client.get_schedule_handle(id)\n", "\n", " desc = await handle.describe()\n", "\n", " for arg in desc.schedule.action.args:\n", " data = MessageToDict(arg)['data']\n", " data = base64.b64decode(data).decode('utf-8')\n", "\n", " frequency = desc.schedule.spec.intervals[0].every.seconds\n", "\n", " schedules_config[id] = {\n", " 'frequency': frequency,\n", " 'data': json.loads(data),\n", " 'handle': handle\n", " }\n", " \n", " \n", " \n", " " ] }, { "cell_type": "code", "execution_count": 4, "id": "f81b3728", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'id': 'orchestrator', 'schedule': ScheduleListSchedule(action=ScheduleListActionStartWorkflow(workflow='orchestrator'), spec=ScheduleSpec(calendars=[], intervals=[ScheduleIntervalSpec(every=datetime.timedelta(seconds=3600), offset=None)], cron_expressions=[], skip=[], start_at=None, end_at=None, jitter=None, time_zone_name=None), state=ScheduleListState(note=None, paused=False)), 'info': ScheduleListInfo(recent_actions=[ScheduleActionResult(scheduled_at=datetime.datetime(2025, 7, 14, 9, 0, tzinfo=datetime.timezone.utc), started_at=datetime.datetime(2025, 7, 14, 9, 0, 0, 165083, tzinfo=datetime.timezone.utc), action=ScheduleActionExecutionStartWorkflow(workflow_id='orchestrator-2025-07-14T09:00:00Z', first_execution_run_id='01980829-9700-771a-9c9d-2450eecc9af4')), ScheduleActionResult(scheduled_at=datetime.datetime(2025, 7, 14, 10, 0, tzinfo=datetime.timezone.utc), started_at=datetime.datetime(2025, 7, 14, 10, 0, 0, 125309, tzinfo=datetime.timezone.utc), action=ScheduleActionExecutionStartWorkflow(workflow_id='orchestrator-2025-07-14T10:00:00Z', first_execution_run_id='01980860-8578-759d-97fa-dbdfc784f88a')), ScheduleActionResult(scheduled_at=datetime.datetime(2025, 7, 14, 11, 0, tzinfo=datetime.timezone.utc), started_at=datetime.datetime(2025, 7, 14, 11, 0, 0, 124356, tzinfo=datetime.timezone.utc), action=ScheduleActionExecutionStartWorkflow(workflow_id='orchestrator-2025-07-14T11:00:00Z', first_execution_run_id='01980897-73f7-7e83-a7a6-20eaac38e7d4')), ScheduleActionResult(scheduled_at=datetime.datetime(2025, 7, 14, 12, 0, tzinfo=datetime.timezone.utc), started_at=datetime.datetime(2025, 7, 14, 12, 0, 0, 167329, tzinfo=datetime.timezone.utc), action=ScheduleActionExecutionStartWorkflow(workflow_id='orchestrator-2025-07-14T12:00:00Z', first_execution_run_id='019808ce-62a2-75d6-b11d-c5387cfa258f')), ScheduleActionResult(scheduled_at=datetime.datetime(2025, 7, 14, 13, 0, tzinfo=datetime.timezone.utc), started_at=datetime.datetime(2025, 7, 14, 13, 0, 0, 167575, tzinfo=datetime.timezone.utc), action=ScheduleActionExecutionStartWorkflow(workflow_id='orchestrator-2025-07-14T13:00:00Z', first_execution_run_id='01980905-5122-7807-90ae-3782d0332e79'))], next_action_times=[datetime.datetime(2025, 7, 14, 14, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2025, 7, 14, 15, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2025, 7, 14, 16, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2025, 7, 14, 17, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2025, 7, 14, 18, 0, tzinfo=datetime.timezone.utc)]), 'typed_search_attributes': TypedSearchAttributes(search_attributes=[]), 'search_attributes': {}, 'data_converter': DataConverter(payload_converter_class=, payload_codec=None, failure_converter_class=, payload_converter=, failure_converter=), 'raw_entry': schedule_id: \"orchestrator\"\n", "info {\n", " spec {\n", " interval {\n", " interval {\n", " seconds: 3600\n", " }\n", " }\n", " }\n", " workflow_type {\n", " name: \"orchestrator\"\n", " }\n", " recent_actions {\n", " schedule_time {\n", " seconds: 1752483600\n", " }\n", " actual_time {\n", " seconds: 1752483600\n", " nanos: 165083498\n", " }\n", " start_workflow_result {\n", " workflow_id: \"orchestrator-2025-07-14T09:00:00Z\"\n", " run_id: \"01980829-9700-771a-9c9d-2450eecc9af4\"\n", " }\n", " start_workflow_status: WORKFLOW_EXECUTION_STATUS_COMPLETED\n", " }\n", " recent_actions {\n", " schedule_time {\n", " seconds: 1752487200\n", " }\n", " actual_time {\n", " seconds: 1752487200\n", " nanos: 125309119\n", " }\n", " start_workflow_result {\n", " workflow_id: \"orchestrator-2025-07-14T10:00:00Z\"\n", " run_id: \"01980860-8578-759d-97fa-dbdfc784f88a\"\n", " }\n", " start_workflow_status: WORKFLOW_EXECUTION_STATUS_COMPLETED\n", " }\n", " recent_actions {\n", " schedule_time {\n", " seconds: 1752490800\n", " }\n", " actual_time {\n", " seconds: 1752490800\n", " nanos: 124356906\n", " }\n", " start_workflow_result {\n", " workflow_id: \"orchestrator-2025-07-14T11:00:00Z\"\n", " run_id: \"01980897-73f7-7e83-a7a6-20eaac38e7d4\"\n", " }\n", " start_workflow_status: WORKFLOW_EXECUTION_STATUS_COMPLETED\n", " }\n", " recent_actions {\n", " schedule_time {\n", " seconds: 1752494400\n", " }\n", " actual_time {\n", " seconds: 1752494400\n", " nanos: 167329351\n", " }\n", " start_workflow_result {\n", " workflow_id: \"orchestrator-2025-07-14T12:00:00Z\"\n", " run_id: \"019808ce-62a2-75d6-b11d-c5387cfa258f\"\n", " }\n", " start_workflow_status: WORKFLOW_EXECUTION_STATUS_COMPLETED\n", " }\n", " recent_actions {\n", " schedule_time {\n", " seconds: 1752498000\n", " }\n", " actual_time {\n", " seconds: 1752498000\n", " nanos: 167575982\n", " }\n", " start_workflow_result {\n", " workflow_id: \"orchestrator-2025-07-14T13:00:00Z\"\n", " run_id: \"01980905-5122-7807-90ae-3782d0332e79\"\n", " }\n", " start_workflow_status: WORKFLOW_EXECUTION_STATUS_RUNNING\n", " }\n", " future_action_times {\n", " seconds: 1752501600\n", " }\n", " future_action_times {\n", " seconds: 1752505200\n", " }\n", " future_action_times {\n", " seconds: 1752508800\n", " }\n", " future_action_times {\n", " seconds: 1752512400\n", " }\n", " future_action_times {\n", " seconds: 1752516000\n", " }\n", "}\n", "}\n", "{'_client': , 'id': 'orchestrator'}\n" ] } ], "source": [ "async for schedule in await temporal_client.list_schedules():\n", " print(vars(schedule))\n", " id = schedule.id\n", "\n", " handle = temporal_client.get_schedule_handle(id)\n", "\n", " print(vars(handle))" ] }, { "cell_type": "code", "execution_count": 13, "id": "988d1718", "metadata": {}, "outputs": [], "source": [ "schedules_config = {\n", " \"scouter-opcua-orchestrated-pipeline\": schedules_config['scouter-opcua-orchestrated-pipeline'],\n", "}" ] }, { "cell_type": "code", "execution_count": 14, "id": "c12f5e75", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "scouter-opcua-orchestrated-pipeline {'frequency': 5, 'data': {'filters': {'NULL_VALUES_FILTER': {'policy': 'DISCARD'}, 'OUT_OF_BOUNDS_FILTER': {'policy': 'DISCARD'}}, 'frequency': '5s', 'max_retry_policy': 1, 'model_id': '1', 'model_name': 'Demo Model-Demo2', 'model_tags': {'Counter': {'aggr_func': 'avg', 'data_range': [-100, 100]}}, 'retention_time': 3600, 'schedule_name': 'scouter-opcua-orchestrated-pipeline', 'schema': 'sientia_data', 'table_name': 'laborious_data', 'topic': 'raw_scouter-opcua-orchestrated-pipeline', 'trigger_laborious': False, 'workflow_type': 'scouter'}, 'handle': }\n", "scouter-opcua-orchestrated-pipeline\n", "{'workflow': 'scouter', 'args': [metadata {\n", " key: \"encoding\"\n", " value: \"json/plain\"\n", "}\n", "data: \"{\\\"filters\\\":{\\\"NULL_VALUES_FILTER\\\":{\\\"policy\\\":\\\"DISCARD\\\"},\\\"OUT_OF_BOUNDS_FILTER\\\":{\\\"policy\\\":\\\"DISCARD\\\"}},\\\"frequency\\\":\\\"5s\\\",\\\"max_retry_policy\\\":1,\\\"model_id\\\":\\\"1\\\",\\\"model_name\\\":\\\"Demo Model-Demo2\\\",\\\"model_tags\\\":{\\\"Counter\\\":{\\\"aggr_func\\\":\\\"avg\\\",\\\"data_range\\\":[-100,100]}},\\\"retention_time\\\":3600,\\\"schedule_name\\\":\\\"scouter-opcua-orchestrated-pipeline\\\",\\\"schema\\\":\\\"sientia_data\\\",\\\"table_name\\\":\\\"laborious_data\\\",\\\"topic\\\":\\\"raw_scouter-opcua-orchestrated-pipeline\\\",\\\"trigger_laborious\\\":false,\\\"workflow_type\\\":\\\"scouter\\\"}\"\n", "], 'id': 'scouter-opcua-orchestrated-pipeline', 'task_queue': 'scouter-queue', 'execution_timeout': datetime.timedelta(seconds=120), 'run_timeout': None, 'task_timeout': None, 'retry_policy': None, 'memo': None, 'typed_search_attributes': TypedSearchAttributes(search_attributes=[]), 'headers': None, 'untyped_search_attributes': {}, 'static_summary': None, 'static_details': None, 'priority': Priority(priority_key=None)}\n" ] } ], "source": [ "from temporalio.client import Client, ScheduleUpdateInput, ScheduleUpdate, ScheduleSpec\n", "\n", "\n", "for id, schedule in schedules_config.items():\n", " print(id, schedule)\n", " \n", " if schedule['frequency'] != 600:\n", " print(id)\n", "\n", " handle = schedule['handle']\n", " \n", " async def update_schedule(input: ScheduleUpdateInput) -> ScheduleUpdate:\n", " schedule_action = input.description.schedule.action\n", "\n", " print(schedule_action.__dict__)\n", " \n", " if hasattr(schedule_action, 'args'):\n", " schedule_action.args = [{}]\n", " \n", " # Atualiza o intervalo de execução\n", " input.description.schedule.spec.intervals = [\n", " ScheduleIntervalSpec(every=timedelta(minutes=10))\n", " ]\n", " \n", " return ScheduleUpdate(schedule=input.description.schedule)\n", "\n", " await handle.update(update_schedule)\n", " \n" ] }, { "cell_type": "code", "execution_count": 1, "id": "87eb10c8", "metadata": {}, "outputs": [], "source": [ "from redis import Redis\n", "\n", "redis = Redis(host='localhost', port=6379)" ] }, { "cell_type": "code", "execution_count": 6, "id": "fe4ad9dc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Chave: slot:opc_tags:2, Valor: {\n", "\"slot2\": \"value\"\n", "}\n", "Chave: slot:opc_tags:1, Valor: {\n", "\"slot1\": \"value\"\n", "}\n" ] } ], "source": [ "matching_keys = redis.keys(\"slot:opc_tags:*\")\n", "\n", "if matching_keys:\n", " decoded_keys = [key.decode('utf-8') for key in matching_keys]\n", " values = redis.mget(decoded_keys)\n", "\n", " items = {}\n", " for i, key in enumerate(decoded_keys):\n", " value = values[i]\n", " if value is not None:\n", " try:\n", " items[key] = value.decode('utf-8')\n", " except (UnicodeDecodeError, AttributeError):\n", " items[key] = value\n", " else:\n", " items[key] = None\n", "\n", " for key, value in items.items():\n", " print(f\"Chave: {key}, Valor: {value}\")" ] }, { "cell_type": "markdown", "id": "e8025cbb", "metadata": {}, "source": [ "## Model Monitoring" ] }, { "cell_type": "code", "execution_count": 1, "id": "87a9dc89", "metadata": {}, "outputs": [], "source": [ "from sientia.ModelAnalysis import ModelAnalysis\n", "\n", "config = {\n", " \"target\": \"Square\",\n", " \"prediction\": \"prediction\",\n", " \"timestamp\": \"timestamp\",\n", " \"features\": [\"Counter\", \"Rollout\"]\n", "}\n", "\n", "model = ModelAnalysis(config=config)\n", "\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "2a74ad2f", "metadata": {}, "outputs": [], "source": [ "from sientia_do.temporal.activities.postgres import Postgres\n", "from unittest.mock import AsyncMock\n", "from pandas import DataFrame\n", "\n", "postgres = Postgres(\n", " host=\"localhost\",\n", " port=5432,\n", " dbname=\"sientia\",\n", " user=\"sientia\",\n", " password=\"sientia\",\n", " min_connections=1,\n", " max_connections=10,\n", " logger=AsyncMock(),\n", " notification_handler=AsyncMock(),\n", " metrics_controller=AsyncMock()\n", ")\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "938cecbd", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:83: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", " self.logger.custom_info(message, metadata)\n", "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:59: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", " self.metrics_controller.start()\n", "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:83: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", " self.logger.custom_info(message, metadata)\n", "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:95: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", " self.logger.custom_debug(message, metadata)\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
variablevaluepredictiontimestamp
0Square42.2950-3.0184652025-11-12 12:44:50+00:00
1Rollout1.8610-3.0184652025-11-12 12:44:50+00:00
2Counter-45.0980-3.0184652025-11-12 12:44:50+00:00
3Rollout0.39702.3414232025-11-12 12:44:20+00:00
4Counter-40.02602.3414232025-11-12 12:44:20+00:00
5Square43.31902.3414232025-11-12 12:44:20+00:00
6Rollout-0.26454.6965912025-11-12 12:42:15+00:00
7Square42.18504.6965912025-11-12 12:42:15+00:00
8Counter-37.83854.6965912025-11-12 12:42:15+00:00
9Square2.8880-70.8451572025-11-11 23:00:48+00:00
10Rollout18.7945-70.8451572025-11-11 23:00:48+00:00
11Counter-12.6565-70.8451572025-11-11 23:00:48+00:00
12Rollout17.7415-71.8945902025-11-11 23:00:18+00:00
13Counter-11.1570-71.8945902025-11-11 23:00:18+00:00
14Square2.1610-71.8945902025-11-11 23:00:18+00:00
15Square-2.0490-67.0339532025-11-11 22:59:48+00:00
16Rollout16.5365-67.0339532025-11-11 22:59:48+00:00
17Counter-15.2205-67.0339532025-11-11 22:59:48+00:00
18Counter-11.7930-70.5144622025-11-11 22:59:18+00:00
19Square-4.3350-70.5144622025-11-11 22:59:18+00:00
20Rollout16.3065-70.5144622025-11-11 22:59:18+00:00
21Square0.6840-72.5232792025-11-11 22:58:48+00:00
22Counter-9.2570-72.5232792025-11-11 22:58:48+00:00
23Rollout14.9965-72.5232792025-11-11 22:58:48+00:00
24Square1.1700-72.8316842025-11-11 22:58:18+00:00
25Counter-7.0635-72.8316842025-11-11 22:58:18+00:00
26Rollout10.9875-72.8316842025-11-11 22:58:18+00:00
27Counter-2.0110-77.8659442025-11-11 22:57:48+00:00
28Rollout10.4545-77.8659442025-11-11 22:57:48+00:00
29Square-0.2070-77.8659442025-11-11 22:57:48+00:00
\n", "
" ], "text/plain": [ " variable value prediction timestamp\n", "0 Square 42.2950 -3.018465 2025-11-12 12:44:50+00:00\n", "1 Rollout 1.8610 -3.018465 2025-11-12 12:44:50+00:00\n", "2 Counter -45.0980 -3.018465 2025-11-12 12:44:50+00:00\n", "3 Rollout 0.3970 2.341423 2025-11-12 12:44:20+00:00\n", "4 Counter -40.0260 2.341423 2025-11-12 12:44:20+00:00\n", "5 Square 43.3190 2.341423 2025-11-12 12:44:20+00:00\n", "6 Rollout -0.2645 4.696591 2025-11-12 12:42:15+00:00\n", "7 Square 42.1850 4.696591 2025-11-12 12:42:15+00:00\n", "8 Counter -37.8385 4.696591 2025-11-12 12:42:15+00:00\n", "9 Square 2.8880 -70.845157 2025-11-11 23:00:48+00:00\n", "10 Rollout 18.7945 -70.845157 2025-11-11 23:00:48+00:00\n", "11 Counter -12.6565 -70.845157 2025-11-11 23:00:48+00:00\n", "12 Rollout 17.7415 -71.894590 2025-11-11 23:00:18+00:00\n", "13 Counter -11.1570 -71.894590 2025-11-11 23:00:18+00:00\n", "14 Square 2.1610 -71.894590 2025-11-11 23:00:18+00:00\n", "15 Square -2.0490 -67.033953 2025-11-11 22:59:48+00:00\n", "16 Rollout 16.5365 -67.033953 2025-11-11 22:59:48+00:00\n", "17 Counter -15.2205 -67.033953 2025-11-11 22:59:48+00:00\n", "18 Counter -11.7930 -70.514462 2025-11-11 22:59:18+00:00\n", "19 Square -4.3350 -70.514462 2025-11-11 22:59:18+00:00\n", "20 Rollout 16.3065 -70.514462 2025-11-11 22:59:18+00:00\n", "21 Square 0.6840 -72.523279 2025-11-11 22:58:48+00:00\n", "22 Counter -9.2570 -72.523279 2025-11-11 22:58:48+00:00\n", "23 Rollout 14.9965 -72.523279 2025-11-11 22:58:48+00:00\n", "24 Square 1.1700 -72.831684 2025-11-11 22:58:18+00:00\n", "25 Counter -7.0635 -72.831684 2025-11-11 22:58:18+00:00\n", "26 Rollout 10.9875 -72.831684 2025-11-11 22:58:18+00:00\n", "27 Counter -2.0110 -77.865944 2025-11-11 22:57:48+00:00\n", "28 Rollout 10.4545 -77.865944 2025-11-11 22:57:48+00:00\n", "29 Square -0.2070 -77.865944 2025-11-11 22:57:48+00:00" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
variablevaluepredictiontimestamp
0Square42.2950-3.0184652025-11-12 12:44:50
1Rollout1.8610-3.0184652025-11-12 12:44:50
2Counter-45.0980-3.0184652025-11-12 12:44:50
3Rollout0.39702.3414232025-11-12 12:44:20
4Counter-40.02602.3414232025-11-12 12:44:20
5Square43.31902.3414232025-11-12 12:44:20
6Rollout-0.26454.6965912025-11-12 12:42:15
7Square42.18504.6965912025-11-12 12:42:15
8Counter-37.83854.6965912025-11-12 12:42:15
9Square2.8880-70.8451572025-11-11 23:00:48
10Rollout18.7945-70.8451572025-11-11 23:00:48
11Counter-12.6565-70.8451572025-11-11 23:00:48
12Rollout17.7415-71.8945902025-11-11 23:00:18
13Counter-11.1570-71.8945902025-11-11 23:00:18
14Square2.1610-71.8945902025-11-11 23:00:18
15Square-2.0490-67.0339532025-11-11 22:59:48
16Rollout16.5365-67.0339532025-11-11 22:59:48
17Counter-15.2205-67.0339532025-11-11 22:59:48
18Counter-11.7930-70.5144622025-11-11 22:59:18
19Square-4.3350-70.5144622025-11-11 22:59:18
20Rollout16.3065-70.5144622025-11-11 22:59:18
21Square0.6840-72.5232792025-11-11 22:58:48
22Counter-9.2570-72.5232792025-11-11 22:58:48
23Rollout14.9965-72.5232792025-11-11 22:58:48
24Square1.1700-72.8316842025-11-11 22:58:18
25Counter-7.0635-72.8316842025-11-11 22:58:18
26Rollout10.9875-72.8316842025-11-11 22:58:18
27Counter-2.0110-77.8659442025-11-11 22:57:48
28Rollout10.4545-77.8659442025-11-11 22:57:48
29Square-0.2070-77.8659442025-11-11 22:57:48
\n", "
" ], "text/plain": [ " variable value prediction timestamp\n", "0 Square 42.2950 -3.018465 2025-11-12 12:44:50\n", "1 Rollout 1.8610 -3.018465 2025-11-12 12:44:50\n", "2 Counter -45.0980 -3.018465 2025-11-12 12:44:50\n", "3 Rollout 0.3970 2.341423 2025-11-12 12:44:20\n", "4 Counter -40.0260 2.341423 2025-11-12 12:44:20\n", "5 Square 43.3190 2.341423 2025-11-12 12:44:20\n", "6 Rollout -0.2645 4.696591 2025-11-12 12:42:15\n", "7 Square 42.1850 4.696591 2025-11-12 12:42:15\n", "8 Counter -37.8385 4.696591 2025-11-12 12:42:15\n", "9 Square 2.8880 -70.845157 2025-11-11 23:00:48\n", "10 Rollout 18.7945 -70.845157 2025-11-11 23:00:48\n", "11 Counter -12.6565 -70.845157 2025-11-11 23:00:48\n", "12 Rollout 17.7415 -71.894590 2025-11-11 23:00:18\n", "13 Counter -11.1570 -71.894590 2025-11-11 23:00:18\n", "14 Square 2.1610 -71.894590 2025-11-11 23:00:18\n", "15 Square -2.0490 -67.033953 2025-11-11 22:59:48\n", "16 Rollout 16.5365 -67.033953 2025-11-11 22:59:48\n", "17 Counter -15.2205 -67.033953 2025-11-11 22:59:48\n", "18 Counter -11.7930 -70.514462 2025-11-11 22:59:18\n", "19 Square -4.3350 -70.514462 2025-11-11 22:59:18\n", "20 Rollout 16.3065 -70.514462 2025-11-11 22:59:18\n", "21 Square 0.6840 -72.523279 2025-11-11 22:58:48\n", "22 Counter -9.2570 -72.523279 2025-11-11 22:58:48\n", "23 Rollout 14.9965 -72.523279 2025-11-11 22:58:48\n", "24 Square 1.1700 -72.831684 2025-11-11 22:58:18\n", "25 Counter -7.0635 -72.831684 2025-11-11 22:58:18\n", "26 Rollout 10.9875 -72.831684 2025-11-11 22:58:18\n", "27 Counter -2.0110 -77.865944 2025-11-11 22:57:48\n", "28 Rollout 10.4545 -77.865944 2025-11-11 22:57:48\n", "29 Square -0.2070 -77.865944 2025-11-11 22:57:48" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from pandas import DataFrame, to_datetime\n", "\n", "query_data = \"\"\"\n", "select ld.variable, ld.value, p.prediction, p.\"timestamp\"\n", "from sientia_data.laborious_data ld\n", " right join sientia_data.predictions p \n", " on ld.timestamp = p.timestamp\n", " and ld.model_id = p.model_id\n", " where\n", " ld.model_id = '1'\n", " order by\n", " ld.created_at desc limit 30;\n", "\"\"\"\n", "\n", "input_data = {\n", " \"query\": query_data,\n", " \"metadata\": {}\n", "}\n", "\n", "\n", "raw_data = DataFrame(await postgres.load_custom_query(\n", " input_data))\n", "\n", "raw_data['timestamp'] = to_datetime(raw_data['timestamp'])\n", "raw_data['timestamp'] = raw_data['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')\n", "\n", "display(raw_data)" ] }, { "cell_type": "code", "execution_count": 7, "id": "297f648f", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
timestamppredictionCounterRolloutSquare
02025-11-11 22:57:48-77.865944-2.011010.4545-0.207
12025-11-11 22:58:18-72.831684-7.063510.98751.170
22025-11-11 22:58:48-72.523279-9.257014.99650.684
32025-11-11 22:59:18-70.514462-11.793016.3065-4.335
42025-11-11 22:59:48-67.033953-15.220516.5365-2.049
52025-11-11 23:00:18-71.894590-11.157017.74152.161
62025-11-11 23:00:48-70.845157-12.656518.79452.888
72025-11-12 12:42:154.696591-37.8385-0.264542.185
82025-11-12 12:44:202.341423-40.02600.397043.319
92025-11-12 12:44:50-3.018465-45.09801.861042.295
\n", "
" ], "text/plain": [ " timestamp prediction Counter Rollout Square\n", "0 2025-11-11 22:57:48 -77.865944 -2.0110 10.4545 -0.207\n", "1 2025-11-11 22:58:18 -72.831684 -7.0635 10.9875 1.170\n", "2 2025-11-11 22:58:48 -72.523279 -9.2570 14.9965 0.684\n", "3 2025-11-11 22:59:18 -70.514462 -11.7930 16.3065 -4.335\n", "4 2025-11-11 22:59:48 -67.033953 -15.2205 16.5365 -2.049\n", "5 2025-11-11 23:00:18 -71.894590 -11.1570 17.7415 2.161\n", "6 2025-11-11 23:00:48 -70.845157 -12.6565 18.7945 2.888\n", "7 2025-11-12 12:42:15 4.696591 -37.8385 -0.2645 42.185\n", "8 2025-11-12 12:44:20 2.341423 -40.0260 0.3970 43.319\n", "9 2025-11-12 12:44:50 -3.018465 -45.0980 1.8610 42.295" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from pandas import merge, pivot\n", "\n", "raw_data.drop_duplicates(subset=[\"timestamp\", \"prediction\", \"variable\"], inplace=True, keep=\"first\")\n", "data = raw_data.pivot(index=[\"timestamp\", \"prediction\"], columns=\"variable\", values=\"value\")\n", "\n", "data.columns.name = None\n", "data.reset_index(inplace=True)\n", "display(data)\n", "\n", "\n", "drift_methods = ['kolmogorov_smirnov']#, 'jensen_shannon', 'wasserstein']\n", "chunk_period = 's'\n", "target_col = \"Square\"\n", "timestamp_col = \"timestamp\"\n", "\n", "data.to_csv(\"data.csv\", index=False)\n", "\n" ] }, { "cell_type": "code", "execution_count": 10, "id": "9ab4774a", "metadata": {}, "outputs": [], "source": [ "from mlflow.tracking import MlflowClient\n", "from os import environ\n", "\n", "environ['MLFLOW_TRACKING_USERNAME'] = 'aignosi'\n", "environ['MLFLOW_TRACKING_PASSWORD'] = '1L0FP50j3ncp123'\n", "environ['MLFLOW_TRACKING_URI'] = 'http://localhost:5080'\n", "mlflow_client = MlflowClient()" ] }, { "cell_type": "code", "execution_count": 19, "id": "e8427625", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "run_id = '9e9fa748822c44ab92dd769d6a45c4f4'\n", "output_dir = './tmp/model'\n", "\n", "artifacts = mlflow_client.list_artifacts(run_id)\n", "paths = [artifact.path for artifact in artifacts]\n", "\n", "any(artifact.path == 'retrain_data.csv' for artifact in artifacts)" ] }, { "cell_type": "code", "execution_count": 12, "id": "cf4ddf41", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
timestampCounterRolloutSquare
02025-11-10 18:58:3913.5970-52.450043.381
12025-11-10 18:58:4915.0660-50.024045.813
22025-11-10 18:58:5414.2780-49.481047.677
32025-11-10 18:59:0414.2080-51.801548.995
42025-11-10 18:59:0916.3530-52.685047.745
52025-11-10 18:59:1915.5415-53.562548.327
62025-11-10 18:59:2416.8580-53.696047.160
72025-11-10 18:59:3414.9745-54.751048.480
82025-11-10 18:59:3912.3020-53.433048.631
92025-11-10 18:59:4912.5640-52.980550.130
\n", "
" ], "text/plain": [ " timestamp Counter Rollout Square\n", "0 2025-11-10 18:58:39 13.5970 -52.4500 43.381\n", "1 2025-11-10 18:58:49 15.0660 -50.0240 45.813\n", "2 2025-11-10 18:58:54 14.2780 -49.4810 47.677\n", "3 2025-11-10 18:59:04 14.2080 -51.8015 48.995\n", "4 2025-11-10 18:59:09 16.3530 -52.6850 47.745\n", "5 2025-11-10 18:59:19 15.5415 -53.5625 48.327\n", "6 2025-11-10 18:59:24 16.8580 -53.6960 47.160\n", "7 2025-11-10 18:59:34 14.9745 -54.7510 48.480\n", "8 2025-11-10 18:59:39 12.3020 -53.4330 48.631\n", "9 2025-11-10 18:59:49 12.5640 -52.9805 50.130" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import mlflow\n", "from io import StringIO\n", "from pandas import read_csv\n", "\n", "if 'retrain_data.csv' in paths:\n", " artifact_path = 'retrain_data.csv'\n", "elif 'train_data.csv' in paths:\n", " artifact_path = 'train_data.csv'\n", "else:\n", " raise RuntimeError(\n", " f\"Could not find 'train_data.csv' or 'retrain_data.csv' for run_id '{run_id}'.\\n\"\n", " )\n", "\n", "# Carregar diretamente na memória como string\n", "artifact_content = mlflow.artifacts.load_text(\n", " f\"runs:/{run_id}/{artifact_path}\"\n", ")\n", "\n", "reference_df = read_csv(StringIO(artifact_content))\n", "\n", "reference_df.drop(columns=['timestamp.1'], inplace=True)\n", "\n", "reference_df['timestamp'] = to_datetime(reference_df['timestamp'])\n", "reference_df['timestamp'] = reference_df['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')\n", "\n", "display(reference_df)\n", "\n", "reference_df.to_csv(\"reference_df.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": 20, "id": "d375752b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/numpy/core/_methods.py:206: RuntimeWarning: Degrees of freedom <= 0 for slice\n", " ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,\n", "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/numpy/core/_methods.py:198: RuntimeWarning: invalid value encountered in scalar divide\n", " ret = ret.dtype.type(ret / rcount)\n", "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/numpy/core/_methods.py:206: RuntimeWarning: Degrees of freedom <= 0 for slice\n", " ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,\n", "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/numpy/core/_methods.py:198: RuntimeWarning: invalid value encountered in scalar divide\n", " ret = ret.dtype.type(ret / rcount)\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
timestampfeaturemetricstatisticp_valuealertchunk_indexchunk_start_datechunk_end_date
02025-11-10 18:58:00+0000Counterkolmogorov_smirnov3.000000e-01NaNFalse02025-11-10 18:582025-11-10 18:58:59.999999999
12025-11-10 18:59:00+0000Counterkolmogorov_smirnov1.285714e-01NaNFalse12025-11-10 18:592025-11-10 18:59:59.999999999
22025-11-11 22:57:00+0000Counterkolmogorov_smirnov1.000000e+00NaNTrue02025-11-11 22:572025-11-11 22:57:59.999999999
32025-11-11 22:58:00+0000Counterkolmogorov_smirnov1.000000e+00NaNTrue12025-11-11 22:582025-11-11 22:58:59.999999999
42025-11-11 22:59:00+0000Counterkolmogorov_smirnov1.000000e+00NaNTrue22025-11-11 22:592025-11-11 22:59:59.999999999
52025-11-11 23:00:00+0000Counterkolmogorov_smirnov1.000000e+00NaNTrue32025-11-11 23:002025-11-11 23:00:59.999999999
62025-11-12 12:42:00+0000Counterkolmogorov_smirnov1.000000e+00NaNTrue42025-11-12 12:422025-11-12 12:42:59.999999999
72025-11-12 12:44:00+0000Counterkolmogorov_smirnov1.000000e+00NaNTrue52025-11-12 12:442025-11-12 12:44:59.999999999
82025-11-10 18:58:00+0000Rolloutkolmogorov_smirnov6.000000e-01NaNFalse02025-11-10 18:582025-11-10 18:58:59.999999999
92025-11-10 18:59:00+0000Rolloutkolmogorov_smirnov2.571429e-01NaNFalse12025-11-10 18:592025-11-10 18:59:59.999999999
102025-11-11 22:57:00+0000Rolloutkolmogorov_smirnov1.000000e+00NaNTrue02025-11-11 22:572025-11-11 22:57:59.999999999
112025-11-11 22:58:00+0000Rolloutkolmogorov_smirnov1.000000e+00NaNTrue12025-11-11 22:582025-11-11 22:58:59.999999999
122025-11-11 22:59:00+0000Rolloutkolmogorov_smirnov1.000000e+00NaNTrue22025-11-11 22:592025-11-11 22:59:59.999999999
132025-11-11 23:00:00+0000Rolloutkolmogorov_smirnov1.000000e+00NaNTrue32025-11-11 23:002025-11-11 23:00:59.999999999
142025-11-12 12:42:00+0000Rolloutkolmogorov_smirnov1.000000e+00NaNTrue42025-11-12 12:422025-11-12 12:42:59.999999999
152025-11-12 12:44:00+0000Rolloutkolmogorov_smirnov1.000000e+00NaNTrue52025-11-12 12:442025-11-12 12:44:59.999999999
162025-11-10 18:58:00+0000multivariatemultivariate2.015043e-163.748123e-17False02025-11-10 18:582025-11-10 18:58:59.999999999
172025-11-10 18:59:00+0000multivariatemultivariate2.493220e-163.998584e-17False12025-11-10 18:592025-11-10 18:59:59.999999999
182025-11-11 22:57:00+0000multivariatemultivariate1.432145e-14NaNTrue02025-11-11 22:572025-11-11 22:57:59.999999999
192025-11-11 22:58:00+0000multivariatemultivariate1.234840e-142.299811e-15True12025-11-11 22:582025-11-11 22:58:59.999999999
202025-11-11 22:59:00+0000multivariatemultivariate7.944109e-150.000000e+00True22025-11-11 22:592025-11-11 22:59:59.999999999
212025-11-11 23:00:00+0000multivariatemultivariate7.524768e-154.193410e-16True32025-11-11 23:002025-11-11 23:00:59.999999999
222025-11-12 12:42:00+0000multivariatemultivariate1.004859e-14NaNTrue42025-11-12 12:422025-11-12 12:42:59.999999999
232025-11-12 12:44:00+0000multivariatemultivariate1.004859e-140.000000e+00True52025-11-12 12:442025-11-12 12:44:59.999999999
\n", "
" ], "text/plain": [ " timestamp feature metric statistic \\\n", "0 2025-11-10 18:58:00+0000 Counter kolmogorov_smirnov 3.000000e-01 \n", "1 2025-11-10 18:59:00+0000 Counter kolmogorov_smirnov 1.285714e-01 \n", "2 2025-11-11 22:57:00+0000 Counter kolmogorov_smirnov 1.000000e+00 \n", "3 2025-11-11 22:58:00+0000 Counter kolmogorov_smirnov 1.000000e+00 \n", "4 2025-11-11 22:59:00+0000 Counter kolmogorov_smirnov 1.000000e+00 \n", "5 2025-11-11 23:00:00+0000 Counter kolmogorov_smirnov 1.000000e+00 \n", "6 2025-11-12 12:42:00+0000 Counter kolmogorov_smirnov 1.000000e+00 \n", "7 2025-11-12 12:44:00+0000 Counter kolmogorov_smirnov 1.000000e+00 \n", "8 2025-11-10 18:58:00+0000 Rollout kolmogorov_smirnov 6.000000e-01 \n", "9 2025-11-10 18:59:00+0000 Rollout kolmogorov_smirnov 2.571429e-01 \n", "10 2025-11-11 22:57:00+0000 Rollout kolmogorov_smirnov 1.000000e+00 \n", "11 2025-11-11 22:58:00+0000 Rollout kolmogorov_smirnov 1.000000e+00 \n", "12 2025-11-11 22:59:00+0000 Rollout kolmogorov_smirnov 1.000000e+00 \n", "13 2025-11-11 23:00:00+0000 Rollout kolmogorov_smirnov 1.000000e+00 \n", "14 2025-11-12 12:42:00+0000 Rollout kolmogorov_smirnov 1.000000e+00 \n", "15 2025-11-12 12:44:00+0000 Rollout kolmogorov_smirnov 1.000000e+00 \n", "16 2025-11-10 18:58:00+0000 multivariate multivariate 2.015043e-16 \n", "17 2025-11-10 18:59:00+0000 multivariate multivariate 2.493220e-16 \n", "18 2025-11-11 22:57:00+0000 multivariate multivariate 1.432145e-14 \n", "19 2025-11-11 22:58:00+0000 multivariate multivariate 1.234840e-14 \n", "20 2025-11-11 22:59:00+0000 multivariate multivariate 7.944109e-15 \n", "21 2025-11-11 23:00:00+0000 multivariate multivariate 7.524768e-15 \n", "22 2025-11-12 12:42:00+0000 multivariate multivariate 1.004859e-14 \n", "23 2025-11-12 12:44:00+0000 multivariate multivariate 1.004859e-14 \n", "\n", " p_value alert chunk_index chunk_start_date \\\n", "0 NaN False 0 2025-11-10 18:58 \n", "1 NaN False 1 2025-11-10 18:59 \n", "2 NaN True 0 2025-11-11 22:57 \n", "3 NaN True 1 2025-11-11 22:58 \n", "4 NaN True 2 2025-11-11 22:59 \n", "5 NaN True 3 2025-11-11 23:00 \n", "6 NaN True 4 2025-11-12 12:42 \n", "7 NaN True 5 2025-11-12 12:44 \n", "8 NaN False 0 2025-11-10 18:58 \n", "9 NaN False 1 2025-11-10 18:59 \n", "10 NaN True 0 2025-11-11 22:57 \n", "11 NaN True 1 2025-11-11 22:58 \n", "12 NaN True 2 2025-11-11 22:59 \n", "13 NaN True 3 2025-11-11 23:00 \n", "14 NaN True 4 2025-11-12 12:42 \n", "15 NaN True 5 2025-11-12 12:44 \n", "16 3.748123e-17 False 0 2025-11-10 18:58 \n", "17 3.998584e-17 False 1 2025-11-10 18:59 \n", "18 NaN True 0 2025-11-11 22:57 \n", "19 2.299811e-15 True 1 2025-11-11 22:58 \n", "20 0.000000e+00 True 2 2025-11-11 22:59 \n", "21 4.193410e-16 True 3 2025-11-11 23:00 \n", "22 NaN True 4 2025-11-12 12:42 \n", "23 0.000000e+00 True 5 2025-11-12 12:44 \n", "\n", " chunk_end_date \n", "0 2025-11-10 18:58:59.999999999 \n", "1 2025-11-10 18:59:59.999999999 \n", "2 2025-11-11 22:57:59.999999999 \n", "3 2025-11-11 22:58:59.999999999 \n", "4 2025-11-11 22:59:59.999999999 \n", "5 2025-11-11 23:00:59.999999999 \n", "6 2025-11-12 12:42:59.999999999 \n", "7 2025-11-12 12:44:59.999999999 \n", "8 2025-11-10 18:58:59.999999999 \n", "9 2025-11-10 18:59:59.999999999 \n", "10 2025-11-11 22:57:59.999999999 \n", "11 2025-11-11 22:58:59.999999999 \n", "12 2025-11-11 22:59:59.999999999 \n", "13 2025-11-11 23:00:59.999999999 \n", "14 2025-11-12 12:42:59.999999999 \n", "15 2025-11-12 12:44:59.999999999 \n", "16 2025-11-10 18:58:59.999999999 \n", "17 2025-11-10 18:59:59.999999999 \n", "18 2025-11-11 22:57:59.999999999 \n", "19 2025-11-11 22:58:59.999999999 \n", "20 2025-11-11 22:59:59.999999999 \n", "21 2025-11-11 23:00:59.999999999 \n", "22 2025-11-12 12:42:59.999999999 \n", "23 2025-11-12 12:44:59.999999999 " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "\n", "\n", "from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ\n", "\n", "\n", "univariate_drift = model.detect_univariate_drift(\n", " reference_df=reference_df,\n", " analysis_df=data,\n", " features=config['features'],\n", " timestamp_col=timestamp_col,\n", " methods=drift_methods,\n", " chunk_period=\"min\"\n", ")\n", "multivariate_drift = model.detect_multivariate_drift(\n", " reference_df=reference_df,\n", " analysis_df=data,\n", " features=config['features'],\n", " timestamp_col=timestamp_col,\n", " chunk_period=\"min\"\n", ")\n", "\n", "drift = model.get_drift_metrics_dataframe(\n", " univariate_drift=univariate_drift,\n", " multivariate_drift=multivariate_drift\n", ")\n", "drift.drop_duplicates(subset=[\"timestamp\", \"feature\", \"metric\"], inplace=True, keep=\"first\")\n", "drift.reset_index(drop=True, inplace=True)\n", "\n", "drift['timestamp'] = to_datetime(drift['timestamp'])\n", "print(type(drift['timestamp'].iloc[0]))\n", "\n", "# Add timezone UTC to timestamp\n", "drift['timestamp'] = drift['timestamp'].dt.tz_localize('UTC')\n", "drift['timestamp'] = drift['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)\n", "\n", "display(drift)\n", "drift.to_csv(\"drift.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": 32, "id": "d30f6475", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
timestampfeaturemetricstatisticp_valuealertchunk_indexchunk_start_datechunk_end_date
102025-11-11 11:55:18Counterkolmogorov_smirnov1.000000e+00NoneFalse02025-11-11 11:55:182025-11-11 11:55:18.999999999
112025-11-11 11:55:48Counterkolmogorov_smirnov1.000000e+00NoneFalse12025-11-11 11:55:482025-11-11 11:55:48.999999999
122025-11-11 11:56:18Counterkolmogorov_smirnov1.000000e+00NoneFalse22025-11-11 11:56:182025-11-11 11:56:18.999999999
132025-11-11 11:56:48Counterkolmogorov_smirnov1.000000e+00NoneFalse32025-11-11 11:56:482025-11-11 11:56:48.999999999
142025-11-11 11:57:18Counterkolmogorov_smirnov1.000000e+00NoneFalse42025-11-11 11:57:182025-11-11 11:57:18.999999999
152025-11-11 11:57:48Counterkolmogorov_smirnov1.000000e+00NoneFalse52025-11-11 11:57:482025-11-11 11:57:48.999999999
162025-11-11 11:58:18Counterkolmogorov_smirnov1.000000e+00NoneFalse62025-11-11 11:58:182025-11-11 11:58:18.999999999
172025-11-11 11:58:48Counterkolmogorov_smirnov1.000000e+00NoneFalse72025-11-11 11:58:482025-11-11 11:58:48.999999999
182025-11-11 11:59:18Counterkolmogorov_smirnov1.000000e+00NoneFalse82025-11-11 11:59:182025-11-11 11:59:18.999999999
192025-11-11 11:59:48Counterkolmogorov_smirnov1.000000e+00NoneFalse92025-11-11 11:59:482025-11-11 11:59:48.999999999
302025-11-11 11:55:18Rolloutkolmogorov_smirnov1.000000e+00NoneFalse02025-11-11 11:55:182025-11-11 11:55:18.999999999
312025-11-11 11:55:48Rolloutkolmogorov_smirnov1.000000e+00NoneFalse12025-11-11 11:55:482025-11-11 11:55:48.999999999
322025-11-11 11:56:18Rolloutkolmogorov_smirnov1.000000e+00NoneFalse22025-11-11 11:56:182025-11-11 11:56:18.999999999
332025-11-11 11:56:48Rolloutkolmogorov_smirnov1.000000e+00NoneFalse32025-11-11 11:56:482025-11-11 11:56:48.999999999
342025-11-11 11:57:18Rolloutkolmogorov_smirnov1.000000e+00NoneFalse42025-11-11 11:57:182025-11-11 11:57:18.999999999
352025-11-11 11:57:48Rolloutkolmogorov_smirnov1.000000e+00NoneFalse52025-11-11 11:57:482025-11-11 11:57:48.999999999
362025-11-11 11:58:18Rolloutkolmogorov_smirnov1.000000e+00NoneFalse62025-11-11 11:58:182025-11-11 11:58:18.999999999
372025-11-11 11:58:48Rolloutkolmogorov_smirnov1.000000e+00NoneFalse72025-11-11 11:58:482025-11-11 11:58:48.999999999
382025-11-11 11:59:18Rolloutkolmogorov_smirnov1.000000e+00NoneFalse82025-11-11 11:59:182025-11-11 11:59:18.999999999
392025-11-11 11:59:48Rolloutkolmogorov_smirnov1.000000e+00NoneFalse92025-11-11 11:59:482025-11-11 11:59:48.999999999
502025-11-11 11:55:18multivariatemultivariate1.517720e-14NaNTrue02025-11-11 11:55:182025-11-11 11:55:18.999999999
512025-11-11 11:55:48multivariatemultivariate1.588822e-14NaNTrue12025-11-11 11:55:482025-11-11 11:55:48.999999999
522025-11-11 11:56:18multivariatemultivariate1.432145e-14NaNTrue22025-11-11 11:56:182025-11-11 11:56:18.999999999
532025-11-11 11:56:48multivariatemultivariate1.421085e-14NaNTrue32025-11-11 11:56:482025-11-11 11:56:48.999999999
542025-11-11 11:57:18multivariatemultivariate1.421085e-14NaNTrue42025-11-11 11:57:182025-11-11 11:57:18.999999999
552025-11-11 11:57:48multivariatemultivariate1.776357e-15NaNTrue52025-11-11 11:57:482025-11-11 11:57:48.999999999
562025-11-11 11:58:18multivariatemultivariate7.324107e-15NaNTrue62025-11-11 11:58:182025-11-11 11:58:18.999999999
572025-11-11 11:58:48multivariatemultivariate1.464821e-14NaNTrue72025-11-11 11:58:482025-11-11 11:58:48.999999999
582025-11-11 11:59:18multivariatemultivariate1.551137e-14NaNTrue82025-11-11 11:59:182025-11-11 11:59:18.999999999
592025-11-11 11:59:48multivariatemultivariate1.427317e-14NaNTrue92025-11-11 11:59:482025-11-11 11:59:48.999999999
\n", "
" ], "text/plain": [ " timestamp feature metric statistic \\\n", "10 2025-11-11 11:55:18 Counter kolmogorov_smirnov 1.000000e+00 \n", "11 2025-11-11 11:55:48 Counter kolmogorov_smirnov 1.000000e+00 \n", "12 2025-11-11 11:56:18 Counter kolmogorov_smirnov 1.000000e+00 \n", "13 2025-11-11 11:56:48 Counter kolmogorov_smirnov 1.000000e+00 \n", "14 2025-11-11 11:57:18 Counter kolmogorov_smirnov 1.000000e+00 \n", "15 2025-11-11 11:57:48 Counter kolmogorov_smirnov 1.000000e+00 \n", "16 2025-11-11 11:58:18 Counter kolmogorov_smirnov 1.000000e+00 \n", "17 2025-11-11 11:58:48 Counter kolmogorov_smirnov 1.000000e+00 \n", "18 2025-11-11 11:59:18 Counter kolmogorov_smirnov 1.000000e+00 \n", "19 2025-11-11 11:59:48 Counter kolmogorov_smirnov 1.000000e+00 \n", "30 2025-11-11 11:55:18 Rollout kolmogorov_smirnov 1.000000e+00 \n", "31 2025-11-11 11:55:48 Rollout kolmogorov_smirnov 1.000000e+00 \n", "32 2025-11-11 11:56:18 Rollout kolmogorov_smirnov 1.000000e+00 \n", "33 2025-11-11 11:56:48 Rollout kolmogorov_smirnov 1.000000e+00 \n", "34 2025-11-11 11:57:18 Rollout kolmogorov_smirnov 1.000000e+00 \n", "35 2025-11-11 11:57:48 Rollout kolmogorov_smirnov 1.000000e+00 \n", "36 2025-11-11 11:58:18 Rollout kolmogorov_smirnov 1.000000e+00 \n", "37 2025-11-11 11:58:48 Rollout kolmogorov_smirnov 1.000000e+00 \n", "38 2025-11-11 11:59:18 Rollout kolmogorov_smirnov 1.000000e+00 \n", "39 2025-11-11 11:59:48 Rollout kolmogorov_smirnov 1.000000e+00 \n", "50 2025-11-11 11:55:18 multivariate multivariate 1.517720e-14 \n", "51 2025-11-11 11:55:48 multivariate multivariate 1.588822e-14 \n", "52 2025-11-11 11:56:18 multivariate multivariate 1.432145e-14 \n", "53 2025-11-11 11:56:48 multivariate multivariate 1.421085e-14 \n", "54 2025-11-11 11:57:18 multivariate multivariate 1.421085e-14 \n", "55 2025-11-11 11:57:48 multivariate multivariate 1.776357e-15 \n", "56 2025-11-11 11:58:18 multivariate multivariate 7.324107e-15 \n", "57 2025-11-11 11:58:48 multivariate multivariate 1.464821e-14 \n", "58 2025-11-11 11:59:18 multivariate multivariate 1.551137e-14 \n", "59 2025-11-11 11:59:48 multivariate multivariate 1.427317e-14 \n", "\n", " p_value alert chunk_index chunk_start_date \\\n", "10 None False 0 2025-11-11 11:55:18 \n", "11 None False 1 2025-11-11 11:55:48 \n", "12 None False 2 2025-11-11 11:56:18 \n", "13 None False 3 2025-11-11 11:56:48 \n", "14 None False 4 2025-11-11 11:57:18 \n", "15 None False 5 2025-11-11 11:57:48 \n", "16 None False 6 2025-11-11 11:58:18 \n", "17 None False 7 2025-11-11 11:58:48 \n", "18 None False 8 2025-11-11 11:59:18 \n", "19 None False 9 2025-11-11 11:59:48 \n", "30 None False 0 2025-11-11 11:55:18 \n", "31 None False 1 2025-11-11 11:55:48 \n", "32 None False 2 2025-11-11 11:56:18 \n", "33 None False 3 2025-11-11 11:56:48 \n", "34 None False 4 2025-11-11 11:57:18 \n", "35 None False 5 2025-11-11 11:57:48 \n", "36 None False 6 2025-11-11 11:58:18 \n", "37 None False 7 2025-11-11 11:58:48 \n", "38 None False 8 2025-11-11 11:59:18 \n", "39 None False 9 2025-11-11 11:59:48 \n", "50 NaN True 0 2025-11-11 11:55:18 \n", "51 NaN True 1 2025-11-11 11:55:48 \n", "52 NaN True 2 2025-11-11 11:56:18 \n", "53 NaN True 3 2025-11-11 11:56:48 \n", "54 NaN True 4 2025-11-11 11:57:18 \n", "55 NaN True 5 2025-11-11 11:57:48 \n", "56 NaN True 6 2025-11-11 11:58:18 \n", "57 NaN True 7 2025-11-11 11:58:48 \n", "58 NaN True 8 2025-11-11 11:59:18 \n", "59 NaN True 9 2025-11-11 11:59:48 \n", "\n", " chunk_end_date \n", "10 2025-11-11 11:55:18.999999999 \n", "11 2025-11-11 11:55:48.999999999 \n", "12 2025-11-11 11:56:18.999999999 \n", "13 2025-11-11 11:56:48.999999999 \n", "14 2025-11-11 11:57:18.999999999 \n", "15 2025-11-11 11:57:48.999999999 \n", "16 2025-11-11 11:58:18.999999999 \n", "17 2025-11-11 11:58:48.999999999 \n", "18 2025-11-11 11:59:18.999999999 \n", "19 2025-11-11 11:59:48.999999999 \n", "30 2025-11-11 11:55:18.999999999 \n", "31 2025-11-11 11:55:48.999999999 \n", "32 2025-11-11 11:56:18.999999999 \n", "33 2025-11-11 11:56:48.999999999 \n", "34 2025-11-11 11:57:18.999999999 \n", "35 2025-11-11 11:57:48.999999999 \n", "36 2025-11-11 11:58:18.999999999 \n", "37 2025-11-11 11:58:48.999999999 \n", "38 2025-11-11 11:59:18.999999999 \n", "39 2025-11-11 11:59:48.999999999 \n", "50 2025-11-11 11:55:18.999999999 \n", "51 2025-11-11 11:55:48.999999999 \n", "52 2025-11-11 11:56:18.999999999 \n", "53 2025-11-11 11:56:48.999999999 \n", "54 2025-11-11 11:57:18.999999999 \n", "55 2025-11-11 11:57:48.999999999 \n", "56 2025-11-11 11:58:18.999999999 \n", "57 2025-11-11 11:58:48.999999999 \n", "58 2025-11-11 11:59:18.999999999 \n", "59 2025-11-11 11:59:48.999999999 " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# drop rows of drift with timestamp in reference data but not in analysis data\n", "drift_filtered = drift[drift['timestamp'].isin(data['timestamp'])]\n", "display(drift_filtered)\n", "\n", "drift_filtered.to_csv(\"drift_filtered.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "70671022", "metadata": {}, "outputs": [ { "ename": "ValueError", "evalue": "Target column 'Square' not found in DataFrames", "output_type": "error", "traceback": [ "\u001b[31m---------------------------------------------------------------------------\u001b[39m", "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[36]\u001b[39m\u001b[32m, line 6\u001b[39m\n\u001b[32m 3\u001b[39m \u001b[38;5;66;03m# rename column square to prediction\u001b[39;00m\n\u001b[32m 4\u001b[39m performance_reference.rename(columns={\u001b[33m'\u001b[39m\u001b[33mSquare\u001b[39m\u001b[33m'\u001b[39m: \u001b[33m'\u001b[39m\u001b[33mprediction\u001b[39m\u001b[33m'\u001b[39m}, inplace=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m real_performance = \u001b[43mmodel\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcalculate_performance\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 7\u001b[39m \u001b[43m \u001b[49m\u001b[43mreference_df\u001b[49m\u001b[43m=\u001b[49m\u001b[43mperformance_reference\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 8\u001b[39m \u001b[43m \u001b[49m\u001b[43manalysis_df\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 9\u001b[39m \u001b[43m \u001b[49m\u001b[43mtarget_col\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtarget_col\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 10\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimestamp_col\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimestamp_col\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 11\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunk_period\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunk_period\u001b[49m\n\u001b[32m 12\u001b[39m \u001b[43m)\u001b[49m\n\u001b[32m 14\u001b[39m metrics_df = model.get_calculated_metrics_dataframe(real_performance)\n\u001b[32m 15\u001b[39m display(metrics_df)\n", "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/sientia/ModelAnalysis.py:397\u001b[39m, in \u001b[36mModelAnalysis.calculate_performance\u001b[39m\u001b[34m(self, reference_df, analysis_df, target_col, timestamp_col, metrics, chunk_period)\u001b[39m\n\u001b[32m 395\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(DATAFRAMES_EMPTY_ERROR)\n\u001b[32m 396\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m target_col \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m reference_df.columns \u001b[38;5;129;01mor\u001b[39;00m target_col \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m analysis_df.columns:\n\u001b[32m--> \u001b[39m\u001b[32m397\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mTarget column \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtarget_col\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m not found in DataFrames\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 398\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m timestamp_col \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m reference_df.columns \u001b[38;5;129;01mor\u001b[39;00m timestamp_col \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m analysis_df.columns:\n\u001b[32m 399\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mTimestamp column \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtimestamp_col\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m not found in DataFrames\u001b[39m\u001b[33m\"\u001b[39m)\n", "\u001b[31mValueError\u001b[39m: Target column 'Square' not found in DataFrames" ] } ], "source": [ "\n", "real_performance = model.calculate_performance(\n", " reference_df=data,\n", " analysis_df=data,\n", " target_col=target_col,\n", " timestamp_col=timestamp_col,\n", " chunk_period=chunk_period\n", ")\n", "\n", "metrics_df = model.get_calculated_metrics_dataframe(real_performance)\n", "display(metrics_df)\n", "\n", "metrics_df.to_csv(\"metrics_df.csv\", index=False)" ] } ], "metadata": { "kernelspec": { "display_name": "venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.13" } }, "nbformat": 4, "nbformat_minor": 5 }