diff --git a/orchestrator/activities/formatters.py b/orchestrator/activities/formatters.py index cb45d57..73f8f86 100644 --- a/orchestrator/activities/formatters.py +++ b/orchestrator/activities/formatters.py @@ -19,6 +19,7 @@ with workflow.unsafe.imports_passed_through(): minimal_retrain, predictions_batch, scouter, + drift, ) topic_separator = '\n ========== \n' @@ -34,7 +35,7 @@ class Formatters(SientiaMonitoring): scheduled reports. Key features: - - Pipeline schedule configuration formatting ("scouter", "predictions_batch", "minimal_retrain") + - Pipeline schedule configuration formatting ("scouter", "predictions_batch", "minimal_retrain", "drift") - OPC slot distribution across active ingestors - Notification filtering for comprehensive scheduled reports - Group-based report filtering with ignore list support @@ -132,6 +133,13 @@ class Formatters(SientiaMonitoring): 'updated_at', now().strftime(DATETIME_FORMAT_MS_WITH_TZ) ), } + elif pipeline['workflow_type'] == 'drift': + schedule_config[self.laborious_namespace][pipeline['schedule_name']] = { + **drift(pipeline), + 'updated_at': pipeline.get( + 'updated_at', now().strftime(DATETIME_FORMAT_MS_WITH_TZ) + ), + } self.info('Processed schedules', metadata=metadata) self.debug(json.dumps(schedule_config, indent=4, sort_keys=True), metadata=metadata) diff --git a/orchestrator/utils/orchestrator_functions.py b/orchestrator/utils/orchestrator_functions.py index 02371a1..54c388b 100644 --- a/orchestrator/utils/orchestrator_functions.py +++ b/orchestrator/utils/orchestrator_functions.py @@ -32,6 +32,25 @@ def common_config(config: dict[str, Any]): } +def drift(config: dict[str, Any]): + """ + Build drift configuration from pipeline config. + """ + model = config['model'] + return { + **common_config(config), + 'schema': 'sientia_data', + 'source_table_name': 'laborious_data', + 'target_table_name': 'drift_metrics', + 'interval': config.get('interval_minutes', 60), + 'drift_metrics': config.get('drift_metrics', [ + "kolmogorov_smirnov", + "jensen_shannon", + "wasserstein" + ]) + } + + def minimal_retrain(config: dict[str, Any]): """ Build minimal retrain configuration from pipeline config. @@ -47,8 +66,6 @@ def minimal_retrain(config: dict[str, Any]): """ return { **common_config(config), - 'workflow_type': 'minimal_retrain', - 'schedule_name': config['schedule_name'], 'query': config['query'], 'schema': 'sientia_data', 'table_name': 'log_retrain', diff --git a/test.ipynb b/test.ipynb index 1843727..a101c3c 100644 --- a/test.ipynb +++ b/test.ipynb @@ -617,13 +617,2049 @@ " print(f\"Chave: {key}, Valor: {value}\")" ] }, + { + "cell_type": "markdown", + "id": "e8025cbb", + "metadata": {}, + "source": [ + "## Model Monitoring" + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "87a9dc89", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "from sientia.ModelAnalysis import ModelAnalysis\n", + "\n", + "config = {\n", + " \"target\": \"Square\",\n", + " \"prediction\": \"prediction\",\n", + " \"timestamp\": \"timestamp\",\n", + " \"features\": [\"Counter\", \"Rollout\"]\n", + "}\n", + "\n", + "model = ModelAnalysis(config=config)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2a74ad2f", + "metadata": {}, + "outputs": [], + "source": [ + "from sientia_do.temporal.activities.postgres import Postgres\n", + "from unittest.mock import AsyncMock\n", + "from pandas import DataFrame\n", + "\n", + "postgres = Postgres(\n", + " host=\"localhost\",\n", + " port=5432,\n", + " dbname=\"sientia\",\n", + " user=\"sientia\",\n", + " password=\"sientia\",\n", + " min_connections=1,\n", + " max_connections=10,\n", + " logger=AsyncMock(),\n", + " notification_handler=AsyncMock(),\n", + " metrics_controller=AsyncMock()\n", + ")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "938cecbd", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:83: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.logger.custom_info(message, metadata)\n", + "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:59: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.metrics_controller.start()\n", + "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:83: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.logger.custom_info(message, metadata)\n", + "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:95: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", + " self.logger.custom_debug(message, metadata)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
variablevaluepredictiontimestamp
0Square42.2950-3.0184652025-11-12 12:44:50+00:00
1Rollout1.8610-3.0184652025-11-12 12:44:50+00:00
2Counter-45.0980-3.0184652025-11-12 12:44:50+00:00
3Rollout0.39702.3414232025-11-12 12:44:20+00:00
4Counter-40.02602.3414232025-11-12 12:44:20+00:00
5Square43.31902.3414232025-11-12 12:44:20+00:00
6Rollout-0.26454.6965912025-11-12 12:42:15+00:00
7Square42.18504.6965912025-11-12 12:42:15+00:00
8Counter-37.83854.6965912025-11-12 12:42:15+00:00
9Square2.8880-70.8451572025-11-11 23:00:48+00:00
10Rollout18.7945-70.8451572025-11-11 23:00:48+00:00
11Counter-12.6565-70.8451572025-11-11 23:00:48+00:00
12Rollout17.7415-71.8945902025-11-11 23:00:18+00:00
13Counter-11.1570-71.8945902025-11-11 23:00:18+00:00
14Square2.1610-71.8945902025-11-11 23:00:18+00:00
15Square-2.0490-67.0339532025-11-11 22:59:48+00:00
16Rollout16.5365-67.0339532025-11-11 22:59:48+00:00
17Counter-15.2205-67.0339532025-11-11 22:59:48+00:00
18Counter-11.7930-70.5144622025-11-11 22:59:18+00:00
19Square-4.3350-70.5144622025-11-11 22:59:18+00:00
20Rollout16.3065-70.5144622025-11-11 22:59:18+00:00
21Square0.6840-72.5232792025-11-11 22:58:48+00:00
22Counter-9.2570-72.5232792025-11-11 22:58:48+00:00
23Rollout14.9965-72.5232792025-11-11 22:58:48+00:00
24Square1.1700-72.8316842025-11-11 22:58:18+00:00
25Counter-7.0635-72.8316842025-11-11 22:58:18+00:00
26Rollout10.9875-72.8316842025-11-11 22:58:18+00:00
27Counter-2.0110-77.8659442025-11-11 22:57:48+00:00
28Rollout10.4545-77.8659442025-11-11 22:57:48+00:00
29Square-0.2070-77.8659442025-11-11 22:57:48+00:00
\n", + "
" + ], + "text/plain": [ + " variable value prediction timestamp\n", + "0 Square 42.2950 -3.018465 2025-11-12 12:44:50+00:00\n", + "1 Rollout 1.8610 -3.018465 2025-11-12 12:44:50+00:00\n", + "2 Counter -45.0980 -3.018465 2025-11-12 12:44:50+00:00\n", + "3 Rollout 0.3970 2.341423 2025-11-12 12:44:20+00:00\n", + "4 Counter -40.0260 2.341423 2025-11-12 12:44:20+00:00\n", + "5 Square 43.3190 2.341423 2025-11-12 12:44:20+00:00\n", + "6 Rollout -0.2645 4.696591 2025-11-12 12:42:15+00:00\n", + "7 Square 42.1850 4.696591 2025-11-12 12:42:15+00:00\n", + "8 Counter -37.8385 4.696591 2025-11-12 12:42:15+00:00\n", + "9 Square 2.8880 -70.845157 2025-11-11 23:00:48+00:00\n", + "10 Rollout 18.7945 -70.845157 2025-11-11 23:00:48+00:00\n", + "11 Counter -12.6565 -70.845157 2025-11-11 23:00:48+00:00\n", + "12 Rollout 17.7415 -71.894590 2025-11-11 23:00:18+00:00\n", + "13 Counter -11.1570 -71.894590 2025-11-11 23:00:18+00:00\n", + "14 Square 2.1610 -71.894590 2025-11-11 23:00:18+00:00\n", + "15 Square -2.0490 -67.033953 2025-11-11 22:59:48+00:00\n", + "16 Rollout 16.5365 -67.033953 2025-11-11 22:59:48+00:00\n", + "17 Counter -15.2205 -67.033953 2025-11-11 22:59:48+00:00\n", + "18 Counter -11.7930 -70.514462 2025-11-11 22:59:18+00:00\n", + "19 Square -4.3350 -70.514462 2025-11-11 22:59:18+00:00\n", + "20 Rollout 16.3065 -70.514462 2025-11-11 22:59:18+00:00\n", + "21 Square 0.6840 -72.523279 2025-11-11 22:58:48+00:00\n", + "22 Counter -9.2570 -72.523279 2025-11-11 22:58:48+00:00\n", + "23 Rollout 14.9965 -72.523279 2025-11-11 22:58:48+00:00\n", + "24 Square 1.1700 -72.831684 2025-11-11 22:58:18+00:00\n", + "25 Counter -7.0635 -72.831684 2025-11-11 22:58:18+00:00\n", + "26 Rollout 10.9875 -72.831684 2025-11-11 22:58:18+00:00\n", + "27 Counter -2.0110 -77.865944 2025-11-11 22:57:48+00:00\n", + "28 Rollout 10.4545 -77.865944 2025-11-11 22:57:48+00:00\n", + "29 Square -0.2070 -77.865944 2025-11-11 22:57:48+00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
variablevaluepredictiontimestamp
0Square42.2950-3.0184652025-11-12 12:44:50
1Rollout1.8610-3.0184652025-11-12 12:44:50
2Counter-45.0980-3.0184652025-11-12 12:44:50
3Rollout0.39702.3414232025-11-12 12:44:20
4Counter-40.02602.3414232025-11-12 12:44:20
5Square43.31902.3414232025-11-12 12:44:20
6Rollout-0.26454.6965912025-11-12 12:42:15
7Square42.18504.6965912025-11-12 12:42:15
8Counter-37.83854.6965912025-11-12 12:42:15
9Square2.8880-70.8451572025-11-11 23:00:48
10Rollout18.7945-70.8451572025-11-11 23:00:48
11Counter-12.6565-70.8451572025-11-11 23:00:48
12Rollout17.7415-71.8945902025-11-11 23:00:18
13Counter-11.1570-71.8945902025-11-11 23:00:18
14Square2.1610-71.8945902025-11-11 23:00:18
15Square-2.0490-67.0339532025-11-11 22:59:48
16Rollout16.5365-67.0339532025-11-11 22:59:48
17Counter-15.2205-67.0339532025-11-11 22:59:48
18Counter-11.7930-70.5144622025-11-11 22:59:18
19Square-4.3350-70.5144622025-11-11 22:59:18
20Rollout16.3065-70.5144622025-11-11 22:59:18
21Square0.6840-72.5232792025-11-11 22:58:48
22Counter-9.2570-72.5232792025-11-11 22:58:48
23Rollout14.9965-72.5232792025-11-11 22:58:48
24Square1.1700-72.8316842025-11-11 22:58:18
25Counter-7.0635-72.8316842025-11-11 22:58:18
26Rollout10.9875-72.8316842025-11-11 22:58:18
27Counter-2.0110-77.8659442025-11-11 22:57:48
28Rollout10.4545-77.8659442025-11-11 22:57:48
29Square-0.2070-77.8659442025-11-11 22:57:48
\n", + "
" + ], + "text/plain": [ + " variable value prediction timestamp\n", + "0 Square 42.2950 -3.018465 2025-11-12 12:44:50\n", + "1 Rollout 1.8610 -3.018465 2025-11-12 12:44:50\n", + "2 Counter -45.0980 -3.018465 2025-11-12 12:44:50\n", + "3 Rollout 0.3970 2.341423 2025-11-12 12:44:20\n", + "4 Counter -40.0260 2.341423 2025-11-12 12:44:20\n", + "5 Square 43.3190 2.341423 2025-11-12 12:44:20\n", + "6 Rollout -0.2645 4.696591 2025-11-12 12:42:15\n", + "7 Square 42.1850 4.696591 2025-11-12 12:42:15\n", + "8 Counter -37.8385 4.696591 2025-11-12 12:42:15\n", + "9 Square 2.8880 -70.845157 2025-11-11 23:00:48\n", + "10 Rollout 18.7945 -70.845157 2025-11-11 23:00:48\n", + "11 Counter -12.6565 -70.845157 2025-11-11 23:00:48\n", + "12 Rollout 17.7415 -71.894590 2025-11-11 23:00:18\n", + "13 Counter -11.1570 -71.894590 2025-11-11 23:00:18\n", + "14 Square 2.1610 -71.894590 2025-11-11 23:00:18\n", + "15 Square -2.0490 -67.033953 2025-11-11 22:59:48\n", + "16 Rollout 16.5365 -67.033953 2025-11-11 22:59:48\n", + "17 Counter -15.2205 -67.033953 2025-11-11 22:59:48\n", + "18 Counter -11.7930 -70.514462 2025-11-11 22:59:18\n", + "19 Square -4.3350 -70.514462 2025-11-11 22:59:18\n", + "20 Rollout 16.3065 -70.514462 2025-11-11 22:59:18\n", + "21 Square 0.6840 -72.523279 2025-11-11 22:58:48\n", + "22 Counter -9.2570 -72.523279 2025-11-11 22:58:48\n", + "23 Rollout 14.9965 -72.523279 2025-11-11 22:58:48\n", + "24 Square 1.1700 -72.831684 2025-11-11 22:58:18\n", + "25 Counter -7.0635 -72.831684 2025-11-11 22:58:18\n", + "26 Rollout 10.9875 -72.831684 2025-11-11 22:58:18\n", + "27 Counter -2.0110 -77.865944 2025-11-11 22:57:48\n", + "28 Rollout 10.4545 -77.865944 2025-11-11 22:57:48\n", + "29 Square -0.2070 -77.865944 2025-11-11 22:57:48" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from pandas import DataFrame, to_datetime\n", + "\n", + "query_data = \"\"\"\n", + "select ld.variable, ld.value, p.prediction, p.\"timestamp\"\n", + "from sientia_data.laborious_data ld\n", + " right join sientia_data.predictions p \n", + " on ld.timestamp = p.timestamp\n", + " and ld.model_id = p.model_id\n", + " where\n", + " ld.model_id = '1'\n", + " order by\n", + " ld.created_at desc limit 30;\n", + "\"\"\"\n", + "\n", + "input_data = {\n", + " \"query\": query_data,\n", + " \"metadata\": {}\n", + "}\n", + "\n", + "\n", + "raw_data = DataFrame(await postgres.load_custom_query(\n", + " input_data))\n", + "\n", + "raw_data['timestamp'] = to_datetime(raw_data['timestamp'])\n", + "raw_data['timestamp'] = raw_data['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')\n", + "\n", + "display(raw_data)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "297f648f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
timestamppredictionCounterRolloutSquare
02025-11-11 22:57:48-77.865944-2.011010.4545-0.207
12025-11-11 22:58:18-72.831684-7.063510.98751.170
22025-11-11 22:58:48-72.523279-9.257014.99650.684
32025-11-11 22:59:18-70.514462-11.793016.3065-4.335
42025-11-11 22:59:48-67.033953-15.220516.5365-2.049
52025-11-11 23:00:18-71.894590-11.157017.74152.161
62025-11-11 23:00:48-70.845157-12.656518.79452.888
72025-11-12 12:42:154.696591-37.8385-0.264542.185
82025-11-12 12:44:202.341423-40.02600.397043.319
92025-11-12 12:44:50-3.018465-45.09801.861042.295
\n", + "
" + ], + "text/plain": [ + " timestamp prediction Counter Rollout Square\n", + "0 2025-11-11 22:57:48 -77.865944 -2.0110 10.4545 -0.207\n", + "1 2025-11-11 22:58:18 -72.831684 -7.0635 10.9875 1.170\n", + "2 2025-11-11 22:58:48 -72.523279 -9.2570 14.9965 0.684\n", + "3 2025-11-11 22:59:18 -70.514462 -11.7930 16.3065 -4.335\n", + "4 2025-11-11 22:59:48 -67.033953 -15.2205 16.5365 -2.049\n", + "5 2025-11-11 23:00:18 -71.894590 -11.1570 17.7415 2.161\n", + "6 2025-11-11 23:00:48 -70.845157 -12.6565 18.7945 2.888\n", + "7 2025-11-12 12:42:15 4.696591 -37.8385 -0.2645 42.185\n", + "8 2025-11-12 12:44:20 2.341423 -40.0260 0.3970 43.319\n", + "9 2025-11-12 12:44:50 -3.018465 -45.0980 1.8610 42.295" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from pandas import merge, pivot\n", + "\n", + "raw_data.drop_duplicates(subset=[\"timestamp\", \"prediction\", \"variable\"], inplace=True, keep=\"first\")\n", + "data = raw_data.pivot(index=[\"timestamp\", \"prediction\"], columns=\"variable\", values=\"value\")\n", + "\n", + "data.columns.name = None\n", + "data.reset_index(inplace=True)\n", + "display(data)\n", + "\n", + "\n", + "drift_methods = ['kolmogorov_smirnov']#, 'jensen_shannon', 'wasserstein']\n", + "chunk_period = 's'\n", + "target_col = \"Square\"\n", + "timestamp_col = \"timestamp\"\n", + "\n", + "data.to_csv(\"data.csv\", index=False)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "9ab4774a", + "metadata": {}, + "outputs": [], + "source": [ + "from mlflow.tracking import MlflowClient\n", + "from os import environ\n", + "\n", + "environ['MLFLOW_TRACKING_USERNAME'] = 'aignosi'\n", + "environ['MLFLOW_TRACKING_PASSWORD'] = '1L0FP50j3ncp123'\n", + "environ['MLFLOW_TRACKING_URI'] = 'http://localhost:5080'\n", + "mlflow_client = MlflowClient()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "e8427625", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "run_id = '9e9fa748822c44ab92dd769d6a45c4f4'\n", + "output_dir = './tmp/model'\n", + "\n", + "artifacts = mlflow_client.list_artifacts(run_id)\n", + "paths = [artifact.path for artifact in artifacts]\n", + "\n", + "any(artifact.path == 'retrain_data.csv' for artifact in artifacts)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cf4ddf41", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
timestampCounterRolloutSquare
02025-11-10 18:58:3913.5970-52.450043.381
12025-11-10 18:58:4915.0660-50.024045.813
22025-11-10 18:58:5414.2780-49.481047.677
32025-11-10 18:59:0414.2080-51.801548.995
42025-11-10 18:59:0916.3530-52.685047.745
52025-11-10 18:59:1915.5415-53.562548.327
62025-11-10 18:59:2416.8580-53.696047.160
72025-11-10 18:59:3414.9745-54.751048.480
82025-11-10 18:59:3912.3020-53.433048.631
92025-11-10 18:59:4912.5640-52.980550.130
\n", + "
" + ], + "text/plain": [ + " timestamp Counter Rollout Square\n", + "0 2025-11-10 18:58:39 13.5970 -52.4500 43.381\n", + "1 2025-11-10 18:58:49 15.0660 -50.0240 45.813\n", + "2 2025-11-10 18:58:54 14.2780 -49.4810 47.677\n", + "3 2025-11-10 18:59:04 14.2080 -51.8015 48.995\n", + "4 2025-11-10 18:59:09 16.3530 -52.6850 47.745\n", + "5 2025-11-10 18:59:19 15.5415 -53.5625 48.327\n", + "6 2025-11-10 18:59:24 16.8580 -53.6960 47.160\n", + "7 2025-11-10 18:59:34 14.9745 -54.7510 48.480\n", + "8 2025-11-10 18:59:39 12.3020 -53.4330 48.631\n", + "9 2025-11-10 18:59:49 12.5640 -52.9805 50.130" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import mlflow\n", + "from io import StringIO\n", + "from pandas import read_csv\n", + "\n", + "if 'retrain_data.csv' in paths:\n", + " artifact_path = 'retrain_data.csv'\n", + "elif 'train_data.csv' in paths:\n", + " artifact_path = 'train_data.csv'\n", + "else:\n", + " raise RuntimeError(\n", + " f\"Could not find 'train_data.csv' or 'retrain_data.csv' for run_id '{run_id}'.\\n\"\n", + " )\n", + "\n", + "# Carregar diretamente na memória como string\n", + "artifact_content = mlflow.artifacts.load_text(\n", + " f\"runs:/{run_id}/{artifact_path}\"\n", + ")\n", + "\n", + "reference_df = read_csv(StringIO(artifact_content))\n", + "\n", + "reference_df.drop(columns=['timestamp.1'], inplace=True)\n", + "\n", + "reference_df['timestamp'] = to_datetime(reference_df['timestamp'])\n", + "reference_df['timestamp'] = reference_df['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')\n", + "\n", + "display(reference_df)\n", + "\n", + "reference_df.to_csv(\"reference_df.csv\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "d375752b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/numpy/core/_methods.py:206: RuntimeWarning: Degrees of freedom <= 0 for slice\n", + " ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,\n", + "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/numpy/core/_methods.py:198: RuntimeWarning: invalid value encountered in scalar divide\n", + " ret = ret.dtype.type(ret / rcount)\n", + "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/numpy/core/_methods.py:206: RuntimeWarning: Degrees of freedom <= 0 for slice\n", + " ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,\n", + "/home/grezewave/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/numpy/core/_methods.py:198: RuntimeWarning: invalid value encountered in scalar divide\n", + " ret = ret.dtype.type(ret / rcount)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
timestampfeaturemetricstatisticp_valuealertchunk_indexchunk_start_datechunk_end_date
02025-11-10 18:58:00+0000Counterkolmogorov_smirnov3.000000e-01NaNFalse02025-11-10 18:582025-11-10 18:58:59.999999999
12025-11-10 18:59:00+0000Counterkolmogorov_smirnov1.285714e-01NaNFalse12025-11-10 18:592025-11-10 18:59:59.999999999
22025-11-11 22:57:00+0000Counterkolmogorov_smirnov1.000000e+00NaNTrue02025-11-11 22:572025-11-11 22:57:59.999999999
32025-11-11 22:58:00+0000Counterkolmogorov_smirnov1.000000e+00NaNTrue12025-11-11 22:582025-11-11 22:58:59.999999999
42025-11-11 22:59:00+0000Counterkolmogorov_smirnov1.000000e+00NaNTrue22025-11-11 22:592025-11-11 22:59:59.999999999
52025-11-11 23:00:00+0000Counterkolmogorov_smirnov1.000000e+00NaNTrue32025-11-11 23:002025-11-11 23:00:59.999999999
62025-11-12 12:42:00+0000Counterkolmogorov_smirnov1.000000e+00NaNTrue42025-11-12 12:422025-11-12 12:42:59.999999999
72025-11-12 12:44:00+0000Counterkolmogorov_smirnov1.000000e+00NaNTrue52025-11-12 12:442025-11-12 12:44:59.999999999
82025-11-10 18:58:00+0000Rolloutkolmogorov_smirnov6.000000e-01NaNFalse02025-11-10 18:582025-11-10 18:58:59.999999999
92025-11-10 18:59:00+0000Rolloutkolmogorov_smirnov2.571429e-01NaNFalse12025-11-10 18:592025-11-10 18:59:59.999999999
102025-11-11 22:57:00+0000Rolloutkolmogorov_smirnov1.000000e+00NaNTrue02025-11-11 22:572025-11-11 22:57:59.999999999
112025-11-11 22:58:00+0000Rolloutkolmogorov_smirnov1.000000e+00NaNTrue12025-11-11 22:582025-11-11 22:58:59.999999999
122025-11-11 22:59:00+0000Rolloutkolmogorov_smirnov1.000000e+00NaNTrue22025-11-11 22:592025-11-11 22:59:59.999999999
132025-11-11 23:00:00+0000Rolloutkolmogorov_smirnov1.000000e+00NaNTrue32025-11-11 23:002025-11-11 23:00:59.999999999
142025-11-12 12:42:00+0000Rolloutkolmogorov_smirnov1.000000e+00NaNTrue42025-11-12 12:422025-11-12 12:42:59.999999999
152025-11-12 12:44:00+0000Rolloutkolmogorov_smirnov1.000000e+00NaNTrue52025-11-12 12:442025-11-12 12:44:59.999999999
162025-11-10 18:58:00+0000multivariatemultivariate2.015043e-163.748123e-17False02025-11-10 18:582025-11-10 18:58:59.999999999
172025-11-10 18:59:00+0000multivariatemultivariate2.493220e-163.998584e-17False12025-11-10 18:592025-11-10 18:59:59.999999999
182025-11-11 22:57:00+0000multivariatemultivariate1.432145e-14NaNTrue02025-11-11 22:572025-11-11 22:57:59.999999999
192025-11-11 22:58:00+0000multivariatemultivariate1.234840e-142.299811e-15True12025-11-11 22:582025-11-11 22:58:59.999999999
202025-11-11 22:59:00+0000multivariatemultivariate7.944109e-150.000000e+00True22025-11-11 22:592025-11-11 22:59:59.999999999
212025-11-11 23:00:00+0000multivariatemultivariate7.524768e-154.193410e-16True32025-11-11 23:002025-11-11 23:00:59.999999999
222025-11-12 12:42:00+0000multivariatemultivariate1.004859e-14NaNTrue42025-11-12 12:422025-11-12 12:42:59.999999999
232025-11-12 12:44:00+0000multivariatemultivariate1.004859e-140.000000e+00True52025-11-12 12:442025-11-12 12:44:59.999999999
\n", + "
" + ], + "text/plain": [ + " timestamp feature metric statistic \\\n", + "0 2025-11-10 18:58:00+0000 Counter kolmogorov_smirnov 3.000000e-01 \n", + "1 2025-11-10 18:59:00+0000 Counter kolmogorov_smirnov 1.285714e-01 \n", + "2 2025-11-11 22:57:00+0000 Counter kolmogorov_smirnov 1.000000e+00 \n", + "3 2025-11-11 22:58:00+0000 Counter kolmogorov_smirnov 1.000000e+00 \n", + "4 2025-11-11 22:59:00+0000 Counter kolmogorov_smirnov 1.000000e+00 \n", + "5 2025-11-11 23:00:00+0000 Counter kolmogorov_smirnov 1.000000e+00 \n", + "6 2025-11-12 12:42:00+0000 Counter kolmogorov_smirnov 1.000000e+00 \n", + "7 2025-11-12 12:44:00+0000 Counter kolmogorov_smirnov 1.000000e+00 \n", + "8 2025-11-10 18:58:00+0000 Rollout kolmogorov_smirnov 6.000000e-01 \n", + "9 2025-11-10 18:59:00+0000 Rollout kolmogorov_smirnov 2.571429e-01 \n", + "10 2025-11-11 22:57:00+0000 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "11 2025-11-11 22:58:00+0000 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "12 2025-11-11 22:59:00+0000 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "13 2025-11-11 23:00:00+0000 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "14 2025-11-12 12:42:00+0000 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "15 2025-11-12 12:44:00+0000 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "16 2025-11-10 18:58:00+0000 multivariate multivariate 2.015043e-16 \n", + "17 2025-11-10 18:59:00+0000 multivariate multivariate 2.493220e-16 \n", + "18 2025-11-11 22:57:00+0000 multivariate multivariate 1.432145e-14 \n", + "19 2025-11-11 22:58:00+0000 multivariate multivariate 1.234840e-14 \n", + "20 2025-11-11 22:59:00+0000 multivariate multivariate 7.944109e-15 \n", + "21 2025-11-11 23:00:00+0000 multivariate multivariate 7.524768e-15 \n", + "22 2025-11-12 12:42:00+0000 multivariate multivariate 1.004859e-14 \n", + "23 2025-11-12 12:44:00+0000 multivariate multivariate 1.004859e-14 \n", + "\n", + " p_value alert chunk_index chunk_start_date \\\n", + "0 NaN False 0 2025-11-10 18:58 \n", + "1 NaN False 1 2025-11-10 18:59 \n", + "2 NaN True 0 2025-11-11 22:57 \n", + "3 NaN True 1 2025-11-11 22:58 \n", + "4 NaN True 2 2025-11-11 22:59 \n", + "5 NaN True 3 2025-11-11 23:00 \n", + "6 NaN True 4 2025-11-12 12:42 \n", + "7 NaN True 5 2025-11-12 12:44 \n", + "8 NaN False 0 2025-11-10 18:58 \n", + "9 NaN False 1 2025-11-10 18:59 \n", + "10 NaN True 0 2025-11-11 22:57 \n", + "11 NaN True 1 2025-11-11 22:58 \n", + "12 NaN True 2 2025-11-11 22:59 \n", + "13 NaN True 3 2025-11-11 23:00 \n", + "14 NaN True 4 2025-11-12 12:42 \n", + "15 NaN True 5 2025-11-12 12:44 \n", + "16 3.748123e-17 False 0 2025-11-10 18:58 \n", + "17 3.998584e-17 False 1 2025-11-10 18:59 \n", + "18 NaN True 0 2025-11-11 22:57 \n", + "19 2.299811e-15 True 1 2025-11-11 22:58 \n", + "20 0.000000e+00 True 2 2025-11-11 22:59 \n", + "21 4.193410e-16 True 3 2025-11-11 23:00 \n", + "22 NaN True 4 2025-11-12 12:42 \n", + "23 0.000000e+00 True 5 2025-11-12 12:44 \n", + "\n", + " chunk_end_date \n", + "0 2025-11-10 18:58:59.999999999 \n", + "1 2025-11-10 18:59:59.999999999 \n", + "2 2025-11-11 22:57:59.999999999 \n", + "3 2025-11-11 22:58:59.999999999 \n", + "4 2025-11-11 22:59:59.999999999 \n", + "5 2025-11-11 23:00:59.999999999 \n", + "6 2025-11-12 12:42:59.999999999 \n", + "7 2025-11-12 12:44:59.999999999 \n", + "8 2025-11-10 18:58:59.999999999 \n", + "9 2025-11-10 18:59:59.999999999 \n", + "10 2025-11-11 22:57:59.999999999 \n", + "11 2025-11-11 22:58:59.999999999 \n", + "12 2025-11-11 22:59:59.999999999 \n", + "13 2025-11-11 23:00:59.999999999 \n", + "14 2025-11-12 12:42:59.999999999 \n", + "15 2025-11-12 12:44:59.999999999 \n", + "16 2025-11-10 18:58:59.999999999 \n", + "17 2025-11-10 18:59:59.999999999 \n", + "18 2025-11-11 22:57:59.999999999 \n", + "19 2025-11-11 22:58:59.999999999 \n", + "20 2025-11-11 22:59:59.999999999 \n", + "21 2025-11-11 23:00:59.999999999 \n", + "22 2025-11-12 12:42:59.999999999 \n", + "23 2025-11-12 12:44:59.999999999 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "\n", + "\n", + "from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ\n", + "\n", + "\n", + "univariate_drift = model.detect_univariate_drift(\n", + " reference_df=reference_df,\n", + " analysis_df=data,\n", + " features=config['features'],\n", + " timestamp_col=timestamp_col,\n", + " methods=drift_methods,\n", + " chunk_period=\"min\"\n", + ")\n", + "multivariate_drift = model.detect_multivariate_drift(\n", + " reference_df=reference_df,\n", + " analysis_df=data,\n", + " features=config['features'],\n", + " timestamp_col=timestamp_col,\n", + " chunk_period=\"min\"\n", + ")\n", + "\n", + "drift = model.get_drift_metrics_dataframe(\n", + " univariate_drift=univariate_drift,\n", + " multivariate_drift=multivariate_drift\n", + ")\n", + "drift.drop_duplicates(subset=[\"timestamp\", \"feature\", \"metric\"], inplace=True, keep=\"first\")\n", + "drift.reset_index(drop=True, inplace=True)\n", + "\n", + "drift['timestamp'] = to_datetime(drift['timestamp'])\n", + "print(type(drift['timestamp'].iloc[0]))\n", + "\n", + "# Add timezone UTC to timestamp\n", + "drift['timestamp'] = drift['timestamp'].dt.tz_localize('UTC')\n", + "drift['timestamp'] = drift['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)\n", + "\n", + "display(drift)\n", + "drift.to_csv(\"drift.csv\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "d30f6475", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
timestampfeaturemetricstatisticp_valuealertchunk_indexchunk_start_datechunk_end_date
102025-11-11 11:55:18Counterkolmogorov_smirnov1.000000e+00NoneFalse02025-11-11 11:55:182025-11-11 11:55:18.999999999
112025-11-11 11:55:48Counterkolmogorov_smirnov1.000000e+00NoneFalse12025-11-11 11:55:482025-11-11 11:55:48.999999999
122025-11-11 11:56:18Counterkolmogorov_smirnov1.000000e+00NoneFalse22025-11-11 11:56:182025-11-11 11:56:18.999999999
132025-11-11 11:56:48Counterkolmogorov_smirnov1.000000e+00NoneFalse32025-11-11 11:56:482025-11-11 11:56:48.999999999
142025-11-11 11:57:18Counterkolmogorov_smirnov1.000000e+00NoneFalse42025-11-11 11:57:182025-11-11 11:57:18.999999999
152025-11-11 11:57:48Counterkolmogorov_smirnov1.000000e+00NoneFalse52025-11-11 11:57:482025-11-11 11:57:48.999999999
162025-11-11 11:58:18Counterkolmogorov_smirnov1.000000e+00NoneFalse62025-11-11 11:58:182025-11-11 11:58:18.999999999
172025-11-11 11:58:48Counterkolmogorov_smirnov1.000000e+00NoneFalse72025-11-11 11:58:482025-11-11 11:58:48.999999999
182025-11-11 11:59:18Counterkolmogorov_smirnov1.000000e+00NoneFalse82025-11-11 11:59:182025-11-11 11:59:18.999999999
192025-11-11 11:59:48Counterkolmogorov_smirnov1.000000e+00NoneFalse92025-11-11 11:59:482025-11-11 11:59:48.999999999
302025-11-11 11:55:18Rolloutkolmogorov_smirnov1.000000e+00NoneFalse02025-11-11 11:55:182025-11-11 11:55:18.999999999
312025-11-11 11:55:48Rolloutkolmogorov_smirnov1.000000e+00NoneFalse12025-11-11 11:55:482025-11-11 11:55:48.999999999
322025-11-11 11:56:18Rolloutkolmogorov_smirnov1.000000e+00NoneFalse22025-11-11 11:56:182025-11-11 11:56:18.999999999
332025-11-11 11:56:48Rolloutkolmogorov_smirnov1.000000e+00NoneFalse32025-11-11 11:56:482025-11-11 11:56:48.999999999
342025-11-11 11:57:18Rolloutkolmogorov_smirnov1.000000e+00NoneFalse42025-11-11 11:57:182025-11-11 11:57:18.999999999
352025-11-11 11:57:48Rolloutkolmogorov_smirnov1.000000e+00NoneFalse52025-11-11 11:57:482025-11-11 11:57:48.999999999
362025-11-11 11:58:18Rolloutkolmogorov_smirnov1.000000e+00NoneFalse62025-11-11 11:58:182025-11-11 11:58:18.999999999
372025-11-11 11:58:48Rolloutkolmogorov_smirnov1.000000e+00NoneFalse72025-11-11 11:58:482025-11-11 11:58:48.999999999
382025-11-11 11:59:18Rolloutkolmogorov_smirnov1.000000e+00NoneFalse82025-11-11 11:59:182025-11-11 11:59:18.999999999
392025-11-11 11:59:48Rolloutkolmogorov_smirnov1.000000e+00NoneFalse92025-11-11 11:59:482025-11-11 11:59:48.999999999
502025-11-11 11:55:18multivariatemultivariate1.517720e-14NaNTrue02025-11-11 11:55:182025-11-11 11:55:18.999999999
512025-11-11 11:55:48multivariatemultivariate1.588822e-14NaNTrue12025-11-11 11:55:482025-11-11 11:55:48.999999999
522025-11-11 11:56:18multivariatemultivariate1.432145e-14NaNTrue22025-11-11 11:56:182025-11-11 11:56:18.999999999
532025-11-11 11:56:48multivariatemultivariate1.421085e-14NaNTrue32025-11-11 11:56:482025-11-11 11:56:48.999999999
542025-11-11 11:57:18multivariatemultivariate1.421085e-14NaNTrue42025-11-11 11:57:182025-11-11 11:57:18.999999999
552025-11-11 11:57:48multivariatemultivariate1.776357e-15NaNTrue52025-11-11 11:57:482025-11-11 11:57:48.999999999
562025-11-11 11:58:18multivariatemultivariate7.324107e-15NaNTrue62025-11-11 11:58:182025-11-11 11:58:18.999999999
572025-11-11 11:58:48multivariatemultivariate1.464821e-14NaNTrue72025-11-11 11:58:482025-11-11 11:58:48.999999999
582025-11-11 11:59:18multivariatemultivariate1.551137e-14NaNTrue82025-11-11 11:59:182025-11-11 11:59:18.999999999
592025-11-11 11:59:48multivariatemultivariate1.427317e-14NaNTrue92025-11-11 11:59:482025-11-11 11:59:48.999999999
\n", + "
" + ], + "text/plain": [ + " timestamp feature metric statistic \\\n", + "10 2025-11-11 11:55:18 Counter kolmogorov_smirnov 1.000000e+00 \n", + "11 2025-11-11 11:55:48 Counter kolmogorov_smirnov 1.000000e+00 \n", + "12 2025-11-11 11:56:18 Counter kolmogorov_smirnov 1.000000e+00 \n", + "13 2025-11-11 11:56:48 Counter kolmogorov_smirnov 1.000000e+00 \n", + "14 2025-11-11 11:57:18 Counter kolmogorov_smirnov 1.000000e+00 \n", + "15 2025-11-11 11:57:48 Counter kolmogorov_smirnov 1.000000e+00 \n", + "16 2025-11-11 11:58:18 Counter kolmogorov_smirnov 1.000000e+00 \n", + "17 2025-11-11 11:58:48 Counter kolmogorov_smirnov 1.000000e+00 \n", + "18 2025-11-11 11:59:18 Counter kolmogorov_smirnov 1.000000e+00 \n", + "19 2025-11-11 11:59:48 Counter kolmogorov_smirnov 1.000000e+00 \n", + "30 2025-11-11 11:55:18 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "31 2025-11-11 11:55:48 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "32 2025-11-11 11:56:18 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "33 2025-11-11 11:56:48 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "34 2025-11-11 11:57:18 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "35 2025-11-11 11:57:48 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "36 2025-11-11 11:58:18 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "37 2025-11-11 11:58:48 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "38 2025-11-11 11:59:18 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "39 2025-11-11 11:59:48 Rollout kolmogorov_smirnov 1.000000e+00 \n", + "50 2025-11-11 11:55:18 multivariate multivariate 1.517720e-14 \n", + "51 2025-11-11 11:55:48 multivariate multivariate 1.588822e-14 \n", + "52 2025-11-11 11:56:18 multivariate multivariate 1.432145e-14 \n", + "53 2025-11-11 11:56:48 multivariate multivariate 1.421085e-14 \n", + "54 2025-11-11 11:57:18 multivariate multivariate 1.421085e-14 \n", + "55 2025-11-11 11:57:48 multivariate multivariate 1.776357e-15 \n", + "56 2025-11-11 11:58:18 multivariate multivariate 7.324107e-15 \n", + "57 2025-11-11 11:58:48 multivariate multivariate 1.464821e-14 \n", + "58 2025-11-11 11:59:18 multivariate multivariate 1.551137e-14 \n", + "59 2025-11-11 11:59:48 multivariate multivariate 1.427317e-14 \n", + "\n", + " p_value alert chunk_index chunk_start_date \\\n", + "10 None False 0 2025-11-11 11:55:18 \n", + "11 None False 1 2025-11-11 11:55:48 \n", + "12 None False 2 2025-11-11 11:56:18 \n", + "13 None False 3 2025-11-11 11:56:48 \n", + "14 None False 4 2025-11-11 11:57:18 \n", + "15 None False 5 2025-11-11 11:57:48 \n", + "16 None False 6 2025-11-11 11:58:18 \n", + "17 None False 7 2025-11-11 11:58:48 \n", + "18 None False 8 2025-11-11 11:59:18 \n", + "19 None False 9 2025-11-11 11:59:48 \n", + "30 None False 0 2025-11-11 11:55:18 \n", + "31 None False 1 2025-11-11 11:55:48 \n", + "32 None False 2 2025-11-11 11:56:18 \n", + "33 None False 3 2025-11-11 11:56:48 \n", + "34 None False 4 2025-11-11 11:57:18 \n", + "35 None False 5 2025-11-11 11:57:48 \n", + "36 None False 6 2025-11-11 11:58:18 \n", + "37 None False 7 2025-11-11 11:58:48 \n", + "38 None False 8 2025-11-11 11:59:18 \n", + "39 None False 9 2025-11-11 11:59:48 \n", + "50 NaN True 0 2025-11-11 11:55:18 \n", + "51 NaN True 1 2025-11-11 11:55:48 \n", + "52 NaN True 2 2025-11-11 11:56:18 \n", + "53 NaN True 3 2025-11-11 11:56:48 \n", + "54 NaN True 4 2025-11-11 11:57:18 \n", + "55 NaN True 5 2025-11-11 11:57:48 \n", + "56 NaN True 6 2025-11-11 11:58:18 \n", + "57 NaN True 7 2025-11-11 11:58:48 \n", + "58 NaN True 8 2025-11-11 11:59:18 \n", + "59 NaN True 9 2025-11-11 11:59:48 \n", + "\n", + " chunk_end_date \n", + "10 2025-11-11 11:55:18.999999999 \n", + "11 2025-11-11 11:55:48.999999999 \n", + "12 2025-11-11 11:56:18.999999999 \n", + "13 2025-11-11 11:56:48.999999999 \n", + "14 2025-11-11 11:57:18.999999999 \n", + "15 2025-11-11 11:57:48.999999999 \n", + "16 2025-11-11 11:58:18.999999999 \n", + "17 2025-11-11 11:58:48.999999999 \n", + "18 2025-11-11 11:59:18.999999999 \n", + "19 2025-11-11 11:59:48.999999999 \n", + "30 2025-11-11 11:55:18.999999999 \n", + "31 2025-11-11 11:55:48.999999999 \n", + "32 2025-11-11 11:56:18.999999999 \n", + "33 2025-11-11 11:56:48.999999999 \n", + "34 2025-11-11 11:57:18.999999999 \n", + "35 2025-11-11 11:57:48.999999999 \n", + "36 2025-11-11 11:58:18.999999999 \n", + "37 2025-11-11 11:58:48.999999999 \n", + "38 2025-11-11 11:59:18.999999999 \n", + "39 2025-11-11 11:59:48.999999999 \n", + "50 2025-11-11 11:55:18.999999999 \n", + "51 2025-11-11 11:55:48.999999999 \n", + "52 2025-11-11 11:56:18.999999999 \n", + "53 2025-11-11 11:56:48.999999999 \n", + "54 2025-11-11 11:57:18.999999999 \n", + "55 2025-11-11 11:57:48.999999999 \n", + "56 2025-11-11 11:58:18.999999999 \n", + "57 2025-11-11 11:58:48.999999999 \n", + "58 2025-11-11 11:59:18.999999999 \n", + "59 2025-11-11 11:59:48.999999999 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# drop rows of drift with timestamp in reference data but not in analysis data\n", + "drift_filtered = drift[drift['timestamp'].isin(data['timestamp'])]\n", + "display(drift_filtered)\n", + "\n", + "drift_filtered.to_csv(\"drift_filtered.csv\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70671022", + "metadata": {}, + "outputs": [ + { + "ename": "ValueError", + "evalue": "Target column 'Square' not found in DataFrames", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[36]\u001b[39m\u001b[32m, line 6\u001b[39m\n\u001b[32m 3\u001b[39m \u001b[38;5;66;03m# rename column square to prediction\u001b[39;00m\n\u001b[32m 4\u001b[39m performance_reference.rename(columns={\u001b[33m'\u001b[39m\u001b[33mSquare\u001b[39m\u001b[33m'\u001b[39m: \u001b[33m'\u001b[39m\u001b[33mprediction\u001b[39m\u001b[33m'\u001b[39m}, inplace=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m real_performance = \u001b[43mmodel\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcalculate_performance\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 7\u001b[39m \u001b[43m \u001b[49m\u001b[43mreference_df\u001b[49m\u001b[43m=\u001b[49m\u001b[43mperformance_reference\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 8\u001b[39m \u001b[43m \u001b[49m\u001b[43manalysis_df\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 9\u001b[39m \u001b[43m \u001b[49m\u001b[43mtarget_col\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtarget_col\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 10\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimestamp_col\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimestamp_col\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 11\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunk_period\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunk_period\u001b[49m\n\u001b[32m 12\u001b[39m \u001b[43m)\u001b[49m\n\u001b[32m 14\u001b[39m metrics_df = model.get_calculated_metrics_dataframe(real_performance)\n\u001b[32m 15\u001b[39m display(metrics_df)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-orchestrator_temporal/venv/lib/python3.11/site-packages/sientia/ModelAnalysis.py:397\u001b[39m, in \u001b[36mModelAnalysis.calculate_performance\u001b[39m\u001b[34m(self, reference_df, analysis_df, target_col, timestamp_col, metrics, chunk_period)\u001b[39m\n\u001b[32m 395\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(DATAFRAMES_EMPTY_ERROR)\n\u001b[32m 396\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m target_col \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m reference_df.columns \u001b[38;5;129;01mor\u001b[39;00m target_col \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m analysis_df.columns:\n\u001b[32m--> \u001b[39m\u001b[32m397\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mTarget column \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtarget_col\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m not found in DataFrames\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 398\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m timestamp_col \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m reference_df.columns \u001b[38;5;129;01mor\u001b[39;00m timestamp_col \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m analysis_df.columns:\n\u001b[32m 399\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mTimestamp column \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtimestamp_col\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m not found in DataFrames\u001b[39m\u001b[33m\"\u001b[39m)\n", + "\u001b[31mValueError\u001b[39m: Target column 'Square' not found in DataFrames" + ] + } + ], + "source": [ + "\n", + "real_performance = model.calculate_performance(\n", + " reference_df=data,\n", + " analysis_df=data,\n", + " target_col=target_col,\n", + " timestamp_col=timestamp_col,\n", + " chunk_period=chunk_period\n", + ")\n", + "\n", + "metrics_df = model.get_calculated_metrics_dataframe(real_performance)\n", + "display(metrics_df)\n", + "\n", + "metrics_df.to_csv(\"metrics_df.csv\", index=False)" + ] } ], "metadata": { diff --git a/tests/orchestrator/activities/test_formatters.py b/tests/orchestrator/activities/test_formatters.py index 223227e..12f6901 100644 --- a/tests/orchestrator/activities/test_formatters.py +++ b/tests/orchestrator/activities/test_formatters.py @@ -44,8 +44,12 @@ metadata = { 'orchestrator.activities.formatters.minimal_retrain', return_value={'test_minimal_retrain': 'test_minimal_retrain'}, ) +@patch( + 'orchestrator.activities.formatters.drift', + return_value={'test_drift': 'test_drift'}, +) async def test_process_schedules( - mock_minimal_retrain, mock_predictions_batch, mock_scouter, formatters + mock_drift, mock_minimal_retrain, mock_predictions_batch, mock_scouter, formatters ): input_data = { 'pipelines': [ @@ -70,6 +74,13 @@ async def test_process_schedules( 'model_id': 'test_model_id', 'updated_at': '2021-01-03', }, + { + 'schedule_name': 'test_schedule_name4', + 'workflow_type': 'drift', + 'model_name': 'test_model_name', + 'model_id': 'test_model_id', + 'updated_at': '2021-01-04', + }, ] } @@ -88,11 +99,17 @@ async def test_process_schedules( 'test_minimal_retrain': 'test_minimal_retrain', 'updated_at': '2021-01-03', }, + 'test_schedule_name4': { + 'test_drift': 'test_drift', + 'updated_at': '2021-01-04', + }, }, } mock_scouter.assert_called_once_with(input_data['pipelines'][0]) mock_predictions_batch.assert_called_once_with(input_data['pipelines'][1]) + mock_minimal_retrain.assert_called_once_with(input_data['pipelines'][2]) + mock_drift.assert_called_once_with(input_data['pipelines'][3]) @mark.asyncio diff --git a/tests/orchestrator/utils/test_orchestrator_functions.py b/tests/orchestrator/utils/test_orchestrator_functions.py index f74a936..7c8f5da 100644 --- a/tests/orchestrator/utils/test_orchestrator_functions.py +++ b/tests/orchestrator/utils/test_orchestrator_functions.py @@ -3,6 +3,7 @@ from unittest.mock import call, patch from orchestrator.utils.orchestrator_functions import ( build_tag_config, common_config, + drift, gather_read_tags, minimal_retrain, overlap_filter_config, @@ -35,6 +36,36 @@ def test_common_config(): assert result == expected +def test_drift(): + config = { + 'workflow_type': 'drift', + 'schedule_name': 'test_schedule', + 'model_id': 'test_model_id', + 'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}}, + 'interval_minutes': 120, + 'drift_metrics': ['kolmogorov_smirnov', 'jensen_shannon'], + } + result = drift(config) + expected = { + 'workflow_type': 'drift', + 'schedule_name': 'test_schedule', + 'frequency': '1m', + 'offset': '0m', + 'max_retry_policy': 1, + 'model_id': 'test_model_id', + 'model_name': 'test_model_name', + 'model_config': {'test_config': 'test_config'}, + 'schema': 'sientia_data', + 'source_table_name': 'laborious_data', + 'target_table_name': 'drift_metrics', + 'interval': 120, + 'drift_metrics': ['kolmogorov_smirnov', 'jensen_shannon'], + 'execution_timeout_seconds': 300, + 'task_timeout_seconds': 300, + } + assert result == expected + + def test_minimal_retrain(): config = { 'workflow_type': 'minimal_retrain',