From 70f1abe68118a089abf2db626aee48c716e15546 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 1 Aug 2025 10:00:47 -0300 Subject: [PATCH 01/12] SIENTIAPDE-1174 Update dependencies, modify replica count, and implement metrics tracking - Updated sientia-dataops-library version from 1.3.5 to 1.3.7 in requirements.txt. - Changed replicaCount in values.yaml from 5 to 3 and incremented image tag from 0.2.7 to 0.3.1. - Added Prometheus metrics tracking in gates.py and worker.py, including a new write_metrics method. - Configured Prometheus service and ServiceMonitor in values.yaml for metrics collection. --- laborious/activities/gates.py | 32 +++ laborious/metrics.py | 27 +++ .../utils/repository/model_repository.py | 2 +- laborious/worker/worker.py | 21 +- .../format_and_export_prediction.py | 10 + requirements.txt | 3 +- tests.ipynb | 226 ++++++++++++++++++ values.yaml | 39 ++- 8 files changed, 349 insertions(+), 11 deletions(-) create mode 100644 laborious/metrics.py create mode 100644 tests.ipynb diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index 2a54052..739fa4b 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -15,6 +15,7 @@ with workflow.unsafe.imports_passed_through(): ) from pandas import DataFrame from datetime import datetime + from laborious import metrics input_filter_functions = { 'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values, @@ -309,3 +310,34 @@ class Gates(BaseActivity): if data.empty: return datetime.now().strftime('%Y-%m-%d %H:%M:%S') return max(data['timestamp'].values.tolist()) + + @activity.defn(name="write_metrics") + async def write_metrics(self, input_data: dict[str, Any]): + """ + Write metrics to the database. + input_data: + metadata: dict[str, Any] + prediction: dict[str, Any] + """ + metadata = input_data['metadata'] + prediction = DataFrame(input_data['prediction']) + prediction_confidence = prediction['prediction_confidence'].values[0] + response_time = prediction['response_time'].values[0] + + metrics.PREDICTIONS_WRITTEN_COUNT.labels( + pod_id=self.pod_id, + model_name=metadata['model_name'], + pipeline_name=metadata['workflow_name'] + ).inc() + + metrics.PREDICTION_CONFIDENCE_MONITOR.labels( + pod_id=self.pod_id, + model_name=metadata['model_name'], + pipeline_name=metadata['workflow_name'] + ).set(prediction_confidence) + + metrics.PREDICTION_RESPONSE_TIME_MONITOR.labels( + pod_id=self.pod_id, + model_name=metadata['model_name'], + pipeline_name=metadata['workflow_name'] + ).set(response_time) diff --git a/laborious/metrics.py b/laborious/metrics.py new file mode 100644 index 0000000..eaad6a7 --- /dev/null +++ b/laborious/metrics.py @@ -0,0 +1,27 @@ +from prometheus_client import Gauge, Counter + +APP_UP = Gauge( + "app_up", + "Indicates if the application is running (1) or shutting down (0)", + ["pod_id"], +) + +CORE_LABELS = ["pod_id", "model_name", "pipeline_name"] + +PREDICTIONS_WRITTEN_COUNT = Counter( + "laborious_predictions_written_count", + "Number of predictions written to the database table predictions", + CORE_LABELS, +) + +PREDICTION_CONFIDENCE_MONITOR = Gauge( + "laborious_prediction_confidence_monitor", + "Current confidence of each prediction", + CORE_LABELS, +) + +PREDICTION_RESPONSE_TIME_MONITOR = Gauge( + "laborious_prediction_response_time_monitor", + "Current response time of each prediction", + CORE_LABELS, +) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 2f10a9d..edc99e8 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -68,7 +68,7 @@ class MLFlowRepository(): try: start_time = datetime.now() data = self.model_serving.get_cached_predict( - model_name, data, model_retention)[-1:] + model_name, data, model_retention)[0:1] end_time = datetime.now() data = pd.DataFrame(data, columns=['prediction']) diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index 375c84f..4e72058 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -20,13 +20,20 @@ with workflow.unsafe.imports_passed_through(): ) from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.temporal.utils.logger import get_logger + from laborious import metrics + from prometheus_client import start_http_server + +POD_ID = os.getenv('POD_ID') async def main(): host = os.getenv('TEMPORAL_HOST', 'localhost:7233') logger = get_logger(__name__) - logger.info('Starting Worker...') + logger.info(f'Starting Worker with POD_ID: {POD_ID}') + + logger.info("Starting prometheus client...") + start_prometheus_server() logger.info('Starting Notification Handler...') @@ -125,5 +132,17 @@ async def main(): # Exit with a non-zero status code to indicate failure to Kubernetes sys.exit(1) + +def start_prometheus_server(): + try: + port = int(os.getenv("HTTP_METRICS_PORT", 9090)) + start_http_server(port) + print(f"Prometheus server started on port {port}.") + metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP + except Exception as e: + print(f"Failed to start Prometheus server: {e}") + os._exit(1) + + if __name__ == '__main__': asyncio.run(main()) diff --git a/laborious/workflows/sub_workflows/format_and_export_prediction.py b/laborious/workflows/sub_workflows/format_and_export_prediction.py index b021bfd..8ed79fb 100644 --- a/laborious/workflows/sub_workflows/format_and_export_prediction.py +++ b/laborious/workflows/sub_workflows/format_and_export_prediction.py @@ -95,3 +95,13 @@ class FormatAndExportPrediction(): retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=60) ) + + await workflow.execute_activity_method( + Activities.write_metrics, + { + **metadata, + 'prediction': prediction + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60) + ) diff --git a/requirements.txt b/requirements.txt index 6653e74..38a0818 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,6 @@ psycopg2-binary sqlalchemy asyncua redis -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.5 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.7 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.5 +prometheus-client diff --git a/tests.ipynb b/tests.ipynb new file mode 100644 index 0000000..d83bd27 --- /dev/null +++ b/tests.ipynb @@ -0,0 +1,226 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 6, + "id": "b10e5c25", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ 0.5479121 -0.12224312 0.71719584 0.39473606 -0.8116453 0.9512447\n", + " 0.5222794 0.57212861 -0.74377273 -0.09922812 -0.25840395 0.85352998\n", + " 0.28773024 0.64552323 -0.1131716 -0.54552256 0.10916957 -0.87236549\n", + " 0.65526234 0.2633288 0.51617548 -0.29094806 0.94139605 0.78624224\n", + " 0.55676699 -0.61072258 -0.06655799 -0.91239247 -0.69142102 0.36609791\n", + " 0.48952431 0.93501946 -0.34834928 -0.25908059 -0.06088838 -0.62105728\n", + " -0.74015699 -0.04859015 -0.5461813 0.33962799 -0.12569616 0.66535639\n", + " 0.4005302 -0.37526672 0.6645196 0.60952871 -0.22504324 -0.42334379\n", + " 0.36499101 -0.72049503 -0.6001836 -0.98527546 0.57384876 0.32970171\n", + " 0.41033076 0.56145806 -0.08216845 0.13748239 -0.720406 -0.77093985\n", + " 0.33680592 -0.05780759 0.13047221 0.52999771 0.26943664 0.1071588\n", + " 0.11841432 -0.3920998 -0.93836433 -0.12656522 -0.57083065 -0.18294271\n", + " 0.70680615 -0.53212103 -0.88339452 -0.43723222 -0.41281248 0.32383303\n", + " 0.1140643 0.56779642 0.32862708 -0.18722628 0.62804077 -0.66605416\n", + " -0.95457585 -0.81990428 0.4447187 -0.07624554 -0.67745644 0.00208955\n", + " -0.69537579 0.39264075 -0.10768745 -0.23795755 -0.39697582 0.26056519\n", + " -0.27637478 -0.82470016 -0.7639882 0.92379533 0.81716138 0.39941427\n", + " -0.46826008 0.93835275 0.55750181 0.43378038 -0.101277 -0.45551688\n", + " -0.80721808 0.80520479 -0.08844742 -0.59527327 -0.38808675 0.15843914\n", + " -0.64645443 0.71322857 0.51703906 0.43892591 -0.13581392 0.25461768\n", + " 0.16819594 0.2996932 -0.83111136 -0.1683852 -0.91677165 -0.01201836\n", + " -0.34027758 -0.71095162 -0.79319406 0.17528914 -0.65881406 0.85024024\n", + " 0.16212228 -0.30626039 0.18183098 -0.95439226 0.91711843 -0.03539313\n", + " 0.56547045 -0.83454 -0.02668334 -0.01858601 0.87565291 0.1434561\n", + " -0.0530212 -0.46604867 -0.33686201 0.0413448 -0.12217708 -0.95677584\n", + " 0.65258385 0.79232154 -0.71950182 0.10807229 -0.78284852 0.34448019\n", + " -0.43753243 0.31884527 0.45398923 0.53729498 -0.78451811 0.83202369\n", + " -0.53957202 -0.92517489 0.10970494 -0.25815543 0.65957949 0.61650294\n", + " -0.36572221 0.90579879 -0.41816432 0.03011426 -0.48806982 0.87208714\n", + " -0.67078436 -0.91017876 -0.12980588 0.98475113 0.78335453 0.49721604\n", + " 0.78158498 0.78689328 0.03771672 -0.3681419 0.54402486 0.32332253\n", + " -0.25268454 -0.81106666 0.49357922 -0.47507897 0.8736263 -0.51805885\n", + " -0.75448414 0.66222534 -0.69343137 -0.64146338 0.19876558 0.74912408\n", + " -0.60713067 -0.37935265 0.55480968 0.94365285 0.00148237 -0.71220499\n", + " -0.97212742 -0.54068794 -0.73635556 0.35531735 -0.75633499 0.01265986\n", + " 0.38852487 0.16223322 -0.6004487 0.60824905 0.43081426 0.47796801\n", + " -0.7378845 -0.75249239 0.8551251 -0.20484361 -0.39810262 -0.02283191\n", + " 0.32572843 0.91124651 -0.42710755 0.84961686 -0.95028102 0.11039608\n", + " 0.26795022 -0.78820519 -0.71932081 -0.16177136 0.93246382 0.19208511\n", + " 0.86604644 0.60872183 -0.0652368 0.5695269 -0.96432643 -0.78171201\n", + " 0.65885723 0.59363418 -0.53471852 0.06153918 0.21203164 0.73547791\n", + " 0.20621431 -0.17485686 -0.25163191 -0.14823583 0.30386205 0.73498126\n", + " -0.09220624 -0.50432087 -0.52667527 0.49202856 0.63313753 -0.78944384\n", + " -0.86688229 0.18886733 -0.70765351 0.64932838 -0.37933065 -0.71225613\n", + " 0.84194094 -0.66893655 -0.43055984 -0.69277321 -0.76901987 -0.95770397\n", + " -0.88920918 -0.65071706 -0.89323613 0.18228763 0.36142905 -0.21273909\n", + " -0.36401781 0.00905247 0.75000988 0.70226325 -0.91304988 -0.63700318\n", + " -0.52651026 -0.50122485 0.1424653 -0.16747515 -0.90149176 -0.25277172\n", + " 0.0475059 -0.79665619 0.66691711 -0.89607627 0.84968374 -0.80177372\n", + " 0.6871499 0.80530629 0.95914136 0.60405176 0.55895508]\n", + "[ 0.28496655 0.55799271 -0.73089558 0.07213607 0.02844574 0.71514429\n", + " -0.07440127 -0.22982101 0.27912654 -0.46707336 -0.72046318 -0.04424545\n", + " -0.16622126 -0.53486012 -0.26497638 -0.2672151 -0.34500887 -0.24107184\n", + " 0.37148669 -0.40624705 0.89771585 0.83269604 -0.03817914 -0.34327759\n", + " 0.07086958 0.69712098 0.30517468 0.60878366 0.06544455 0.26583526\n", + " -0.42368877 0.46978632 -0.59519081 0.38959626 0.72143814 -0.73579433\n", + " 0.22875948 -0.8098085 0.45143126 -0.83101356 0.87187965 -0.72518414\n", + " 0.91776049 0.60176835 0.18736401 0.56524821 0.59022968 0.89205413\n", + " -0.49323329 0.18015179 -0.8099016 0.2323314 -0.65741739 0.12990122\n", + " 0.14486103 -0.06802969 0.04526355 0.52784678 0.59848943 -0.01569357\n", + " 0.19918688 0.86247247 -0.76053282 -0.76579287 -0.82458198 0.31572657\n", + " -0.1627834 0.54864283 0.34246283 -0.33272448 0.79673309 0.52506429\n", + " -0.45893012 -0.27161596 -0.37112004 -0.6847767 -0.70443325 0.87225493\n", + " -0.12419193 -0.23336035 0.45937142 0.10598613 0.87227997 0.56060299\n", + " -0.04126087 -0.24728105 0.97326309 0.43552047 0.90238932 -0.76304285\n", + " 0.70106736 0.27414777 -0.75615664 0.176516 0.37219273 -0.97539463\n", + " -0.09136408 0.65079902 -0.40928195 -0.08290384 -0.11537175 -0.39614522\n", + " 0.83688379 0.56258807 -0.77882318 0.99406932 0.75840005 -0.43218312\n", + " 0.67379316 -0.78716094 0.99820946 0.33136947 0.30025003 -0.81911855\n", + " 0.7940668 -0.94200099 -0.51834388 -0.71395625 0.55353588 -0.60359155\n", + " 0.82127645 0.31253808 -0.92767458 -0.98914033 -0.89668417 0.21185036\n", + " 0.60296362 -0.52289436 0.69881769 -0.88553612 0.60192771 0.85559086\n", + " 0.5442168 0.39624157 0.67596044 -0.9196974 -0.59643578 -0.75015264\n", + " 0.00906198 0.49037626 0.26002369 0.7022622 -0.68957402 0.46924218\n", + " -0.61391702 -0.4584825 0.41980939 0.96040957 0.22308721 -0.89099937\n", + " 0.23261794 -0.9152989 0.76829142 0.41915657 -0.65374431 -0.81655799\n", + " -0.63293354 0.96005436 -0.08287872 0.5681619 0.27281668 0.1448263\n", + " -0.70973949 0.89204891 -0.39731473 0.15603443 0.39955189 0.29846631\n", + " 0.88118882 -0.70312202 0.01670548 -0.19193122 -0.05166254 -0.76156495\n", + " -0.73181078 -0.44384891 -0.39059079 -0.14419357 0.22197509 0.26925823\n", + " -0.17637821 -0.18243378 -0.56474295 0.1766125 -0.36591818 -0.92788033\n", + " -0.16319991 -0.05173465 -0.54881426 0.14491587 0.1315438 0.40400436\n", + " 0.29589696 0.30486611 -0.3675717 0.57486444 0.09828877 -0.13716361\n", + " 0.25202496 -0.27868533 0.02547849 0.47341138 0.77280577 0.84211439\n", + " 0.00726585 0.04055023 0.59974082 -0.37109862 0.67476472 -0.01171671\n", + " -0.76828655 -0.85588171 0.68398642 -0.88886417 -0.43877713 -0.33173992\n", + " -0.65401111 -0.37221326 0.48538513 -0.97063431 0.65434685 0.71309605\n", + " -0.25547685 -0.6927742 0.20168082 -0.76065489 -0.27016128 0.91685836\n", + " 0.99092895 0.54420978 -0.37807698 0.3753301 0.41081273 -0.22431661\n", + " 0.28177727 -0.97854471 -0.58188468 0.05017661 -0.67249739 -0.66818626\n", + " 0.67260858 0.97826601 0.11193886 0.67813946 0.98064333 -0.71680822\n", + " -0.10350877 -0.21485457 -0.83990143 0.51066035 -0.13244195 -0.06134613\n", + " -0.69865405 -0.6381467 0.81420724 -0.91070182 -0.53429543 -0.41588134\n", + " -0.01960492 0.17289035 -0.01342005 -0.83176933 -0.51266509 0.68717677\n", + " 0.2751774 0.2982981 0.34040651 0.52580604 -0.88378304 -0.26678323\n", + " 0.07905487 -0.32308703 0.68895775 -0.03485498 0.53725518 0.70403103\n", + " 0.00958297 0.81910449 0.17424788 0.7005486 -0.31881841 -0.00236608\n", + " 0.06282208 -0.79004057 -0.20289499 0.83467535 0.26166448 -0.64498684\n", + " -0.32228873 -0.61679398 -0.95035374 0.85492092 -0.10358534 -0.38492986\n", + " 0.19695438 -0.98537109 -0.44395579 0.40606693 0.26753955]\n", + " Counter Rollout CounterPlusRollout Timestamp\n", + "0 0.000000 0.000000 0.000000 2025-01-01 00:00:00\n", + "1 0.547912 0.284967 0.832879 2025-01-01 00:00:01\n", + "2 0.425669 0.842959 1.268628 2025-01-01 00:00:02\n", + "3 1.142865 0.112064 1.254928 2025-01-01 00:00:03\n", + "4 1.537601 0.184200 1.721801 2025-01-01 00:00:04\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "# Set random seed for reproducibility\n", + "rng = np.random.default_rng(42)\n", + "\n", + "# Generate random walks starting at 0\n", + "counter = np.zeros(300)\n", + "rollout = np.zeros(300)\n", + "\n", + "# Generate random steps between -1 and 1\n", + "counter_steps = rng.uniform(-1, 1, 299)\n", + "rollout_steps = rng.uniform(-1, 1, 299)\n", + "\n", + "print(counter_steps)\n", + "print(rollout_steps)\n", + "\n", + "# Calculate cumulative sum and scale to -100 to 100 range\n", + "for i in range(1, 300):\n", + " counter[i] = counter[i-1] + counter_steps[i-1]\n", + " rollout[i] = rollout[i-1] + rollout_steps[i-1]\n", + "\n", + "# if any of the values is greater than 100, set it to 100\n", + "counter = np.where(counter > 100, 100, counter)\n", + "rollout = np.where(rollout > 100, 100, rollout)\n", + "\n", + "# Create DataFrame\n", + "df = pd.DataFrame({\n", + " 'Counter': counter,\n", + " 'Rollout': rollout,\n", + " 'CounterPlusRollout': counter + rollout\n", + "})\n", + "\n", + "# add a timestamp column\n", + "df['Timestamp'] = pd.date_range(start='2025-01-01', periods=300, freq='1s')\n", + "\n", + "# Save to CSV\n", + "df.to_csv('random_walks.csv', index=False)\n", + "\n", + "# Display first few rows\n", + "print(df.head())\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "e7c8eeb1", + "metadata": {}, + "outputs": [ + { + "ename": "RestException", + "evalue": "RESOURCE_DOES_NOT_EXIST: Run with id=1 not found", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mRestException\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 9\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mlaborious\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mrepository\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmodel_repository\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m MLFlowRepository\n\u001b[32m 3\u001b[39m mlflow_repository = MLFlowRepository(\n\u001b[32m 4\u001b[39m host=\u001b[33m\"\u001b[39m\u001b[33mhttp://localhost:5080/\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 5\u001b[39m username=\u001b[33m\"\u001b[39m\u001b[33maignosi\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 6\u001b[39m password=\u001b[33m\"\u001b[39m\u001b[33maignosi\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 7\u001b[39m )\n\u001b[32m----> \u001b[39m\u001b[32m9\u001b[39m \u001b[43mmlflow_repository\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_experiment_by_run_id\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m1\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/laborious/utils/repository/model_repository.py:93\u001b[39m, in \u001b[36mMLFlowRepository.get_experiment_by_run_id\u001b[39m\u001b[34m(self, run_id)\u001b[39m\n\u001b[32m 91\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mget_experiment_by_run_id\u001b[39m(\u001b[38;5;28mself\u001b[39m, run_id: \u001b[38;5;28mstr\u001b[39m) -> \u001b[38;5;28mdict\u001b[39m:\n\u001b[32m 92\u001b[39m \u001b[38;5;66;03m# Get the run information using the run_id\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m93\u001b[39m run = \u001b[43mmlflow\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 95\u001b[39m \u001b[38;5;66;03m# Extract the experiment ID from the run\u001b[39;00m\n\u001b[32m 96\u001b[39m experiment_id = run.info.experiment_id\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/tracking/fluent.py:580\u001b[39m, in \u001b[36mget_run\u001b[39m\u001b[34m(run_id)\u001b[39m\n\u001b[32m 546\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mget_run\u001b[39m(run_id: \u001b[38;5;28mstr\u001b[39m) -> Run:\n\u001b[32m 547\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 548\u001b[39m \u001b[33;03m Fetch the run from backend store. The resulting :py:class:`Run `\u001b[39;00m\n\u001b[32m 549\u001b[39m \u001b[33;03m contains a collection of run metadata -- :py:class:`RunInfo `,\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 578\u001b[39m \u001b[33;03m run_id: 7472befefc754e388e8e922824a0cca5; lifecycle_stage: active\u001b[39;00m\n\u001b[32m 579\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m580\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mMlflowClient\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/tracking/client.py:179\u001b[39m, in \u001b[36mMlflowClient.get_run\u001b[39m\u001b[34m(self, run_id)\u001b[39m\n\u001b[32m 139\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mget_run\u001b[39m(\u001b[38;5;28mself\u001b[39m, run_id: \u001b[38;5;28mstr\u001b[39m) -> Run:\n\u001b[32m 140\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 141\u001b[39m \u001b[33;03m Fetch the run from backend store. The resulting :py:class:`Run `\u001b[39;00m\n\u001b[32m 142\u001b[39m \u001b[33;03m contains a collection of run metadata -- :py:class:`RunInfo `,\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 177\u001b[39m \u001b[33;03m status: FINISHED\u001b[39;00m\n\u001b[32m 178\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m179\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_tracking_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/tracking/_tracking_service/client.py:73\u001b[39m, in \u001b[36mTrackingServiceClient.get_run\u001b[39m\u001b[34m(self, run_id)\u001b[39m\n\u001b[32m 59\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 60\u001b[39m \u001b[33;03mFetch the run from backend store. The resulting :py:class:`Run `\u001b[39;00m\n\u001b[32m 61\u001b[39m \u001b[33;03mcontains a collection of run metadata -- :py:class:`RunInfo `,\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 70\u001b[39m \u001b[33;03m raises an exception.\u001b[39;00m\n\u001b[32m 71\u001b[39m \u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 72\u001b[39m _validate_run_id(run_id)\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mstore\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/store/tracking/rest_store.py:137\u001b[39m, in \u001b[36mRestStore.get_run\u001b[39m\u001b[34m(self, run_id)\u001b[39m\n\u001b[32m 129\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 130\u001b[39m \u001b[33;03mFetch the run from backend store\u001b[39;00m\n\u001b[32m 131\u001b[39m \n\u001b[32m (...)\u001b[39m\u001b[32m 134\u001b[39m \u001b[33;03m:return: A single Run object if it exists, otherwise raises an Exception\u001b[39;00m\n\u001b[32m 135\u001b[39m \u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 136\u001b[39m req_body = message_to_json(GetRun(run_uuid=run_id, run_id=run_id))\n\u001b[32m--> \u001b[39m\u001b[32m137\u001b[39m response_proto = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_call_endpoint\u001b[49m\u001b[43m(\u001b[49m\u001b[43mGetRun\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq_body\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 138\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m Run.from_proto(response_proto.run)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/store/tracking/rest_store.py:59\u001b[39m, in \u001b[36mRestStore._call_endpoint\u001b[39m\u001b[34m(self, api, json_body)\u001b[39m\n\u001b[32m 57\u001b[39m endpoint, method = _METHOD_TO_INFO[api]\n\u001b[32m 58\u001b[39m response_proto = api.Response()\n\u001b[32m---> \u001b[39m\u001b[32m59\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcall_endpoint\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mget_host_creds\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse_proto\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/utils/rest_utils.py:220\u001b[39m, in \u001b[36mcall_endpoint\u001b[39m\u001b[34m(host_creds, endpoint, method, json_body, response_proto, extra_headers)\u001b[39m\n\u001b[32m 218\u001b[39m call_kwargs[\u001b[33m\"\u001b[39m\u001b[33mjson\u001b[39m\u001b[33m\"\u001b[39m] = json_body\n\u001b[32m 219\u001b[39m response = http_request(**call_kwargs)\n\u001b[32m--> \u001b[39m\u001b[32m220\u001b[39m response = \u001b[43mverify_rest_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 221\u001b[39m js_dict = json.loads(response.text)\n\u001b[32m 222\u001b[39m parse_dict(js_dict=js_dict, message=response_proto)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/utils/rest_utils.py:152\u001b[39m, in \u001b[36mverify_rest_response\u001b[39m\u001b[34m(response, endpoint)\u001b[39m\n\u001b[32m 150\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m response.status_code != \u001b[32m200\u001b[39m:\n\u001b[32m 151\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m _can_parse_as_json_object(response.text):\n\u001b[32m--> \u001b[39m\u001b[32m152\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m RestException(json.loads(response.text))\n\u001b[32m 153\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 154\u001b[39m base_msg = (\n\u001b[32m 155\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mAPI request to endpoint \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mendpoint\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 156\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mfailed with error code \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresponse.status_code\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m != 200\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 157\u001b[39m )\n", + "\u001b[31mRestException\u001b[39m: RESOURCE_DOES_NOT_EXIST: Run with id=1 not found" + ] + } + ], + "source": [ + "from laborious.utils.repository.model_repository import MLFlowRepository\n", + "\n", + "mlflow_repository = MLFlowRepository(\n", + " host=\"http://localhost:5080/\",\n", + " username=\"aignosi\",\n", + " password=\"aignosi\"\n", + ")\n", + "\n", + "mlflow_repository.get_experiment_by_run_id(\"1\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/values.yaml b/values.yaml index b488144..b3025ae 100644 --- a/values.yaml +++ b/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 5 +replicaCount: 3 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.2.7" + tag: "0.3.1" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: @@ -111,11 +111,32 @@ tolerations: [] affinity: {} -service: - enabled: false - type: ClusterIP - port: 4840 - targetPort: 4840 +services: + metrics: + enabled: true + type: ClusterIP + port: 9090 + targetPort: 9090 + name: metrics + +# Configuração do ServiceMonitor para o Prometheus Operator +# ref: https://github.com/prometheus-operator/prometheus-operator +serviceMonitor: + # Se true, um recurso ServiceMonitor será criado. + enabled: true + # O intervalo no qual as métricas devem ser coletadas (ex: 30s, 1m). + interval: 30s + # O path do endpoint de métricas na sua aplicação. + path: /metrics + # Labels adicionais para o recurso ServiceMonitor. + # Essencial para que o Prometheus Operator o descubra. Se você usa o helm chart kube-prometheus-stack, + # ele procura por ServiceMonitors com o label "release: kube-prometheus-stack". + additionalLabels: + release: kube-prometheus-stack + # Configurações de relabeling adicionais, se necessário. + # ref: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + port: metrics env: @@ -123,7 +144,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-laborious_temporal.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1172-criar-pipeline-de-alertas-orquestrador" + value: "SIENTIAPDE-1174-mapear-e-implementar-metricas-a-serem-criadas" - name: PYTHON_APP value: "laborious.worker.worker" @@ -162,6 +183,8 @@ env: - name: LOG_LEVEL value: "DEBUG" + - name: HTTP_METRICS_PORT + value: "9090" - name: PROJECT_NAME value: "sientia-laborious" From d2952654b4cc97a523ec92b356b8a8f44fe8d804 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 1 Aug 2025 10:03:19 -0300 Subject: [PATCH 02/12] SIENTIAPDE-1174 SIENTIAPDE-1174 Add write_metrics activity to main workflow for enhanced metrics tracking - Included the write_metrics activity in the main workflow to support Prometheus metrics tracking. --- laborious/worker/worker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index 4e72058..674c564 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -102,7 +102,8 @@ async def main(): # Postgres activities.load_custom_query, activities.repeat_last_prediction, - activities.export_data_to_postgres + activities.export_data_to_postgres, + activities.write_metrics ], max_concurrent_workflow_tasks=100, max_concurrent_activities=100, From 4fb89b3862f93796b9234f99b6eb5522852c32ce Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 1 Aug 2025 10:16:41 -0300 Subject: [PATCH 03/12] SIENTIAPDE-1174 Comment out data.reset_index in MLFlow to prevent index reset during data processing, improving data handling without altering existing functionality. --- laborious/activities/mlflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index b5c3d8a..ecaf3c5 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -56,7 +56,7 @@ class MLFlow(BaseActivity): index='timestamp', columns='variable', values='value') data.fillna(np.nan, inplace=True) - data.reset_index(inplace=True) + # data.reset_index(inplace=True) data.columns.name = None self.debug("Processed input data:", metadata) From 4f6f4d1fa9c8a556d43331e4be5fc865abf296ea Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 1 Aug 2025 10:20:42 -0300 Subject: [PATCH 04/12] SIENTIAPDE-1174 Enable data index reset in MLFlow by uncommenting data.reset_index, improving data processing consistency. --- laborious/activities/mlflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index ecaf3c5..b5c3d8a 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -56,7 +56,7 @@ class MLFlow(BaseActivity): index='timestamp', columns='variable', values='value') data.fillna(np.nan, inplace=True) - # data.reset_index(inplace=True) + data.reset_index(inplace=True) data.columns.name = None self.debug("Processed input data:", metadata) From 963f089c7b1d1a3e70ca37de37321a8bd74f7d73 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 1 Aug 2025 10:32:06 -0300 Subject: [PATCH 05/12] SIENTIAPDE-1174 SIENTIAPDE-1174 Update replica count and enhance logging in MLFlow - Changed replicaCount in values.yaml from 3 to 1 for reduced resource usage. - Added debug logging for prediction response data in MLFlow to improve traceability. --- laborious/activities/mlflow.py | 1 + laborious/utils/repository/model_repository.py | 2 +- values.yaml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index b5c3d8a..4e5e3a6 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -95,6 +95,7 @@ class MLFlow(BaseActivity): response_data = self.model_monitoring_repository.predict( model_name, data, model_retention) + self.debug("Prediction response data:", metadata) self.debug(response_data, metadata) return response_data diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index edc99e8..c3fe550 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -68,7 +68,7 @@ class MLFlowRepository(): try: start_time = datetime.now() data = self.model_serving.get_cached_predict( - model_name, data, model_retention)[0:1] + model_name, data, model_retention) end_time = datetime.now() data = pd.DataFrame(data, columns=['prediction']) diff --git a/values.yaml b/values.yaml index b3025ae..490dba6 100644 --- a/values.yaml +++ b/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 3 +replicaCount: 1 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: From 669494c90b3d133b209842a294207a257f9aeb88 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 1 Aug 2025 12:01:29 -0300 Subject: [PATCH 06/12] SIENTIAPDE-1174 Mark application as DOWN in metrics on shutdown to improve failure reporting --- laborious/worker/worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index 674c564..9540258 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -131,6 +131,7 @@ async def main(): if activities: activities.shutdown() # Exit with a non-zero status code to indicate failure to Kubernetes + metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN sys.exit(1) From 47df5ebc60e9a234fe052e6f754e44e13e05c95e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 1 Aug 2025 17:15:49 -0300 Subject: [PATCH 07/12] SIENTIAPDE-1174 SIENTIAPDE-1174 Update MLFlowRepository initialization to include logger for enhanced logging capabilities --- laborious/activities/mlflow.py | 2 +- laborious/utils/repository/model_repository.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 4e5e3a6..af53a22 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -23,7 +23,7 @@ class MLFlow(BaseActivity): self.mlflow_password = mlflow_password self.model_monitoring_repository = MLFlowRepository( - f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password + f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger.base_logger ) @activity.defn(name="request_transform") diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index c3fe550..d518750 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -19,10 +19,11 @@ from sientia.ModelServing import ModelServing class MLFlowRepository(): - def __init__(self, host, username, password): + def __init__(self, host, username, password, logger): self.model_serving = ModelServing(tracking_uri=host, - username=username, password=password) + username=username, password=password, + logger=logger) def transform(self, model_name: str, data: pd.DataFrame, model_retention: int): """ From 663b286ad7ea1db9a6e94330a140ad90dbfbfe27 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 4 Aug 2025 10:03:15 -0300 Subject: [PATCH 08/12] SIENTIAPDE-1174 SIENTIAPDE-1174 Update tests.ipynb to normalize counter and rollout values, and fix MLFlowRepository initialization error - Changed execution_count for a cell to null for consistency. - Updated normalization logic for counter and rollout values to scale between -100 and 100. - Fixed TypeError in MLFlowRepository initialization by ensuring the logger argument is provided. --- tests.ipynb | 28 ++++++++++------------------ values.yaml | 2 +- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/tests.ipynb b/tests.ipynb index d83bd27..0c8c8d0 100644 --- a/tests.ipynb +++ b/tests.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "b10e5c25", "metadata": {}, "outputs": [ @@ -142,9 +142,9 @@ " counter[i] = counter[i-1] + counter_steps[i-1]\n", " rollout[i] = rollout[i-1] + rollout_steps[i-1]\n", "\n", - "# if any of the values is greater than 100, set it to 100\n", - "counter = np.where(counter > 100, 100, counter)\n", - "rollout = np.where(rollout > 100, 100, rollout)\n", + "# normalize values between -100 and 100, lowest value is -100, highest value is 100\n", + "counter = (counter - min(counter)) / (max(counter) - min(counter)) * 200 - 100\n", + "rollout = (rollout - min(rollout)) / (max(rollout) - min(rollout)) * 200 - 100\n", "\n", "# Create DataFrame\n", "df = pd.DataFrame({\n", @@ -165,27 +165,19 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 2, "id": "e7c8eeb1", "metadata": {}, "outputs": [ { - "ename": "RestException", - "evalue": "RESOURCE_DOES_NOT_EXIST: Run with id=1 not found", + "ename": "TypeError", + "evalue": "MLFlowRepository.__init__() missing 1 required positional argument: 'logger'", "output_type": "error", "traceback": [ "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mRestException\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 9\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mlaborious\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mrepository\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmodel_repository\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m MLFlowRepository\n\u001b[32m 3\u001b[39m mlflow_repository = MLFlowRepository(\n\u001b[32m 4\u001b[39m host=\u001b[33m\"\u001b[39m\u001b[33mhttp://localhost:5080/\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 5\u001b[39m username=\u001b[33m\"\u001b[39m\u001b[33maignosi\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 6\u001b[39m password=\u001b[33m\"\u001b[39m\u001b[33maignosi\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 7\u001b[39m )\n\u001b[32m----> \u001b[39m\u001b[32m9\u001b[39m \u001b[43mmlflow_repository\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_experiment_by_run_id\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m1\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/laborious/utils/repository/model_repository.py:93\u001b[39m, in \u001b[36mMLFlowRepository.get_experiment_by_run_id\u001b[39m\u001b[34m(self, run_id)\u001b[39m\n\u001b[32m 91\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mget_experiment_by_run_id\u001b[39m(\u001b[38;5;28mself\u001b[39m, run_id: \u001b[38;5;28mstr\u001b[39m) -> \u001b[38;5;28mdict\u001b[39m:\n\u001b[32m 92\u001b[39m \u001b[38;5;66;03m# Get the run information using the run_id\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m93\u001b[39m run = \u001b[43mmlflow\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 95\u001b[39m \u001b[38;5;66;03m# Extract the experiment ID from the run\u001b[39;00m\n\u001b[32m 96\u001b[39m experiment_id = run.info.experiment_id\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/tracking/fluent.py:580\u001b[39m, in \u001b[36mget_run\u001b[39m\u001b[34m(run_id)\u001b[39m\n\u001b[32m 546\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mget_run\u001b[39m(run_id: \u001b[38;5;28mstr\u001b[39m) -> Run:\n\u001b[32m 547\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 548\u001b[39m \u001b[33;03m Fetch the run from backend store. The resulting :py:class:`Run `\u001b[39;00m\n\u001b[32m 549\u001b[39m \u001b[33;03m contains a collection of run metadata -- :py:class:`RunInfo `,\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 578\u001b[39m \u001b[33;03m run_id: 7472befefc754e388e8e922824a0cca5; lifecycle_stage: active\u001b[39;00m\n\u001b[32m 579\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m580\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mMlflowClient\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/tracking/client.py:179\u001b[39m, in \u001b[36mMlflowClient.get_run\u001b[39m\u001b[34m(self, run_id)\u001b[39m\n\u001b[32m 139\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mget_run\u001b[39m(\u001b[38;5;28mself\u001b[39m, run_id: \u001b[38;5;28mstr\u001b[39m) -> Run:\n\u001b[32m 140\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 141\u001b[39m \u001b[33;03m Fetch the run from backend store. The resulting :py:class:`Run `\u001b[39;00m\n\u001b[32m 142\u001b[39m \u001b[33;03m contains a collection of run metadata -- :py:class:`RunInfo `,\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 177\u001b[39m \u001b[33;03m status: FINISHED\u001b[39;00m\n\u001b[32m 178\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m179\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_tracking_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/tracking/_tracking_service/client.py:73\u001b[39m, in \u001b[36mTrackingServiceClient.get_run\u001b[39m\u001b[34m(self, run_id)\u001b[39m\n\u001b[32m 59\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 60\u001b[39m \u001b[33;03mFetch the run from backend store. The resulting :py:class:`Run `\u001b[39;00m\n\u001b[32m 61\u001b[39m \u001b[33;03mcontains a collection of run metadata -- :py:class:`RunInfo `,\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 70\u001b[39m \u001b[33;03m raises an exception.\u001b[39;00m\n\u001b[32m 71\u001b[39m \u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 72\u001b[39m _validate_run_id(run_id)\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mstore\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/store/tracking/rest_store.py:137\u001b[39m, in \u001b[36mRestStore.get_run\u001b[39m\u001b[34m(self, run_id)\u001b[39m\n\u001b[32m 129\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 130\u001b[39m \u001b[33;03mFetch the run from backend store\u001b[39;00m\n\u001b[32m 131\u001b[39m \n\u001b[32m (...)\u001b[39m\u001b[32m 134\u001b[39m \u001b[33;03m:return: A single Run object if it exists, otherwise raises an Exception\u001b[39;00m\n\u001b[32m 135\u001b[39m \u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 136\u001b[39m req_body = message_to_json(GetRun(run_uuid=run_id, run_id=run_id))\n\u001b[32m--> \u001b[39m\u001b[32m137\u001b[39m response_proto = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_call_endpoint\u001b[49m\u001b[43m(\u001b[49m\u001b[43mGetRun\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq_body\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 138\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m Run.from_proto(response_proto.run)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/store/tracking/rest_store.py:59\u001b[39m, in \u001b[36mRestStore._call_endpoint\u001b[39m\u001b[34m(self, api, json_body)\u001b[39m\n\u001b[32m 57\u001b[39m endpoint, method = _METHOD_TO_INFO[api]\n\u001b[32m 58\u001b[39m response_proto = api.Response()\n\u001b[32m---> \u001b[39m\u001b[32m59\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcall_endpoint\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mget_host_creds\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse_proto\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/utils/rest_utils.py:220\u001b[39m, in \u001b[36mcall_endpoint\u001b[39m\u001b[34m(host_creds, endpoint, method, json_body, response_proto, extra_headers)\u001b[39m\n\u001b[32m 218\u001b[39m call_kwargs[\u001b[33m\"\u001b[39m\u001b[33mjson\u001b[39m\u001b[33m\"\u001b[39m] = json_body\n\u001b[32m 219\u001b[39m response = http_request(**call_kwargs)\n\u001b[32m--> \u001b[39m\u001b[32m220\u001b[39m response = \u001b[43mverify_rest_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 221\u001b[39m js_dict = json.loads(response.text)\n\u001b[32m 222\u001b[39m parse_dict(js_dict=js_dict, message=response_proto)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/utils/rest_utils.py:152\u001b[39m, in \u001b[36mverify_rest_response\u001b[39m\u001b[34m(response, endpoint)\u001b[39m\n\u001b[32m 150\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m response.status_code != \u001b[32m200\u001b[39m:\n\u001b[32m 151\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m _can_parse_as_json_object(response.text):\n\u001b[32m--> \u001b[39m\u001b[32m152\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m RestException(json.loads(response.text))\n\u001b[32m 153\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 154\u001b[39m base_msg = (\n\u001b[32m 155\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mAPI request to endpoint \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mendpoint\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 156\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mfailed with error code \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresponse.status_code\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m != 200\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 157\u001b[39m )\n", - "\u001b[31mRestException\u001b[39m: RESOURCE_DOES_NOT_EXIST: Run with id=1 not found" + "\u001b[31mTypeError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mlaborious\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mrepository\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmodel_repository\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m MLFlowRepository\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m mlflow_repository = \u001b[43mMLFlowRepository\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 4\u001b[39m \u001b[43m \u001b[49m\u001b[43mhost\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mhttp://localhost:5080/\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 5\u001b[39m \u001b[43m \u001b[49m\u001b[43musername\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43maignosi\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 6\u001b[39m \u001b[43m \u001b[49m\u001b[43mpassword\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43maignosi\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\n\u001b[32m 7\u001b[39m \u001b[43m)\u001b[49m\n\u001b[32m 9\u001b[39m mlflow_repository.get_experiment_by_run_id(\u001b[33m\"\u001b[39m\u001b[33m1\u001b[39m\u001b[33m\"\u001b[39m)\n", + "\u001b[31mTypeError\u001b[39m: MLFlowRepository.__init__() missing 1 required positional argument: 'logger'" ] } ], diff --git a/values.yaml b/values.yaml index 490dba6..5b58b3a 100644 --- a/values.yaml +++ b/values.yaml @@ -212,7 +212,7 @@ ssh: # kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp -# helm upgrade --install sientia-laborious-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.4.0-uat +# helm upgrade --install sientia-laborious-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.4.0 # kubectl create secret generic git-ssh-key-sientia-laborious-worker \ # --namespace sientia \ From 3cf7b4ec16884896f3053dcf0108d28bee602ce9 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 4 Aug 2025 15:24:20 -0300 Subject: [PATCH 09/12] SIENTIAPDE-1174 SIENTIAPDE-1174 Update replica count and image tag in values.yaml - Increased replicaCount from 1 to 3 for improved availability. - Updated image tag from 0.3.1 to 0.3.2 for the latest features and fixes. --- values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/values.yaml b/values.yaml index 5b58b3a..dfd27ad 100644 --- a/values.yaml +++ b/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 1 +replicaCount: 3 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.3.1" + tag: "0.3.2" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: From c92e2f41c3ad05c150df2742a5b1e66843fa5773 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 4 Aug 2025 16:09:26 -0300 Subject: [PATCH 10/12] SIENTIAPDE-1174 pdate metrics tracking to include response time histogram - Changed PREDICTION_RESPONSE_TIME_MONITOR from Gauge to Histogram for better response time analysis. - Updated response time observation method in gates.py to utilize the new Histogram functionality. --- laborious/activities/gates.py | 2 +- laborious/metrics.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index 739fa4b..38ee483 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -340,4 +340,4 @@ class Gates(BaseActivity): pod_id=self.pod_id, model_name=metadata['model_name'], pipeline_name=metadata['workflow_name'] - ).set(response_time) + ).observe(response_time) diff --git a/laborious/metrics.py b/laborious/metrics.py index eaad6a7..a3adcf9 100644 --- a/laborious/metrics.py +++ b/laborious/metrics.py @@ -1,4 +1,4 @@ -from prometheus_client import Gauge, Counter +from prometheus_client import Gauge, Counter, Histogram APP_UP = Gauge( "app_up", @@ -20,8 +20,9 @@ PREDICTION_CONFIDENCE_MONITOR = Gauge( CORE_LABELS, ) -PREDICTION_RESPONSE_TIME_MONITOR = Gauge( +PREDICTION_RESPONSE_TIME_MONITOR = Histogram( "laborious_prediction_response_time_monitor", "Current response time of each prediction", CORE_LABELS, + buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0] ) From fbdf9c3c6c937a6b135a2a596283f71f4f5bfc1e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 5 Aug 2025 14:19:53 -0300 Subject: [PATCH 11/12] SIENTIAPDE-1174 SIENTIAPDE-1174 Update tests to enhance MLFlow and workflow activity assertions - Added logger initialization in MLFlowRepository for improved logging. - Updated prediction test assertions to reflect changes in output structure. - Increased activity method call count assertions in workflow tests for accuracy. --- tests/laborious/activities/test_mlflow.py | 2 +- tests/laborious/utils/repository/test_model_repository.py | 7 +++++-- .../subworkflows/test_format_and_export_prediction.py | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index 8995157..fc80f5a 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -24,7 +24,7 @@ def test___init__(mock_mlflow_repository): assert mlflow.mlflow_password == "admin" mock_mlflow_repository.assert_called_once_with( - "http://localhost:5000", "admin", "admin" + "http://localhost:5000", "admin", "admin", ANY ) diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 62f4fc3..29cbe5c 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -16,7 +16,8 @@ def mlflow_repository(): repo = MLFlowRepository( host='http://localhost:5000', username='admin', - password='admin' + password='admin', + logger=MagicMock() ) return repo @@ -71,7 +72,9 @@ def test_predict_success(mlflow_repository): assert output['success'] is True assert output['content'] == {'prediction': { - 0: 3}, 'response_time': ANY} + 0: 2, + 1: 3 + }, 'response_time': ANY} def test_predict_error(mlflow_repository): diff --git a/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py b/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py index 7dbf39a..648a38f 100644 --- a/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py +++ b/tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py @@ -79,7 +79,7 @@ async def test_run_none_path_flag(workflow_mock, format_and_export_prediction): start_to_close_timeout=ANY )]) - assert workflow_mock.execute_activity_method.call_count == 2 + assert workflow_mock.execute_activity_method.call_count == 3 assert workflow_mock.execute_local_activity_method.call_count == 1 @@ -145,5 +145,5 @@ async def test_run_default_path_flag(workflow_mock, format_and_export_prediction ) ]) - assert workflow_mock.execute_activity_method.call_count == 2 + assert workflow_mock.execute_activity_method.call_count == 3 assert workflow_mock.execute_local_activity_method.call_count == 1 From dd513ddfafb99f2f0f11b620cabb6b555c1f3493 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 5 Aug 2025 14:39:02 -0300 Subject: [PATCH 12/12] SIENTIAPDE-1174 Add async test for write_metrics method in gates_activity - Implemented a new test to validate the write_metrics functionality, ensuring metrics are correctly recorded for predictions, confidence, and response time. - Utilized mocking to verify interactions with the metrics tracking system. --- tests/laborious/activities/test_gates.py | 39 ++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/laborious/activities/test_gates.py b/tests/laborious/activities/test_gates.py index 19d52cd..e1ba49d 100644 --- a/tests/laborious/activities/test_gates.py +++ b/tests/laborious/activities/test_gates.py @@ -402,3 +402,42 @@ async def test_get_last_timestamp_no_data(gates_activity): # Assert assert isinstance(result, str) # Should be a timestamp string assert len(result) > 0 + + +@mark.asyncio +@patch('laborious.activities.gates.metrics') +async def test_write_metrics(mock_metrics, gates_activity): + """Test write_metrics method.""" + input_data = { + **metadata, + 'prediction': { + 'prediction': [1, 2, 3], + 'prediction_confidence': [0.9, 0.8, 0.7], + 'response_time': [0.1, 0.2, 0.3] + } + } + await gates_activity.write_metrics(input_data) + mock_metrics.PREDICTIONS_WRITTEN_COUNT.labels.assert_called_once_with( + pod_id=gates_activity.pod_id, + model_name=metadata['metadata']['model_name'], + pipeline_name=metadata['metadata']['workflow_name'] + ) + mock_metrics.PREDICTIONS_WRITTEN_COUNT.labels.return_value.inc.assert_called_once_with() + + mock_metrics.PREDICTION_CONFIDENCE_MONITOR.labels.assert_called_once_with( + pod_id=gates_activity.pod_id, + model_name=metadata['metadata']['model_name'], + pipeline_name=metadata['metadata']['workflow_name'] + ) + mock_metrics.PREDICTION_CONFIDENCE_MONITOR.labels.return_value.set.assert_called_once_with( + 0.9 + ) + + mock_metrics.PREDICTION_RESPONSE_TIME_MONITOR.labels.assert_called_once_with( + pod_id=gates_activity.pod_id, + model_name=metadata['metadata']['model_name'], + pipeline_name=metadata['metadata']['workflow_name'] + ) + mock_metrics.PREDICTION_RESPONSE_TIME_MONITOR.labels.return_value.observe.assert_called_once_with( + 0.1 + )