gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1174

Browse Source 
Update dependencies, modify replica count, and implement metrics tracking

- Updated sientia-dataops-library version from 1.3.5 to 1.3.7 in requirements.txt.
- Changed replicaCount in values.yaml from 5 to 3 and incremented image tag from 0.2.7 to 0.3.1.
- Added Prometheus metrics tracking in gates.py and worker.py, including a new write_metrics method.
- Configured Prometheus service and ServiceMonitor in values.yaml for metrics collection.
This commit is contained in:
vitor-aignosi
2025-08-01 10:00:47 -03:00
parent 6fb28a0050
commit 70f1abe681
8 changed files with 349 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
laborious/activities/gates.py
View File

@@ -15,6 +15,7 @@ with workflow.unsafe.imports_passed_through():
)
from pandas import DataFrame
from datetime import datetime
from laborious import metrics
input_filter_functions = {
'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values,
@@ -309,3 +310,34 @@ class Gates(BaseActivity):
if data.empty:
return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
return max(data['timestamp'].values.tolist())
@activity.defn(name="write_metrics")
async def write_metrics(self, input_data: dict[str, Any]):
"""
Write metrics to the database.
input_data:
metadata: dict[str, Any]
prediction: dict[str, Any]
"""
metadata = input_data['metadata']
prediction = DataFrame(input_data['prediction'])
prediction_confidence = prediction['prediction_confidence'].values[0]
response_time = prediction['response_time'].values[0]
metrics.PREDICTIONS_WRITTEN_COUNT.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name']
).inc()
metrics.PREDICTION_CONFIDENCE_MONITOR.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name']
).set(prediction_confidence)
metrics.PREDICTION_RESPONSE_TIME_MONITOR.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name']
).set(response_time)

27
laborious/metrics.py Normal file
View File

@@ -0,0 +1,27 @@
from prometheus_client import Gauge, Counter
APP_UP = Gauge(
"app_up",
"Indicates if the application is running (1) or shutting down (0)",
["pod_id"],
)
CORE_LABELS = ["pod_id", "model_name", "pipeline_name"]
PREDICTIONS_WRITTEN_COUNT = Counter(
"laborious_predictions_written_count",
"Number of predictions written to the database table predictions",
CORE_LABELS,
)
PREDICTION_CONFIDENCE_MONITOR = Gauge(
"laborious_prediction_confidence_monitor",
"Current confidence of each prediction",
CORE_LABELS,
)
PREDICTION_RESPONSE_TIME_MONITOR = Gauge(
"laborious_prediction_response_time_monitor",
"Current response time of each prediction",
CORE_LABELS,
)

2
laborious/utils/repository/model_repository.py
View File

@@ -68,7 +68,7 @@ class MLFlowRepository():
try:
start_time = datetime.now()
data = self.model_serving.get_cached_predict(
model_name, data, model_retention)[-1:]
model_name, data, model_retention)[0:1]
end_time = datetime.now()
data = pd.DataFrame(data, columns=['prediction'])

21
laborious/worker/worker.py
View File

@@ -20,13 +20,20 @@ with workflow.unsafe.imports_passed_through():
)
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.temporal.utils.logger import get_logger
from laborious import metrics
from prometheus_client import start_http_server
POD_ID = os.getenv('POD_ID')
async def main():
host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
logger = get_logger(__name__)
logger.info('Starting Worker...')
logger.info(f'Starting Worker with POD_ID: {POD_ID}')
logger.info("Starting prometheus client...")
start_prometheus_server()
logger.info('Starting Notification Handler...')
@@ -125,5 +132,17 @@ async def main():
# Exit with a non-zero status code to indicate failure to Kubernetes
sys.exit(1)
def start_prometheus_server():
try:
port = int(os.getenv("HTTP_METRICS_PORT", 9090))
start_http_server(port)
print(f"Prometheus server started on port {port}.")
metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP
except Exception as e:
print(f"Failed to start Prometheus server: {e}")
os._exit(1)
if __name__ == '__main__':
asyncio.run(main())

10
laborious/workflows/sub_workflows/format_and_export_prediction.py
View File

@@ -95,3 +95,13 @@ class FormatAndExportPrediction():
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
)
await workflow.execute_activity_method(
Activities.write_metrics,
{
**metadata,
'prediction': prediction
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
)

3
requirements.txt
View File

@@ -3,5 +3,6 @@ psycopg2-binary
sqlalchemy
asyncua
redis
git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.5
git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.7
git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.5
prometheus-client

226
tests.ipynb Normal file
View File

@@ -0,0 +1,226 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 6,
"id": "b10e5c25",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[ 0.5479121 -0.12224312 0.71719584 0.39473606 -0.8116453 0.9512447\n",
" 0.5222794 0.57212861 -0.74377273 -0.09922812 -0.25840395 0.85352998\n",
" 0.28773024 0.64552323 -0.1131716 -0.54552256 0.10916957 -0.87236549\n",
" 0.65526234 0.2633288 0.51617548 -0.29094806 0.94139605 0.78624224\n",
" 0.55676699 -0.61072258 -0.06655799 -0.91239247 -0.69142102 0.36609791\n",
" 0.48952431 0.93501946 -0.34834928 -0.25908059 -0.06088838 -0.62105728\n",
" -0.74015699 -0.04859015 -0.5461813 0.33962799 -0.12569616 0.66535639\n",
" 0.4005302 -0.37526672 0.6645196 0.60952871 -0.22504324 -0.42334379\n",
" 0.36499101 -0.72049503 -0.6001836 -0.98527546 0.57384876 0.32970171\n",
" 0.41033076 0.56145806 -0.08216845 0.13748239 -0.720406 -0.77093985\n",
" 0.33680592 -0.05780759 0.13047221 0.52999771 0.26943664 0.1071588\n",
" 0.11841432 -0.3920998 -0.93836433 -0.12656522 -0.57083065 -0.18294271\n",
" 0.70680615 -0.53212103 -0.88339452 -0.43723222 -0.41281248 0.32383303\n",
" 0.1140643 0.56779642 0.32862708 -0.18722628 0.62804077 -0.66605416\n",
" -0.95457585 -0.81990428 0.4447187 -0.07624554 -0.67745644 0.00208955\n",
" -0.69537579 0.39264075 -0.10768745 -0.23795755 -0.39697582 0.26056519\n",
" -0.27637478 -0.82470016 -0.7639882 0.92379533 0.81716138 0.39941427\n",
" -0.46826008 0.93835275 0.55750181 0.43378038 -0.101277 -0.45551688\n",
" -0.80721808 0.80520479 -0.08844742 -0.59527327 -0.38808675 0.15843914\n",
" -0.64645443 0.71322857 0.51703906 0.43892591 -0.13581392 0.25461768\n",
" 0.16819594 0.2996932 -0.83111136 -0.1683852 -0.91677165 -0.01201836\n",
" -0.34027758 -0.71095162 -0.79319406 0.17528914 -0.65881406 0.85024024\n",
" 0.16212228 -0.30626039 0.18183098 -0.95439226 0.91711843 -0.03539313\n",
" 0.56547045 -0.83454 -0.02668334 -0.01858601 0.87565291 0.1434561\n",
" -0.0530212 -0.46604867 -0.33686201 0.0413448 -0.12217708 -0.95677584\n",
" 0.65258385 0.79232154 -0.71950182 0.10807229 -0.78284852 0.34448019\n",
" -0.43753243 0.31884527 0.45398923 0.53729498 -0.78451811 0.83202369\n",
" -0.53957202 -0.92517489 0.10970494 -0.25815543 0.65957949 0.61650294\n",
" -0.36572221 0.90579879 -0.41816432 0.03011426 -0.48806982 0.87208714\n",
" -0.67078436 -0.91017876 -0.12980588 0.98475113 0.78335453 0.49721604\n",
" 0.78158498 0.78689328 0.03771672 -0.3681419 0.54402486 0.32332253\n",
" -0.25268454 -0.81106666 0.49357922 -0.47507897 0.8736263 -0.51805885\n",
" -0.75448414 0.66222534 -0.69343137 -0.64146338 0.19876558 0.74912408\n",
" -0.60713067 -0.37935265 0.55480968 0.94365285 0.00148237 -0.71220499\n",
" -0.97212742 -0.54068794 -0.73635556 0.35531735 -0.75633499 0.01265986\n",
" 0.38852487 0.16223322 -0.6004487 0.60824905 0.43081426 0.47796801\n",
" -0.7378845 -0.75249239 0.8551251 -0.20484361 -0.39810262 -0.02283191\n",
" 0.32572843 0.91124651 -0.42710755 0.84961686 -0.95028102 0.11039608\n",
" 0.26795022 -0.78820519 -0.71932081 -0.16177136 0.93246382 0.19208511\n",
" 0.86604644 0.60872183 -0.0652368 0.5695269 -0.96432643 -0.78171201\n",
" 0.65885723 0.59363418 -0.53471852 0.06153918 0.21203164 0.73547791\n",
" 0.20621431 -0.17485686 -0.25163191 -0.14823583 0.30386205 0.73498126\n",
" -0.09220624 -0.50432087 -0.52667527 0.49202856 0.63313753 -0.78944384\n",
" -0.86688229 0.18886733 -0.70765351 0.64932838 -0.37933065 -0.71225613\n",
" 0.84194094 -0.66893655 -0.43055984 -0.69277321 -0.76901987 -0.95770397\n",
" -0.88920918 -0.65071706 -0.89323613 0.18228763 0.36142905 -0.21273909\n",
" -0.36401781 0.00905247 0.75000988 0.70226325 -0.91304988 -0.63700318\n",
" -0.52651026 -0.50122485 0.1424653 -0.16747515 -0.90149176 -0.25277172\n",
" 0.0475059 -0.79665619 0.66691711 -0.89607627 0.84968374 -0.80177372\n",
" 0.6871499 0.80530629 0.95914136 0.60405176 0.55895508]\n",
"[ 0.28496655 0.55799271 -0.73089558 0.07213607 0.02844574 0.71514429\n",
" -0.07440127 -0.22982101 0.27912654 -0.46707336 -0.72046318 -0.04424545\n",
" -0.16622126 -0.53486012 -0.26497638 -0.2672151 -0.34500887 -0.24107184\n",
" 0.37148669 -0.40624705 0.89771585 0.83269604 -0.03817914 -0.34327759\n",
" 0.07086958 0.69712098 0.30517468 0.60878366 0.06544455 0.26583526\n",
" -0.42368877 0.46978632 -0.59519081 0.38959626 0.72143814 -0.73579433\n",
" 0.22875948 -0.8098085 0.45143126 -0.83101356 0.87187965 -0.72518414\n",
" 0.91776049 0.60176835 0.18736401 0.56524821 0.59022968 0.89205413\n",
" -0.49323329 0.18015179 -0.8099016 0.2323314 -0.65741739 0.12990122\n",
" 0.14486103 -0.06802969 0.04526355 0.52784678 0.59848943 -0.01569357\n",
" 0.19918688 0.86247247 -0.76053282 -0.76579287 -0.82458198 0.31572657\n",
" -0.1627834 0.54864283 0.34246283 -0.33272448 0.79673309 0.52506429\n",
" -0.45893012 -0.27161596 -0.37112004 -0.6847767 -0.70443325 0.87225493\n",
" -0.12419193 -0.23336035 0.45937142 0.10598613 0.87227997 0.56060299\n",
" -0.04126087 -0.24728105 0.97326309 0.43552047 0.90238932 -0.76304285\n",
" 0.70106736 0.27414777 -0.75615664 0.176516 0.37219273 -0.97539463\n",
" -0.09136408 0.65079902 -0.40928195 -0.08290384 -0.11537175 -0.39614522\n",
" 0.83688379 0.56258807 -0.77882318 0.99406932 0.75840005 -0.43218312\n",
" 0.67379316 -0.78716094 0.99820946 0.33136947 0.30025003 -0.81911855\n",
" 0.7940668 -0.94200099 -0.51834388 -0.71395625 0.55353588 -0.60359155\n",
" 0.82127645 0.31253808 -0.92767458 -0.98914033 -0.89668417 0.21185036\n",
" 0.60296362 -0.52289436 0.69881769 -0.88553612 0.60192771 0.85559086\n",
" 0.5442168 0.39624157 0.67596044 -0.9196974 -0.59643578 -0.75015264\n",
" 0.00906198 0.49037626 0.26002369 0.7022622 -0.68957402 0.46924218\n",
" -0.61391702 -0.4584825 0.41980939 0.96040957 0.22308721 -0.89099937\n",
" 0.23261794 -0.9152989 0.76829142 0.41915657 -0.65374431 -0.81655799\n",
" -0.63293354 0.96005436 -0.08287872 0.5681619 0.27281668 0.1448263\n",
" -0.70973949 0.89204891 -0.39731473 0.15603443 0.39955189 0.29846631\n",
" 0.88118882 -0.70312202 0.01670548 -0.19193122 -0.05166254 -0.76156495\n",
" -0.73181078 -0.44384891 -0.39059079 -0.14419357 0.22197509 0.26925823\n",
" -0.17637821 -0.18243378 -0.56474295 0.1766125 -0.36591818 -0.92788033\n",
" -0.16319991 -0.05173465 -0.54881426 0.14491587 0.1315438 0.40400436\n",
" 0.29589696 0.30486611 -0.3675717 0.57486444 0.09828877 -0.13716361\n",
" 0.25202496 -0.27868533 0.02547849 0.47341138 0.77280577 0.84211439\n",
" 0.00726585 0.04055023 0.59974082 -0.37109862 0.67476472 -0.01171671\n",
" -0.76828655 -0.85588171 0.68398642 -0.88886417 -0.43877713 -0.33173992\n",
" -0.65401111 -0.37221326 0.48538513 -0.97063431 0.65434685 0.71309605\n",
" -0.25547685 -0.6927742 0.20168082 -0.76065489 -0.27016128 0.91685836\n",
" 0.99092895 0.54420978 -0.37807698 0.3753301 0.41081273 -0.22431661\n",
" 0.28177727 -0.97854471 -0.58188468 0.05017661 -0.67249739 -0.66818626\n",
" 0.67260858 0.97826601 0.11193886 0.67813946 0.98064333 -0.71680822\n",
" -0.10350877 -0.21485457 -0.83990143 0.51066035 -0.13244195 -0.06134613\n",
" -0.69865405 -0.6381467 0.81420724 -0.91070182 -0.53429543 -0.41588134\n",
" -0.01960492 0.17289035 -0.01342005 -0.83176933 -0.51266509 0.68717677\n",
" 0.2751774 0.2982981 0.34040651 0.52580604 -0.88378304 -0.26678323\n",
" 0.07905487 -0.32308703 0.68895775 -0.03485498 0.53725518 0.70403103\n",
" 0.00958297 0.81910449 0.17424788 0.7005486 -0.31881841 -0.00236608\n",
" 0.06282208 -0.79004057 -0.20289499 0.83467535 0.26166448 -0.64498684\n",
" -0.32228873 -0.61679398 -0.95035374 0.85492092 -0.10358534 -0.38492986\n",
" 0.19695438 -0.98537109 -0.44395579 0.40606693 0.26753955]\n",
" Counter Rollout CounterPlusRollout Timestamp\n",
"0 0.000000 0.000000 0.000000 2025-01-01 00:00:00\n",
"1 0.547912 0.284967 0.832879 2025-01-01 00:00:01\n",
"2 0.425669 0.842959 1.268628 2025-01-01 00:00:02\n",
"3 1.142865 0.112064 1.254928 2025-01-01 00:00:03\n",
"4 1.537601 0.184200 1.721801 2025-01-01 00:00:04\n"
]
}
],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"\n",
"# Set random seed for reproducibility\n",
"rng = np.random.default_rng(42)\n",
"\n",
"# Generate random walks starting at 0\n",
"counter = np.zeros(300)\n",
"rollout = np.zeros(300)\n",
"\n",
"# Generate random steps between -1 and 1\n",
"counter_steps = rng.uniform(-1, 1, 299)\n",
"rollout_steps = rng.uniform(-1, 1, 299)\n",
"\n",
"print(counter_steps)\n",
"print(rollout_steps)\n",
"\n",
"# Calculate cumulative sum and scale to -100 to 100 range\n",
"for i in range(1, 300):\n",
" counter[i] = counter[i-1] + counter_steps[i-1]\n",
" rollout[i] = rollout[i-1] + rollout_steps[i-1]\n",
"\n",
"# if any of the values is greater than 100, set it to 100\n",
"counter = np.where(counter > 100, 100, counter)\n",
"rollout = np.where(rollout > 100, 100, rollout)\n",
"\n",
"# Create DataFrame\n",
"df = pd.DataFrame({\n",
" 'Counter': counter,\n",
" 'Rollout': rollout,\n",
" 'CounterPlusRollout': counter + rollout\n",
"})\n",
"\n",
"# add a timestamp column\n",
"df['Timestamp'] = pd.date_range(start='2025-01-01', periods=300, freq='1s')\n",
"\n",
"# Save to CSV\n",
"df.to_csv('random_walks.csv', index=False)\n",
"\n",
"# Display first few rows\n",
"print(df.head())\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "e7c8eeb1",
"metadata": {},
"outputs": [
{
"ename": "RestException",
"evalue": "RESOURCE_DOES_NOT_EXIST: Run with id=1 not found",
"output_type": "error",
"traceback": [
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
"\u001b[31mRestException\u001b[39m Traceback (most recent call last)",
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 9\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mlaborious\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mrepository\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmodel_repository\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m MLFlowRepository\n\u001b[32m 3\u001b[39m mlflow_repository = MLFlowRepository(\n\u001b[32m 4\u001b[39m host=\u001b[33m\"\u001b[39m\u001b[33mhttp://localhost:5080/\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 5\u001b[39m username=\u001b[33m\"\u001b[39m\u001b[33maignosi\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 6\u001b[39m password=\u001b[33m\"\u001b[39m\u001b[33maignosi\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 7\u001b[39m )\n\u001b[32m----> \u001b[39m\u001b[32m9\u001b[39m \u001b[43mmlflow_repository\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_experiment_by_run_id\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m1\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/laborious/utils/repository/model_repository.py:93\u001b[39m, in \u001b[36mMLFlowRepository.get_experiment_by_run_id\u001b[39m\u001b[34m(self, run_id)\u001b[39m\n\u001b[32m 91\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mget_experiment_by_run_id\u001b[39m(\u001b[38;5;28mself\u001b[39m, run_id: \u001b[38;5;28mstr\u001b[39m) -> \u001b[38;5;28mdict\u001b[39m:\n\u001b[32m 92\u001b[39m \u001b[38;5;66;03m# Get the run information using the run_id\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m93\u001b[39m run = \u001b[43mmlflow\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 95\u001b[39m \u001b[38;5;66;03m# Extract the experiment ID from the run\u001b[39;00m\n\u001b[32m 96\u001b[39m experiment_id = run.info.experiment_id\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/tracking/fluent.py:580\u001b[39m, in \u001b[36mget_run\u001b[39m\u001b[34m(run_id)\u001b[39m\n\u001b[32m 546\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mget_run\u001b[39m(run_id: \u001b[38;5;28mstr\u001b[39m) -> Run:\n\u001b[32m 547\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 548\u001b[39m \u001b[33;03m Fetch the run from backend store. The resulting :py:class:`Run <mlflow.entities.Run>`\u001b[39;00m\n\u001b[32m 549\u001b[39m \u001b[33;03m contains a collection of run metadata -- :py:class:`RunInfo <mlflow.entities.RunInfo>`,\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 578\u001b[39m \u001b[33;03m run_id: 7472befefc754e388e8e922824a0cca5; lifecycle_stage: active\u001b[39;00m\n\u001b[32m 579\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m580\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mMlflowClient\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/tracking/client.py:179\u001b[39m, in \u001b[36mMlflowClient.get_run\u001b[39m\u001b[34m(self, run_id)\u001b[39m\n\u001b[32m 139\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mget_run\u001b[39m(\u001b[38;5;28mself\u001b[39m, run_id: \u001b[38;5;28mstr\u001b[39m) -> Run:\n\u001b[32m 140\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 141\u001b[39m \u001b[33;03m Fetch the run from backend store. The resulting :py:class:`Run <mlflow.entities.Run>`\u001b[39;00m\n\u001b[32m 142\u001b[39m \u001b[33;03m contains a collection of run metadata -- :py:class:`RunInfo <mlflow.entities.RunInfo>`,\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 177\u001b[39m \u001b[33;03m status: FINISHED\u001b[39;00m\n\u001b[32m 178\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m179\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_tracking_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/tracking/_tracking_service/client.py:73\u001b[39m, in \u001b[36mTrackingServiceClient.get_run\u001b[39m\u001b[34m(self, run_id)\u001b[39m\n\u001b[32m 59\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 60\u001b[39m \u001b[33;03mFetch the run from backend store. The resulting :py:class:`Run <mlflow.entities.Run>`\u001b[39;00m\n\u001b[32m 61\u001b[39m \u001b[33;03mcontains a collection of run metadata -- :py:class:`RunInfo <mlflow.entities.RunInfo>`,\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 70\u001b[39m \u001b[33;03m raises an exception.\u001b[39;00m\n\u001b[32m 71\u001b[39m \u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 72\u001b[39m _validate_run_id(run_id)\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mstore\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/store/tracking/rest_store.py:137\u001b[39m, in \u001b[36mRestStore.get_run\u001b[39m\u001b[34m(self, run_id)\u001b[39m\n\u001b[32m 129\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 130\u001b[39m \u001b[33;03mFetch the run from backend store\u001b[39;00m\n\u001b[32m 131\u001b[39m \n\u001b[32m (...)\u001b[39m\u001b[32m 134\u001b[39m \u001b[33;03m:return: A single Run object if it exists, otherwise raises an Exception\u001b[39;00m\n\u001b[32m 135\u001b[39m \u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 136\u001b[39m req_body = message_to_json(GetRun(run_uuid=run_id, run_id=run_id))\n\u001b[32m--> \u001b[39m\u001b[32m137\u001b[39m response_proto = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_call_endpoint\u001b[49m\u001b[43m(\u001b[49m\u001b[43mGetRun\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq_body\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 138\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m Run.from_proto(response_proto.run)\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/store/tracking/rest_store.py:59\u001b[39m, in \u001b[36mRestStore._call_endpoint\u001b[39m\u001b[34m(self, api, json_body)\u001b[39m\n\u001b[32m 57\u001b[39m endpoint, method = _METHOD_TO_INFO[api]\n\u001b[32m 58\u001b[39m response_proto = api.Response()\n\u001b[32m---> \u001b[39m\u001b[32m59\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcall_endpoint\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mget_host_creds\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse_proto\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/utils/rest_utils.py:220\u001b[39m, in \u001b[36mcall_endpoint\u001b[39m\u001b[34m(host_creds, endpoint, method, json_body, response_proto, extra_headers)\u001b[39m\n\u001b[32m 218\u001b[39m call_kwargs[\u001b[33m\"\u001b[39m\u001b[33mjson\u001b[39m\u001b[33m\"\u001b[39m] = json_body\n\u001b[32m 219\u001b[39m response = http_request(**call_kwargs)\n\u001b[32m--> \u001b[39m\u001b[32m220\u001b[39m response = \u001b[43mverify_rest_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 221\u001b[39m js_dict = json.loads(response.text)\n\u001b[32m 222\u001b[39m parse_dict(js_dict=js_dict, message=response_proto)\n",
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/mlflow/utils/rest_utils.py:152\u001b[39m, in \u001b[36mverify_rest_response\u001b[39m\u001b[34m(response, endpoint)\u001b[39m\n\u001b[32m 150\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m response.status_code != \u001b[32m200\u001b[39m:\n\u001b[32m 151\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m _can_parse_as_json_object(response.text):\n\u001b[32m--> \u001b[39m\u001b[32m152\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m RestException(json.loads(response.text))\n\u001b[32m 153\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 154\u001b[39m base_msg = (\n\u001b[32m 155\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mAPI request to endpoint \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mendpoint\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 156\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mfailed with error code \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresponse.status_code\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m != 200\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 157\u001b[39m )\n",
"\u001b[31mRestException\u001b[39m: RESOURCE_DOES_NOT_EXIST: Run with id=1 not found"
]
}
],
"source": [
"from laborious.utils.repository.model_repository import MLFlowRepository\n",
"\n",
"mlflow_repository = MLFlowRepository(\n",
" host=\"http://localhost:5080/\",\n",
" username=\"aignosi\",\n",
" password=\"aignosi\"\n",
")\n",
"\n",
"mlflow_repository.get_experiment_by_run_id(\"1\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

39
values.yaml
View File

@@ -3,7 +3,7 @@
# Declare variables to be passed into your templates.
# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
replicaCount: 5
replicaCount: 3
# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
image:
@@ -11,7 +11,7 @@ image:
# This sets the pull policy for images.
pullPolicy: Always
# Overrides the image tag whose default is the chart appVersion.
tag: "0.2.7"
tag: "0.3.1"
# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
imagePullSecrets:
@@ -111,11 +111,32 @@ tolerations: []
affinity: {}
service:
enabled: false
type: ClusterIP
port: 4840
targetPort: 4840
services:
metrics:
enabled: true
type: ClusterIP
port: 9090
targetPort: 9090
name: metrics
# Configuração do ServiceMonitor para o Prometheus Operator
# ref: https://github.com/prometheus-operator/prometheus-operator
serviceMonitor:
# Se true, um recurso ServiceMonitor será criado.
enabled: true
# O intervalo no qual as métricas devem ser coletadas (ex: 30s, 1m).
interval: 30s
# O path do endpoint de métricas na sua aplicação.
path: /metrics
# Labels adicionais para o recurso ServiceMonitor.
# Essencial para que o Prometheus Operator o descubra. Se você usa o helm chart kube-prometheus-stack,
# ele procura por ServiceMonitors com o label "release: kube-prometheus-stack".
additionalLabels:
release: kube-prometheus-stack
# Configurações de relabeling adicionais, se necessário.
# ref: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
relabelings: []
port: metrics
env:
@@ -123,7 +144,7 @@ env:
- name: GITHUB_REPO_URL
value: "git@github.com:Aignosi/sientia-dataops-laborious_temporal.git"
- name: GITHUB_BRANCH
value: "SIENTIAPDE-1172-criar-pipeline-de-alertas-orquestrador"
value: "SIENTIAPDE-1174-mapear-e-implementar-metricas-a-serem-criadas"
- name: PYTHON_APP
value: "laborious.worker.worker"
@@ -162,6 +183,8 @@ env:
- name: LOG_LEVEL
value: "DEBUG"
- name: HTTP_METRICS_PORT
value: "9090"
- name: PROJECT_NAME
value: "sientia-laborious"