3ee7e3687a
Update test.ipynb to generate and save multiple pipeline specifications and input samples in JSON format. Modify values.yaml to increase replica count from 1 to 3 and adjust PostgreSQL max connections from 20 to 40 for improved scalability.
725 lines
52 KiB
Plaintext
725 lines
52 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"from sqlalchemy.engine import create_engine\n",
|
||
"\n",
|
||
"engine = create_engine('druid://localhost:8082/druid/v2/sql/')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"from sqlalchemy import MetaData, Table\n",
|
||
"\n",
|
||
"metadata = MetaData()\n",
|
||
"places = Table('raw_scouter-opcua-orchestrated-pipeline', metadata, autoload_with=engine)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"from sqlalchemy import select\n",
|
||
"\n",
|
||
"stmt = select(places)\n",
|
||
"with engine.connect() as conn:\n",
|
||
" result = conn.execute(stmt)\n",
|
||
" for row in result:\n",
|
||
" print(row)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"/tmp/ipykernel_30057/1086103244.py:5: SADeprecationWarning: The dbapi() classmethod on dialect classes has been renamed to import_dbapi(). Implement an import_dbapi() classmethod directly on class <class 'pydruid.db.sqlalchemy.DruidDialect'> to remove this warning; the old .dbapi() classmethod may be maintained for backwards compatibility.\n",
|
||
" engine = create_engine('druid://localhost:8082/druid/v2/sql/')\n",
|
||
"/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pydruid/db/sqlalchemy.py:188: SAWarning: Dialect druid:rest will not make use of SQL compilation caching as it does not set the 'supports_statement_cache' attribute to ``True``. This can have significant performance implications including some performance degradations in comparison to prior SQLAlchemy versions. Dialect maintainers should seek to set this attribute to True after appropriate development and testing for SQLAlchemy 1.4 caching support. Alternatively, this attribute may be set to False which will disable this warning. (Background on this warning at: https://sqlalche.me/e/20/cprf)\n",
|
||
" result = connection.execute(text(query))\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>name</th>\n",
|
||
" <th>kafka.topic</th>\n",
|
||
" <th>tag</th>\n",
|
||
" <th>value</th>\n",
|
||
" <th>timestamp</th>\n",
|
||
" <th>inserted_at</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>Counter</td>\n",
|
||
" <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
|
||
" <td>ns=2;i=2</td>\n",
|
||
" <td>-50.132</td>\n",
|
||
" <td>2025-07-02 13:05:19</td>\n",
|
||
" <td>2025-07-02 13:05:19.729000</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td>Rollout</td>\n",
|
||
" <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
|
||
" <td>ns=2;i=3</td>\n",
|
||
" <td>70.767</td>\n",
|
||
" <td>2025-07-02 13:05:19</td>\n",
|
||
" <td>2025-07-02 13:05:19.731000</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>Square</td>\n",
|
||
" <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
|
||
" <td>ns=2;i=4</td>\n",
|
||
" <td>-58.448</td>\n",
|
||
" <td>2025-07-02 13:05:19</td>\n",
|
||
" <td>2025-07-02 13:05:19.732000</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td>Counter</td>\n",
|
||
" <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
|
||
" <td>ns=2;i=2</td>\n",
|
||
" <td>-50.126</td>\n",
|
||
" <td>2025-07-02 13:05:24</td>\n",
|
||
" <td>2025-07-02 13:05:24.728000</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td>Rollout</td>\n",
|
||
" <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
|
||
" <td>ns=2;i=3</td>\n",
|
||
" <td>69.199</td>\n",
|
||
" <td>2025-07-02 13:05:24</td>\n",
|
||
" <td>2025-07-02 13:05:24.730000</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>...</th>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>373</th>\n",
|
||
" <td>Rollout</td>\n",
|
||
" <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
|
||
" <td>ns=2;i=3</td>\n",
|
||
" <td>85.921</td>\n",
|
||
" <td>2025-07-02 13:15:40</td>\n",
|
||
" <td>2025-07-02 13:15:40.230000</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>374</th>\n",
|
||
" <td>Square</td>\n",
|
||
" <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
|
||
" <td>ns=2;i=4</td>\n",
|
||
" <td>-69.296</td>\n",
|
||
" <td>2025-07-02 13:15:40</td>\n",
|
||
" <td>2025-07-02 13:15:40.232000</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>375</th>\n",
|
||
" <td>Counter</td>\n",
|
||
" <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
|
||
" <td>ns=2;i=2</td>\n",
|
||
" <td>-71.207</td>\n",
|
||
" <td>2025-07-02 13:15:45</td>\n",
|
||
" <td>2025-07-02 13:15:45.228000</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>376</th>\n",
|
||
" <td>Rollout</td>\n",
|
||
" <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
|
||
" <td>ns=2;i=3</td>\n",
|
||
" <td>84.665</td>\n",
|
||
" <td>2025-07-02 13:15:45</td>\n",
|
||
" <td>2025-07-02 13:15:45.231000</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>377</th>\n",
|
||
" <td>Square</td>\n",
|
||
" <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
|
||
" <td>ns=2;i=4</td>\n",
|
||
" <td>-67.592</td>\n",
|
||
" <td>2025-07-02 13:15:45</td>\n",
|
||
" <td>2025-07-02 13:15:45.233000</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"<p>378 rows × 6 columns</p>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" name kafka.topic tag value \\\n",
|
||
"0 Counter raw_scouter-opcua-orchestrated-pipeline ns=2;i=2 -50.132 \n",
|
||
"1 Rollout raw_scouter-opcua-orchestrated-pipeline ns=2;i=3 70.767 \n",
|
||
"2 Square raw_scouter-opcua-orchestrated-pipeline ns=2;i=4 -58.448 \n",
|
||
"3 Counter raw_scouter-opcua-orchestrated-pipeline ns=2;i=2 -50.126 \n",
|
||
"4 Rollout raw_scouter-opcua-orchestrated-pipeline ns=2;i=3 69.199 \n",
|
||
".. ... ... ... ... \n",
|
||
"373 Rollout raw_scouter-opcua-orchestrated-pipeline ns=2;i=3 85.921 \n",
|
||
"374 Square raw_scouter-opcua-orchestrated-pipeline ns=2;i=4 -69.296 \n",
|
||
"375 Counter raw_scouter-opcua-orchestrated-pipeline ns=2;i=2 -71.207 \n",
|
||
"376 Rollout raw_scouter-opcua-orchestrated-pipeline ns=2;i=3 84.665 \n",
|
||
"377 Square raw_scouter-opcua-orchestrated-pipeline ns=2;i=4 -67.592 \n",
|
||
"\n",
|
||
" timestamp inserted_at \n",
|
||
"0 2025-07-02 13:05:19 2025-07-02 13:05:19.729000 \n",
|
||
"1 2025-07-02 13:05:19 2025-07-02 13:05:19.731000 \n",
|
||
"2 2025-07-02 13:05:19 2025-07-02 13:05:19.732000 \n",
|
||
"3 2025-07-02 13:05:24 2025-07-02 13:05:24.728000 \n",
|
||
"4 2025-07-02 13:05:24 2025-07-02 13:05:24.730000 \n",
|
||
".. ... ... \n",
|
||
"373 2025-07-02 13:15:40 2025-07-02 13:15:40.230000 \n",
|
||
"374 2025-07-02 13:15:40 2025-07-02 13:15:40.232000 \n",
|
||
"375 2025-07-02 13:15:45 2025-07-02 13:15:45.228000 \n",
|
||
"376 2025-07-02 13:15:45 2025-07-02 13:15:45.231000 \n",
|
||
"377 2025-07-02 13:15:45 2025-07-02 13:15:45.233000 \n",
|
||
"\n",
|
||
"[378 rows x 6 columns]"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
}
|
||
],
|
||
"source": [
|
||
"from sqlalchemy import create_engine, MetaData, Table, select, func, text\n",
|
||
"import pandas as pd\n",
|
||
"from datetime import datetime\n",
|
||
"\n",
|
||
"engine = create_engine('druid://localhost:8082/druid/v2/sql/')\n",
|
||
"metadata = MetaData()\n",
|
||
"places = Table('raw_scouter-opcua-orchestrated-pipeline', metadata, autoload_with=engine)\n",
|
||
"date_str = '2025-01-01'\n",
|
||
"stmt = select(places).where(text(f'\"__time\" > TIMESTAMP \\'{date_str}\\''))\n",
|
||
"\n",
|
||
"result = pd.read_sql(stmt, engine)\n",
|
||
"\n",
|
||
"result[\"inserted_at\"] = pd.to_datetime(result[\"__time\"]).dt.strftime(\n",
|
||
" \"%Y-%m-%d %H:%M:%S.%f\")\n",
|
||
"\n",
|
||
"result.drop(columns=[\"__time\"], inplace=True)\n",
|
||
"\n",
|
||
"display(result)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"data = {\n",
|
||
" \"id\": \"1\",\n",
|
||
" \"schedule_name\": \"scouter-opcua-orchestrated-pipeline\",\n",
|
||
" \"model_id\": \"1\",\n",
|
||
" \"workflow_type\": \"scouter\",\n",
|
||
" \"frequency\": \"30s\",\n",
|
||
" \"max_retry_policy\": 1,\n",
|
||
" \"read_tags\": [\n",
|
||
" {\n",
|
||
" \"tag_name\": \"Counter\",\n",
|
||
" \"server_id\": \"1\",\n",
|
||
" \"aggr_func\": \"avg\",\n",
|
||
" \"tag_address\": \"ns=2;i=2\",\n",
|
||
" \"frequency\": \"15000\",\n",
|
||
" \"data_range\": [\n",
|
||
" -100,\n",
|
||
" 100\n",
|
||
" ]\n",
|
||
" },\n",
|
||
" {\n",
|
||
" \"tag_name\": \"Rollout\",\n",
|
||
" \"server_id\": \"1\",\n",
|
||
" \"aggr_func\": \"mdn\",\n",
|
||
" \"tag_address\": \"ns=2;i=3\",\n",
|
||
" \"frequency\": \"15000\",\n",
|
||
" \"data_range\": [\n",
|
||
" -100,\n",
|
||
" 100\n",
|
||
" ]\n",
|
||
" },\n",
|
||
" {\n",
|
||
" \"tag_name\": \"Square\",\n",
|
||
" \"server_id\": \"1\",\n",
|
||
" \"aggr_func\": \"lts\",\n",
|
||
" \"tag_address\": \"ns=2;i=4\",\n",
|
||
" \"frequency\": \"15000\",\n",
|
||
" \"data_range\": [\n",
|
||
" -100,\n",
|
||
" 100\n",
|
||
" ]\n",
|
||
" }\n",
|
||
" ],\n",
|
||
" \"filters\": [\n",
|
||
" {\n",
|
||
" \"filter_name\": \"OUT_OF_BOUNDS_FILTER\",\n",
|
||
" \"policy\": \"DISCARD\"\n",
|
||
" },\n",
|
||
" {\n",
|
||
" \"filter_name\": \"NULL_VALUES_FILTER\",\n",
|
||
" \"policy\": \"DISCARD\"\n",
|
||
" }\n",
|
||
" ],\n",
|
||
" \"tag_retention_minutes\": 60\n",
|
||
"}"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import json\n",
|
||
"\n",
|
||
"\n",
|
||
"# Generate 29 more, changing only id and schedule_name\n",
|
||
"json_list = []\n",
|
||
"for i in range(30):\n",
|
||
" obj = data.copy()\n",
|
||
" obj['id'] = i + 1 # or any other unique id logic\n",
|
||
" obj['schedule_name'] = f\"scouter-opcua-pipeline-{i+1}\"\n",
|
||
" json_list.append(obj)\n",
|
||
"\n",
|
||
"# Save to a new file\n",
|
||
"with open('input_samples_30.json', 'w') as f:\n",
|
||
" json.dump(json_list, f, indent=2)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 9,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"spec = {\n",
|
||
" \"type\": \"kafka\",\n",
|
||
" \"spec\": {\n",
|
||
" \"dataSchema\": {\n",
|
||
" \"dataSource\": \"raw_scouter-opcua-orchestrated-pipeline\",\n",
|
||
" \"timestampSpec\": {\n",
|
||
" \"column\": \"kafka.timestamp\",\n",
|
||
" \"format\": \"millis\",\n",
|
||
" \"missingValue\": None\n",
|
||
" },\n",
|
||
" \"dimensionsSpec\": {\n",
|
||
" \"dimensions\": [],\n",
|
||
" \"dimensionExclusions\": [\n",
|
||
" \"__time\",\n",
|
||
" \"kafka.timestamp\"\n",
|
||
" ],\n",
|
||
" \"includeAllDimensions\": False,\n",
|
||
" \"useSchemaDiscovery\": True\n",
|
||
" },\n",
|
||
" \"metricsSpec\": [],\n",
|
||
" \"granularitySpec\": {\n",
|
||
" \"type\": \"uniform\",\n",
|
||
" \"segmentGranularity\": \"DAY\",\n",
|
||
" \"queryGranularity\": {\n",
|
||
" \"type\": \"none\"\n",
|
||
" },\n",
|
||
" \"rollup\": False,\n",
|
||
" \"intervals\": []\n",
|
||
" },\n",
|
||
" \"transformSpec\": {\n",
|
||
" \"filter\": None,\n",
|
||
" \"transforms\": []\n",
|
||
" }\n",
|
||
" },\n",
|
||
" \"ioConfig\": {\n",
|
||
" \"topic\": \"raw_scouter-opcua-orchestrated-pipeline\",\n",
|
||
" \"topicPattern\": None,\n",
|
||
" \"inputFormat\": {\n",
|
||
" \"type\": \"kafka\",\n",
|
||
" \"headerFormat\": None,\n",
|
||
" \"keyFormat\": None,\n",
|
||
" \"valueFormat\": {\n",
|
||
" \"type\": \"json\",\n",
|
||
" \"keepNoneColumns\": False,\n",
|
||
" \"assumeNewlineDelimited\": False,\n",
|
||
" \"useJsonNodeReader\": False\n",
|
||
" },\n",
|
||
" \"headerColumnPrefix\": \"kafka.header.\",\n",
|
||
" \"keyColumnName\": \"kafka.key\",\n",
|
||
" \"timestampColumnName\": \"kafka.timestamp\",\n",
|
||
" \"topicColumnName\": \"kafka.topic\"\n",
|
||
" },\n",
|
||
" \"replicas\": 1,\n",
|
||
" \"taskCount\": 1,\n",
|
||
" \"taskDuration\": \"PT3600S\",\n",
|
||
" \"consumerProperties\": {\n",
|
||
" \"bootstrap.servers\": \"kafka.kafka.svc.cluster.local:9092\"\n",
|
||
" },\n",
|
||
" \"autoScalerConfig\": None,\n",
|
||
" \"pollTimeout\": 100,\n",
|
||
" \"startDelay\": \"PT5S\",\n",
|
||
" \"period\": \"PT30S\",\n",
|
||
" \"useEarliestOffset\": True,\n",
|
||
" \"completionTimeout\": \"PT1800S\",\n",
|
||
" \"lateMessageRejectionPeriod\": None,\n",
|
||
" \"earlyMessageRejectionPeriod\": None,\n",
|
||
" \"lateMessageRejectionStartDateTime\": None,\n",
|
||
" \"configOverrides\": None,\n",
|
||
" \"idleConfig\": None,\n",
|
||
" \"stopTaskCount\": None,\n",
|
||
" \"stream\": \"raw_scouter-opcua-orchestrated-pipeline\",\n",
|
||
" \"useEarliestSequenceNumber\": True\n",
|
||
" },\n",
|
||
" \"tuningConfig\": {\n",
|
||
" \"type\": \"kafka\",\n",
|
||
" \"appendableIndexSpec\": {\n",
|
||
" \"type\": \"onheap\",\n",
|
||
" \"preserveExistingMetrics\": False\n",
|
||
" },\n",
|
||
" \"maxRowsInMemory\": 150000,\n",
|
||
" \"maxBytesInMemory\": 0,\n",
|
||
" \"skipBytesInMemoryOverheadCheck\": False,\n",
|
||
" \"maxRowsPerSegment\": 5000000,\n",
|
||
" \"maxTotalRows\": None,\n",
|
||
" \"intermediatePersistPeriod\": \"PT10M\",\n",
|
||
" \"maxPendingPersists\": 0,\n",
|
||
" \"indexSpec\": {\n",
|
||
" \"bitmap\": {\n",
|
||
" \"type\": \"roaring\"\n",
|
||
" },\n",
|
||
" \"dimensionCompression\": \"lz4\",\n",
|
||
" \"stringDictionaryEncoding\": {\n",
|
||
" \"type\": \"utf8\"\n",
|
||
" },\n",
|
||
" \"metricCompression\": \"lz4\",\n",
|
||
" \"longEncoding\": \"longs\"\n",
|
||
" },\n",
|
||
" \"indexSpecForIntermediatePersists\": {\n",
|
||
" \"bitmap\": {\n",
|
||
" \"type\": \"roaring\"\n",
|
||
" },\n",
|
||
" \"dimensionCompression\": \"lz4\",\n",
|
||
" \"stringDictionaryEncoding\": {\n",
|
||
" \"type\": \"utf8\"\n",
|
||
" },\n",
|
||
" \"metricCompression\": \"lz4\",\n",
|
||
" \"longEncoding\": \"longs\"\n",
|
||
" },\n",
|
||
" \"reportParseExceptions\": False,\n",
|
||
" \"handoffConditionTimeout\": 900000,\n",
|
||
" \"resetOffsetAutomatically\": False,\n",
|
||
" \"segmentWriteOutMediumFactory\": None,\n",
|
||
" \"workerThreads\": None,\n",
|
||
" \"chatRetries\": 8,\n",
|
||
" \"httpTimeout\": \"PT10S\",\n",
|
||
" \"shutdownTimeout\": \"PT80S\",\n",
|
||
" \"offsetFetchPeriod\": \"PT30S\",\n",
|
||
" \"intermediateHandoffPeriod\": \"P2147483647D\",\n",
|
||
" \"logParseExceptions\": False,\n",
|
||
" \"maxParseExceptions\": 2147483647,\n",
|
||
" \"maxSavedParseExceptions\": 0,\n",
|
||
" \"numPersistThreads\": 1,\n",
|
||
" \"skipSequenceNumberAvailabilityCheck\": False,\n",
|
||
" \"repartitionTransitionDuration\": \"PT120S\"\n",
|
||
" }\n",
|
||
" },\n",
|
||
" \"context\": None,\n",
|
||
" \"suspended\": False\n",
|
||
"}"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 18,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"raw_scouter-opcua-pipeline-2\n",
|
||
"raw_scouter-opcua-pipeline-3\n",
|
||
"raw_scouter-opcua-pipeline-4\n",
|
||
"raw_scouter-opcua-pipeline-5\n",
|
||
"raw_scouter-opcua-pipeline-6\n",
|
||
"raw_scouter-opcua-pipeline-7\n",
|
||
"raw_scouter-opcua-pipeline-8\n",
|
||
"raw_scouter-opcua-pipeline-9\n",
|
||
"raw_scouter-opcua-pipeline-10\n",
|
||
"raw_scouter-opcua-pipeline-11\n",
|
||
"raw_scouter-opcua-pipeline-12\n",
|
||
"raw_scouter-opcua-pipeline-13\n",
|
||
"raw_scouter-opcua-pipeline-14\n",
|
||
"raw_scouter-opcua-pipeline-15\n",
|
||
"raw_scouter-opcua-pipeline-16\n",
|
||
"raw_scouter-opcua-pipeline-17\n",
|
||
"raw_scouter-opcua-pipeline-18\n",
|
||
"raw_scouter-opcua-pipeline-19\n",
|
||
"raw_scouter-opcua-pipeline-20\n",
|
||
"raw_scouter-opcua-pipeline-21\n",
|
||
"raw_scouter-opcua-pipeline-22\n",
|
||
"raw_scouter-opcua-pipeline-23\n",
|
||
"raw_scouter-opcua-pipeline-24\n",
|
||
"raw_scouter-opcua-pipeline-25\n",
|
||
"raw_scouter-opcua-pipeline-26\n",
|
||
"raw_scouter-opcua-pipeline-27\n",
|
||
"raw_scouter-opcua-pipeline-28\n",
|
||
"raw_scouter-opcua-pipeline-29\n",
|
||
"raw_scouter-opcua-pipeline-30\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"import json\n",
|
||
"from copy import deepcopy\n",
|
||
"\n",
|
||
"json_list = []\n",
|
||
"i = 0\n",
|
||
"for i in range(1, 30):\n",
|
||
" topic = f\"raw_scouter-opcua-pipeline-{i+1}\"\n",
|
||
" print(topic)\n",
|
||
" obj = deepcopy(spec)\n",
|
||
" obj['spec']['dataSchema']['dataSource'] = topic\n",
|
||
" obj['spec']['ioConfig']['topic'] = topic\n",
|
||
" obj['spec']['ioConfig']['stream'] = topic\n",
|
||
"\n",
|
||
" json_list.append(obj)\n",
|
||
"\n",
|
||
"# Save to a new file\n",
|
||
"with open('specs_30.json', 'w') as f:\n",
|
||
" json.dump(json_list, f, indent=2)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 19,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[OK] raw_scouter-opcua-pipeline-2 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-3 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-4 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-5 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-6 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-7 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-8 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-9 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-10 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-11 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-12 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-13 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-14 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-15 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-16 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-17 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-18 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-19 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-20 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-21 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-22 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-23 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-24 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-25 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-26 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-27 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-28 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-29 enviado.\n",
|
||
"[OK] raw_scouter-opcua-pipeline-30 enviado.\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"import os\n",
|
||
"import requests\n",
|
||
"\n",
|
||
"DRUID_OVERLORD = os.getenv(\"DRUID_OVERLORD\", \"http://localhost:8082\")\n",
|
||
"SUPERVISOR_ENDPOINT = f\"{DRUID_OVERLORD}/druid/indexer/v1/supervisor\"\n",
|
||
"\n",
|
||
"def enviar_supervisores(specs):\n",
|
||
" for spec in specs:\n",
|
||
" resp = requests.post(\n",
|
||
" SUPERVISOR_ENDPOINT,\n",
|
||
" headers={\"Content-Type\": \"application/json\"},\n",
|
||
" json=spec\n",
|
||
" )\n",
|
||
" if resp.status_code == 200:\n",
|
||
" print(f\"[OK] {spec['spec']['dataSchema']['dataSource']} enviado.\")\n",
|
||
" else:\n",
|
||
" print(f\"[ERRO] {spec['spec']['dataSchema']['dataSource']}: {resp.status_code} → {resp.text}\")\n",
|
||
"\n",
|
||
"with open(\"./specs_30.json\", \"r\") as f:\n",
|
||
" specs = json.load(f)\n",
|
||
"enviar_supervisores(specs)\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 20,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"ename": "ConnectionError",
|
||
"evalue": "HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /druid/indexer/v1/supervisor/raw_scouter-opcua-orchestrated-pipeline-2/terminate (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x72128094cc50>: Failed to establish a new connection: [Errno 111] Connection refused'))",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
||
"\u001b[31mConnectionRefusedError\u001b[39m Traceback (most recent call last)",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:198\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 197\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m198\u001b[39m sock = \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 199\u001b[39m \u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 200\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 201\u001b[39m \u001b[43m \u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 202\u001b[39m \u001b[43m \u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 203\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 204\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.gaierror \u001b[38;5;28;01mas\u001b[39;00m e:\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 84\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m85\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[32m 86\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 87\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/util/connection.py:73\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 72\u001b[39m sock.bind(source_address)\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[43msock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 74\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n",
|
||
"\u001b[31mConnectionRefusedError\u001b[39m: [Errno 111] Connection refused",
|
||
"\nThe above exception was the direct cause of the following exception:\n",
|
||
"\u001b[31mNewConnectionError\u001b[39m Traceback (most recent call last)",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connectionpool.py:787\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 786\u001b[39m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m787\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 788\u001b[39m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 789\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 790\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 791\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 792\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 793\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 794\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 795\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 796\u001b[39m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 797\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 798\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 799\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 800\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 802\u001b[39m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connectionpool.py:493\u001b[39m, in \u001b[36mHTTPConnectionPool._make_request\u001b[39m\u001b[34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 492\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m493\u001b[39m \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 494\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 495\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 497\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 498\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 499\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 500\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 501\u001b[39m \u001b[43m \u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m=\u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 502\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 504\u001b[39m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[32m 505\u001b[39m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[32m 506\u001b[39m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:494\u001b[39m, in \u001b[36mHTTPConnection.request\u001b[39m\u001b[34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 493\u001b[39m \u001b[38;5;28mself\u001b[39m.putheader(header, value)\n\u001b[32m--> \u001b[39m\u001b[32m494\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[38;5;66;03m# If we're given a body we start sending that in chunks.\u001b[39;00m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.11/http/client.py:1298\u001b[39m, in \u001b[36mHTTPConnection.endheaders\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1297\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[32m-> \u001b[39m\u001b[32m1298\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.11/http/client.py:1058\u001b[39m, in \u001b[36mHTTPConnection._send_output\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1057\u001b[39m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m._buffer[:]\n\u001b[32m-> \u001b[39m\u001b[32m1058\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1060\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1061\u001b[39m \n\u001b[32m 1062\u001b[39m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.11/http/client.py:996\u001b[39m, in \u001b[36mHTTPConnection.send\u001b[39m\u001b[34m(self, data)\u001b[39m\n\u001b[32m 995\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.auto_open:\n\u001b[32m--> \u001b[39m\u001b[32m996\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 997\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:325\u001b[39m, in \u001b[36mHTTPConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 324\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m325\u001b[39m \u001b[38;5;28mself\u001b[39m.sock = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 326\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._tunnel_host:\n\u001b[32m 327\u001b[39m \u001b[38;5;66;03m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:213\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 212\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m--> \u001b[39m\u001b[32m213\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[32m 214\u001b[39m \u001b[38;5;28mself\u001b[39m, \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 215\u001b[39m ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m 217\u001b[39m sys.audit(\u001b[33m\"\u001b[39m\u001b[33mhttp.client.connect\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mself\u001b[39m.host, \u001b[38;5;28mself\u001b[39m.port)\n",
|
||
"\u001b[31mNewConnectionError\u001b[39m: <urllib3.connection.HTTPConnection object at 0x72128094cc50>: Failed to establish a new connection: [Errno 111] Connection refused",
|
||
"\nThe above exception was the direct cause of the following exception:\n",
|
||
"\u001b[31mMaxRetryError\u001b[39m Traceback (most recent call last)",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/adapters.py:667\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 666\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m667\u001b[39m resp = \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 668\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 669\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 670\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 671\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 672\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 673\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 674\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 675\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 676\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 677\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 678\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 679\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 681\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connectionpool.py:841\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 839\u001b[39m new_e = ProtocolError(\u001b[33m\"\u001b[39m\u001b[33mConnection aborted.\u001b[39m\u001b[33m\"\u001b[39m, new_e)\n\u001b[32m--> \u001b[39m\u001b[32m841\u001b[39m retries = \u001b[43mretries\u001b[49m\u001b[43m.\u001b[49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 842\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m=\u001b[49m\u001b[43msys\u001b[49m\u001b[43m.\u001b[49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[32m 843\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 844\u001b[39m retries.sleep()\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/util/retry.py:519\u001b[39m, in \u001b[36mRetry.increment\u001b[39m\u001b[34m(self, method, url, response, error, _pool, _stacktrace)\u001b[39m\n\u001b[32m 518\u001b[39m reason = error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[32m--> \u001b[39m\u001b[32m519\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mreason\u001b[39;00m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[32m 521\u001b[39m log.debug(\u001b[33m\"\u001b[39m\u001b[33mIncremented Retry for (url=\u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m\"\u001b[39m, url, new_retry)\n",
|
||
"\u001b[31mMaxRetryError\u001b[39m: HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /druid/indexer/v1/supervisor/raw_scouter-opcua-orchestrated-pipeline-2/terminate (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x72128094cc50>: Failed to establish a new connection: [Errno 111] Connection refused'))",
|
||
"\nDuring handling of the above exception, another exception occurred:\n",
|
||
"\u001b[31mConnectionError\u001b[39m Traceback (most recent call last)",
|
||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[20]\u001b[39m\u001b[32m, line 16\u001b[39m\n\u001b[32m 13\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m❌ Erro \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresp.status_code\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresp.text\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m 15\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[32m1\u001b[39m, \u001b[32m30\u001b[39m):\n\u001b[32m---> \u001b[39m\u001b[32m16\u001b[39m \u001b[43mterminate_supervisor\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43mf\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mraw_scouter-opcua-orchestrated-pipeline-\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mi\u001b[49m\u001b[43m+\u001b[49m\u001b[32;43m1\u001b[39;49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
|
||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[20]\u001b[39m\u001b[32m, line 7\u001b[39m, in \u001b[36mterminate_supervisor\u001b[39m\u001b[34m(supervisor_id)\u001b[39m\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mterminate_supervisor\u001b[39m(supervisor_id):\n\u001b[32m 6\u001b[39m url = \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mDRUID\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/druid/indexer/v1/supervisor/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msupervisor_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/terminate\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m7\u001b[39m resp = \u001b[43mrequests\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 8\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m resp.status_code == \u001b[32m200\u001b[39m:\n\u001b[32m 9\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m✅ Supervisor \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msupervisor_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m encerrado.\u001b[39m\u001b[33m\"\u001b[39m)\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/api.py:115\u001b[39m, in \u001b[36mpost\u001b[39m\u001b[34m(url, data, json, **kwargs)\u001b[39m\n\u001b[32m 103\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpost\u001b[39m(url, data=\u001b[38;5;28;01mNone\u001b[39;00m, json=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m 104\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33mr\u001b[39m\u001b[33;03m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[32m 105\u001b[39m \n\u001b[32m 106\u001b[39m \u001b[33;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 112\u001b[39m \u001b[33;03m :rtype: requests.Response\u001b[39;00m\n\u001b[32m 113\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m115\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpost\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[43m=\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/api.py:59\u001b[39m, in \u001b[36mrequest\u001b[39m\u001b[34m(method, url, **kwargs)\u001b[39m\n\u001b[32m 55\u001b[39m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[32m 56\u001b[39m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[32m 57\u001b[39m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[32m 58\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m sessions.Session() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[32m---> \u001b[39m\u001b[32m59\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/sessions.py:589\u001b[39m, in \u001b[36mSession.request\u001b[39m\u001b[34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[39m\n\u001b[32m 584\u001b[39m send_kwargs = {\n\u001b[32m 585\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mtimeout\u001b[39m\u001b[33m\"\u001b[39m: timeout,\n\u001b[32m 586\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mallow_redirects\u001b[39m\u001b[33m\"\u001b[39m: allow_redirects,\n\u001b[32m 587\u001b[39m }\n\u001b[32m 588\u001b[39m send_kwargs.update(settings)\n\u001b[32m--> \u001b[39m\u001b[32m589\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 591\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/sessions.py:703\u001b[39m, in \u001b[36mSession.send\u001b[39m\u001b[34m(self, request, **kwargs)\u001b[39m\n\u001b[32m 700\u001b[39m start = preferred_clock()\n\u001b[32m 702\u001b[39m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m703\u001b[39m r = \u001b[43madapter\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 705\u001b[39m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[32m 706\u001b[39m elapsed = preferred_clock() - start\n",
|
||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/adapters.py:700\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 696\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e.reason, _SSLError):\n\u001b[32m 697\u001b[39m \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[32m 698\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request=request)\n\u001b[32m--> \u001b[39m\u001b[32m700\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n\u001b[32m 702\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 703\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n",
|
||
"\u001b[31mConnectionError\u001b[39m: HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /druid/indexer/v1/supervisor/raw_scouter-opcua-orchestrated-pipeline-2/terminate (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x72128094cc50>: Failed to establish a new connection: [Errno 111] Connection refused'))"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"import os\n",
|
||
"import requests\n",
|
||
"\n",
|
||
"DRUID = os.getenv(\"DRUID_URL\", \"http://localhost:8082\")\n",
|
||
"def terminate_supervisor(supervisor_id):\n",
|
||
" url = f\"{DRUID}/druid/indexer/v1/supervisor/{supervisor_id}/terminate\"\n",
|
||
" resp = requests.post(url)\n",
|
||
" if resp.status_code == 200:\n",
|
||
" print(f\"✅ Supervisor '{supervisor_id}' encerrado.\")\n",
|
||
" elif resp.status_code == 404:\n",
|
||
" print(f\"⚠️ Supervisor '{supervisor_id}' não encontrado.\")\n",
|
||
" else:\n",
|
||
" print(f\"❌ Erro {resp.status_code}: {resp.text}\")\n",
|
||
"\n",
|
||
"for i in range(1, 30):\n",
|
||
" terminate_supervisor(f\"raw_scouter-opcua-orchestrated-pipeline-{i+1}\")\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"✅ Document deleted successfully\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"from pymongo import MongoClient\n",
|
||
"import os\n",
|
||
"# Get MongoDB connection details from environment variables\n",
|
||
"MONGODB_USERNAME = os.getenv(\"MONGODB_USERNAME\", \"root\")\n",
|
||
"MONGODB_PASSWORD = os.getenv(\"MONGODB_PASSWORD\", \"wKZDbMNU1c\") \n",
|
||
"MONGODB_URL = os.getenv(\"MONGODB_URL\", \"localhost:27018\")\n",
|
||
"MONGODB_DATABASE = os.getenv(\"MONGODB_DATABASE\", \"sientia\")\n",
|
||
"\n",
|
||
"# Create MongoDB client\n",
|
||
"client = MongoClient(\n",
|
||
" f\"mongodb://{MONGODB_USERNAME}:{MONGODB_PASSWORD}@{MONGODB_URL}\"\n",
|
||
")\n",
|
||
"\n",
|
||
"# Get database and collection\n",
|
||
"db = client[MONGODB_DATABASE]\n",
|
||
"collection = db[\"pipelines\"] # Replace with actual collection name\n",
|
||
"\n",
|
||
"for i in range(2, 31):\n",
|
||
" # Delete a document matching specific criteria\n",
|
||
" result = collection.delete_one({\"schedule_name\": f\"scouter-opcua-pipeline-{i}\"}) # Replace with actual query\n",
|
||
"\n",
|
||
"if result.deleted_count > 0:\n",
|
||
" print(\"✅ Document deleted successfully\")\n",
|
||
"else:\n",
|
||
" print(\"⚠️ No matching document found\")\n",
|
||
"\n",
|
||
"# Close the connection\n",
|
||
"client.close()\n"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "venv",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.11.13"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 2
|
||
}
|