SIENTIAPDE-1110
Update test.ipynb to generate and save multiple pipeline specifications and input samples in JSON format. Modify values.yaml to increase replica count from 1 to 3 and adjust PostgreSQL max connections from 20 to 40 for improved scalability.
This commit is contained in:
vitor-aignosi
457
test.ipynb
457
test.ipynb
@@ -241,6 +241,463 @@
|
||||
"\n",
|
||||
"display(result)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"data = {\n",
|
||||
" \"id\": \"1\",\n",
|
||||
" \"schedule_name\": \"scouter-opcua-orchestrated-pipeline\",\n",
|
||||
" \"model_id\": \"1\",\n",
|
||||
" \"workflow_type\": \"scouter\",\n",
|
||||
" \"frequency\": \"30s\",\n",
|
||||
" \"max_retry_policy\": 1,\n",
|
||||
" \"read_tags\": [\n",
|
||||
" {\n",
|
||||
" \"tag_name\": \"Counter\",\n",
|
||||
" \"server_id\": \"1\",\n",
|
||||
" \"aggr_func\": \"avg\",\n",
|
||||
" \"tag_address\": \"ns=2;i=2\",\n",
|
||||
" \"frequency\": \"15000\",\n",
|
||||
" \"data_range\": [\n",
|
||||
" -100,\n",
|
||||
" 100\n",
|
||||
" ]\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"tag_name\": \"Rollout\",\n",
|
||||
" \"server_id\": \"1\",\n",
|
||||
" \"aggr_func\": \"mdn\",\n",
|
||||
" \"tag_address\": \"ns=2;i=3\",\n",
|
||||
" \"frequency\": \"15000\",\n",
|
||||
" \"data_range\": [\n",
|
||||
" -100,\n",
|
||||
" 100\n",
|
||||
" ]\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"tag_name\": \"Square\",\n",
|
||||
" \"server_id\": \"1\",\n",
|
||||
" \"aggr_func\": \"lts\",\n",
|
||||
" \"tag_address\": \"ns=2;i=4\",\n",
|
||||
" \"frequency\": \"15000\",\n",
|
||||
" \"data_range\": [\n",
|
||||
" -100,\n",
|
||||
" 100\n",
|
||||
" ]\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" \"filters\": [\n",
|
||||
" {\n",
|
||||
" \"filter_name\": \"OUT_OF_BOUNDS_FILTER\",\n",
|
||||
" \"policy\": \"DISCARD\"\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"filter_name\": \"NULL_VALUES_FILTER\",\n",
|
||||
" \"policy\": \"DISCARD\"\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" \"tag_retention_minutes\": 60\n",
|
||||
"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate 29 more, changing only id and schedule_name\n",
|
||||
"json_list = []\n",
|
||||
"for i in range(30):\n",
|
||||
" obj = data.copy()\n",
|
||||
" obj['id'] = i + 1 # or any other unique id logic\n",
|
||||
" obj['schedule_name'] = f\"scouter-opcua-pipeline-{i+1}\"\n",
|
||||
" json_list.append(obj)\n",
|
||||
"\n",
|
||||
"# Save to a new file\n",
|
||||
"with open('input_samples_30.json', 'w') as f:\n",
|
||||
" json.dump(json_list, f, indent=2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"spec = {\n",
|
||||
" \"type\": \"kafka\",\n",
|
||||
" \"spec\": {\n",
|
||||
" \"dataSchema\": {\n",
|
||||
" \"dataSource\": \"raw_scouter-opcua-orchestrated-pipeline\",\n",
|
||||
" \"timestampSpec\": {\n",
|
||||
" \"column\": \"kafka.timestamp\",\n",
|
||||
" \"format\": \"millis\",\n",
|
||||
" \"missingValue\": None\n",
|
||||
" },\n",
|
||||
" \"dimensionsSpec\": {\n",
|
||||
" \"dimensions\": [],\n",
|
||||
" \"dimensionExclusions\": [\n",
|
||||
" \"__time\",\n",
|
||||
" \"kafka.timestamp\"\n",
|
||||
" ],\n",
|
||||
" \"includeAllDimensions\": False,\n",
|
||||
" \"useSchemaDiscovery\": True\n",
|
||||
" },\n",
|
||||
" \"metricsSpec\": [],\n",
|
||||
" \"granularitySpec\": {\n",
|
||||
" \"type\": \"uniform\",\n",
|
||||
" \"segmentGranularity\": \"DAY\",\n",
|
||||
" \"queryGranularity\": {\n",
|
||||
" \"type\": \"none\"\n",
|
||||
" },\n",
|
||||
" \"rollup\": False,\n",
|
||||
" \"intervals\": []\n",
|
||||
" },\n",
|
||||
" \"transformSpec\": {\n",
|
||||
" \"filter\": None,\n",
|
||||
" \"transforms\": []\n",
|
||||
" }\n",
|
||||
" },\n",
|
||||
" \"ioConfig\": {\n",
|
||||
" \"topic\": \"raw_scouter-opcua-orchestrated-pipeline\",\n",
|
||||
" \"topicPattern\": None,\n",
|
||||
" \"inputFormat\": {\n",
|
||||
" \"type\": \"kafka\",\n",
|
||||
" \"headerFormat\": None,\n",
|
||||
" \"keyFormat\": None,\n",
|
||||
" \"valueFormat\": {\n",
|
||||
" \"type\": \"json\",\n",
|
||||
" \"keepNoneColumns\": False,\n",
|
||||
" \"assumeNewlineDelimited\": False,\n",
|
||||
" \"useJsonNodeReader\": False\n",
|
||||
" },\n",
|
||||
" \"headerColumnPrefix\": \"kafka.header.\",\n",
|
||||
" \"keyColumnName\": \"kafka.key\",\n",
|
||||
" \"timestampColumnName\": \"kafka.timestamp\",\n",
|
||||
" \"topicColumnName\": \"kafka.topic\"\n",
|
||||
" },\n",
|
||||
" \"replicas\": 1,\n",
|
||||
" \"taskCount\": 1,\n",
|
||||
" \"taskDuration\": \"PT3600S\",\n",
|
||||
" \"consumerProperties\": {\n",
|
||||
" \"bootstrap.servers\": \"kafka.kafka.svc.cluster.local:9092\"\n",
|
||||
" },\n",
|
||||
" \"autoScalerConfig\": None,\n",
|
||||
" \"pollTimeout\": 100,\n",
|
||||
" \"startDelay\": \"PT5S\",\n",
|
||||
" \"period\": \"PT30S\",\n",
|
||||
" \"useEarliestOffset\": True,\n",
|
||||
" \"completionTimeout\": \"PT1800S\",\n",
|
||||
" \"lateMessageRejectionPeriod\": None,\n",
|
||||
" \"earlyMessageRejectionPeriod\": None,\n",
|
||||
" \"lateMessageRejectionStartDateTime\": None,\n",
|
||||
" \"configOverrides\": None,\n",
|
||||
" \"idleConfig\": None,\n",
|
||||
" \"stopTaskCount\": None,\n",
|
||||
" \"stream\": \"raw_scouter-opcua-orchestrated-pipeline\",\n",
|
||||
" \"useEarliestSequenceNumber\": True\n",
|
||||
" },\n",
|
||||
" \"tuningConfig\": {\n",
|
||||
" \"type\": \"kafka\",\n",
|
||||
" \"appendableIndexSpec\": {\n",
|
||||
" \"type\": \"onheap\",\n",
|
||||
" \"preserveExistingMetrics\": False\n",
|
||||
" },\n",
|
||||
" \"maxRowsInMemory\": 150000,\n",
|
||||
" \"maxBytesInMemory\": 0,\n",
|
||||
" \"skipBytesInMemoryOverheadCheck\": False,\n",
|
||||
" \"maxRowsPerSegment\": 5000000,\n",
|
||||
" \"maxTotalRows\": None,\n",
|
||||
" \"intermediatePersistPeriod\": \"PT10M\",\n",
|
||||
" \"maxPendingPersists\": 0,\n",
|
||||
" \"indexSpec\": {\n",
|
||||
" \"bitmap\": {\n",
|
||||
" \"type\": \"roaring\"\n",
|
||||
" },\n",
|
||||
" \"dimensionCompression\": \"lz4\",\n",
|
||||
" \"stringDictionaryEncoding\": {\n",
|
||||
" \"type\": \"utf8\"\n",
|
||||
" },\n",
|
||||
" \"metricCompression\": \"lz4\",\n",
|
||||
" \"longEncoding\": \"longs\"\n",
|
||||
" },\n",
|
||||
" \"indexSpecForIntermediatePersists\": {\n",
|
||||
" \"bitmap\": {\n",
|
||||
" \"type\": \"roaring\"\n",
|
||||
" },\n",
|
||||
" \"dimensionCompression\": \"lz4\",\n",
|
||||
" \"stringDictionaryEncoding\": {\n",
|
||||
" \"type\": \"utf8\"\n",
|
||||
" },\n",
|
||||
" \"metricCompression\": \"lz4\",\n",
|
||||
" \"longEncoding\": \"longs\"\n",
|
||||
" },\n",
|
||||
" \"reportParseExceptions\": False,\n",
|
||||
" \"handoffConditionTimeout\": 900000,\n",
|
||||
" \"resetOffsetAutomatically\": False,\n",
|
||||
" \"segmentWriteOutMediumFactory\": None,\n",
|
||||
" \"workerThreads\": None,\n",
|
||||
" \"chatRetries\": 8,\n",
|
||||
" \"httpTimeout\": \"PT10S\",\n",
|
||||
" \"shutdownTimeout\": \"PT80S\",\n",
|
||||
" \"offsetFetchPeriod\": \"PT30S\",\n",
|
||||
" \"intermediateHandoffPeriod\": \"P2147483647D\",\n",
|
||||
" \"logParseExceptions\": False,\n",
|
||||
" \"maxParseExceptions\": 2147483647,\n",
|
||||
" \"maxSavedParseExceptions\": 0,\n",
|
||||
" \"numPersistThreads\": 1,\n",
|
||||
" \"skipSequenceNumberAvailabilityCheck\": False,\n",
|
||||
" \"repartitionTransitionDuration\": \"PT120S\"\n",
|
||||
" }\n",
|
||||
" },\n",
|
||||
" \"context\": None,\n",
|
||||
" \"suspended\": False\n",
|
||||
"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"raw_scouter-opcua-pipeline-2\n",
|
||||
"raw_scouter-opcua-pipeline-3\n",
|
||||
"raw_scouter-opcua-pipeline-4\n",
|
||||
"raw_scouter-opcua-pipeline-5\n",
|
||||
"raw_scouter-opcua-pipeline-6\n",
|
||||
"raw_scouter-opcua-pipeline-7\n",
|
||||
"raw_scouter-opcua-pipeline-8\n",
|
||||
"raw_scouter-opcua-pipeline-9\n",
|
||||
"raw_scouter-opcua-pipeline-10\n",
|
||||
"raw_scouter-opcua-pipeline-11\n",
|
||||
"raw_scouter-opcua-pipeline-12\n",
|
||||
"raw_scouter-opcua-pipeline-13\n",
|
||||
"raw_scouter-opcua-pipeline-14\n",
|
||||
"raw_scouter-opcua-pipeline-15\n",
|
||||
"raw_scouter-opcua-pipeline-16\n",
|
||||
"raw_scouter-opcua-pipeline-17\n",
|
||||
"raw_scouter-opcua-pipeline-18\n",
|
||||
"raw_scouter-opcua-pipeline-19\n",
|
||||
"raw_scouter-opcua-pipeline-20\n",
|
||||
"raw_scouter-opcua-pipeline-21\n",
|
||||
"raw_scouter-opcua-pipeline-22\n",
|
||||
"raw_scouter-opcua-pipeline-23\n",
|
||||
"raw_scouter-opcua-pipeline-24\n",
|
||||
"raw_scouter-opcua-pipeline-25\n",
|
||||
"raw_scouter-opcua-pipeline-26\n",
|
||||
"raw_scouter-opcua-pipeline-27\n",
|
||||
"raw_scouter-opcua-pipeline-28\n",
|
||||
"raw_scouter-opcua-pipeline-29\n",
|
||||
"raw_scouter-opcua-pipeline-30\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"from copy import deepcopy\n",
|
||||
"\n",
|
||||
"json_list = []\n",
|
||||
"i = 0\n",
|
||||
"for i in range(1, 30):\n",
|
||||
" topic = f\"raw_scouter-opcua-pipeline-{i+1}\"\n",
|
||||
" print(topic)\n",
|
||||
" obj = deepcopy(spec)\n",
|
||||
" obj['spec']['dataSchema']['dataSource'] = topic\n",
|
||||
" obj['spec']['ioConfig']['topic'] = topic\n",
|
||||
" obj['spec']['ioConfig']['stream'] = topic\n",
|
||||
"\n",
|
||||
" json_list.append(obj)\n",
|
||||
"\n",
|
||||
"# Save to a new file\n",
|
||||
"with open('specs_30.json', 'w') as f:\n",
|
||||
" json.dump(json_list, f, indent=2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[OK] raw_scouter-opcua-pipeline-2 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-3 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-4 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-5 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-6 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-7 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-8 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-9 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-10 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-11 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-12 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-13 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-14 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-15 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-16 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-17 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-18 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-19 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-20 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-21 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-22 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-23 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-24 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-25 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-26 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-27 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-28 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-29 enviado.\n",
|
||||
"[OK] raw_scouter-opcua-pipeline-30 enviado.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import requests\n",
|
||||
"\n",
|
||||
"DRUID_OVERLORD = os.getenv(\"DRUID_OVERLORD\", \"http://localhost:8082\")\n",
|
||||
"SUPERVISOR_ENDPOINT = f\"{DRUID_OVERLORD}/druid/indexer/v1/supervisor\"\n",
|
||||
"\n",
|
||||
"def enviar_supervisores(specs):\n",
|
||||
" for spec in specs:\n",
|
||||
" resp = requests.post(\n",
|
||||
" SUPERVISOR_ENDPOINT,\n",
|
||||
" headers={\"Content-Type\": \"application/json\"},\n",
|
||||
" json=spec\n",
|
||||
" )\n",
|
||||
" if resp.status_code == 200:\n",
|
||||
" print(f\"[OK] {spec['spec']['dataSchema']['dataSource']} enviado.\")\n",
|
||||
" else:\n",
|
||||
" print(f\"[ERRO] {spec['spec']['dataSchema']['dataSource']}: {resp.status_code} → {resp.text}\")\n",
|
||||
"\n",
|
||||
"with open(\"./specs_30.json\", \"r\") as f:\n",
|
||||
" specs = json.load(f)\n",
|
||||
"enviar_supervisores(specs)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "ConnectionError",
|
||||
"evalue": "HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /druid/indexer/v1/supervisor/raw_scouter-opcua-orchestrated-pipeline-2/terminate (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x72128094cc50>: Failed to establish a new connection: [Errno 111] Connection refused'))",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
||||
"\u001b[31mConnectionRefusedError\u001b[39m Traceback (most recent call last)",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:198\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 197\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m198\u001b[39m sock = \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 199\u001b[39m \u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 200\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 201\u001b[39m \u001b[43m \u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msource_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 202\u001b[39m \u001b[43m \u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msocket_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 203\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 204\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.gaierror \u001b[38;5;28;01mas\u001b[39;00m e:\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 84\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m85\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[32m 86\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 87\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/util/connection.py:73\u001b[39m, in \u001b[36mcreate_connection\u001b[39m\u001b[34m(address, timeout, source_address, socket_options)\u001b[39m\n\u001b[32m 72\u001b[39m sock.bind(source_address)\n\u001b[32m---> \u001b[39m\u001b[32m73\u001b[39m \u001b[43msock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 74\u001b[39m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n",
|
||||
"\u001b[31mConnectionRefusedError\u001b[39m: [Errno 111] Connection refused",
|
||||
"\nThe above exception was the direct cause of the following exception:\n",
|
||||
"\u001b[31mNewConnectionError\u001b[39m Traceback (most recent call last)",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connectionpool.py:787\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 786\u001b[39m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m787\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 788\u001b[39m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 789\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 790\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 791\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 792\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 793\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 794\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 795\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 796\u001b[39m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 797\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 798\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 799\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 800\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 802\u001b[39m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connectionpool.py:493\u001b[39m, in \u001b[36mHTTPConnectionPool._make_request\u001b[39m\u001b[34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 492\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m493\u001b[39m \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 494\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 495\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 497\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 498\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 499\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 500\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 501\u001b[39m \u001b[43m \u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m=\u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 502\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 504\u001b[39m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[32m 505\u001b[39m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[32m 506\u001b[39m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:494\u001b[39m, in \u001b[36mHTTPConnection.request\u001b[39m\u001b[34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[39m\n\u001b[32m 493\u001b[39m \u001b[38;5;28mself\u001b[39m.putheader(header, value)\n\u001b[32m--> \u001b[39m\u001b[32m494\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 496\u001b[39m \u001b[38;5;66;03m# If we're given a body we start sending that in chunks.\u001b[39;00m\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.11/http/client.py:1298\u001b[39m, in \u001b[36mHTTPConnection.endheaders\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1297\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[32m-> \u001b[39m\u001b[32m1298\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.11/http/client.py:1058\u001b[39m, in \u001b[36mHTTPConnection._send_output\u001b[39m\u001b[34m(self, message_body, encode_chunked)\u001b[39m\n\u001b[32m 1057\u001b[39m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m._buffer[:]\n\u001b[32m-> \u001b[39m\u001b[32m1058\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1060\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1061\u001b[39m \n\u001b[32m 1062\u001b[39m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m/usr/lib/python3.11/http/client.py:996\u001b[39m, in \u001b[36mHTTPConnection.send\u001b[39m\u001b[34m(self, data)\u001b[39m\n\u001b[32m 995\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.auto_open:\n\u001b[32m--> \u001b[39m\u001b[32m996\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 997\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:325\u001b[39m, in \u001b[36mHTTPConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 324\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m325\u001b[39m \u001b[38;5;28mself\u001b[39m.sock = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 326\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._tunnel_host:\n\u001b[32m 327\u001b[39m \u001b[38;5;66;03m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connection.py:213\u001b[39m, in \u001b[36mHTTPConnection._new_conn\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 212\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m--> \u001b[39m\u001b[32m213\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[32m 214\u001b[39m \u001b[38;5;28mself\u001b[39m, \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 215\u001b[39m ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01me\u001b[39;00m\n\u001b[32m 217\u001b[39m sys.audit(\u001b[33m\"\u001b[39m\u001b[33mhttp.client.connect\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mself\u001b[39m.host, \u001b[38;5;28mself\u001b[39m.port)\n",
|
||||
"\u001b[31mNewConnectionError\u001b[39m: <urllib3.connection.HTTPConnection object at 0x72128094cc50>: Failed to establish a new connection: [Errno 111] Connection refused",
|
||||
"\nThe above exception was the direct cause of the following exception:\n",
|
||||
"\u001b[31mMaxRetryError\u001b[39m Traceback (most recent call last)",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/adapters.py:667\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 666\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m667\u001b[39m resp = \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 668\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 669\u001b[39m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 670\u001b[39m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 671\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m.\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 672\u001b[39m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 673\u001b[39m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 674\u001b[39m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 675\u001b[39m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 676\u001b[39m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 677\u001b[39m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 678\u001b[39m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m=\u001b[49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 679\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 681\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/connectionpool.py:841\u001b[39m, in \u001b[36mHTTPConnectionPool.urlopen\u001b[39m\u001b[34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[39m\n\u001b[32m 839\u001b[39m new_e = ProtocolError(\u001b[33m\"\u001b[39m\u001b[33mConnection aborted.\u001b[39m\u001b[33m\"\u001b[39m, new_e)\n\u001b[32m--> \u001b[39m\u001b[32m841\u001b[39m retries = \u001b[43mretries\u001b[49m\u001b[43m.\u001b[49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 842\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[43m=\u001b[49m\u001b[43msys\u001b[49m\u001b[43m.\u001b[49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[32m 843\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 844\u001b[39m retries.sleep()\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/urllib3/util/retry.py:519\u001b[39m, in \u001b[36mRetry.increment\u001b[39m\u001b[34m(self, method, url, response, error, _pool, _stacktrace)\u001b[39m\n\u001b[32m 518\u001b[39m reason = error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[32m--> \u001b[39m\u001b[32m519\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mreason\u001b[39;00m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[32m 521\u001b[39m log.debug(\u001b[33m\"\u001b[39m\u001b[33mIncremented Retry for (url=\u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m\"\u001b[39m, url, new_retry)\n",
|
||||
"\u001b[31mMaxRetryError\u001b[39m: HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /druid/indexer/v1/supervisor/raw_scouter-opcua-orchestrated-pipeline-2/terminate (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x72128094cc50>: Failed to establish a new connection: [Errno 111] Connection refused'))",
|
||||
"\nDuring handling of the above exception, another exception occurred:\n",
|
||||
"\u001b[31mConnectionError\u001b[39m Traceback (most recent call last)",
|
||||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[20]\u001b[39m\u001b[32m, line 16\u001b[39m\n\u001b[32m 13\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m❌ Erro \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresp.status_code\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresp.text\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m 15\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[32m1\u001b[39m, \u001b[32m30\u001b[39m):\n\u001b[32m---> \u001b[39m\u001b[32m16\u001b[39m \u001b[43mterminate_supervisor\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43mf\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mraw_scouter-opcua-orchestrated-pipeline-\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mi\u001b[49m\u001b[43m+\u001b[49m\u001b[32;43m1\u001b[39;49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
|
||||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[20]\u001b[39m\u001b[32m, line 7\u001b[39m, in \u001b[36mterminate_supervisor\u001b[39m\u001b[34m(supervisor_id)\u001b[39m\n\u001b[32m 5\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mterminate_supervisor\u001b[39m(supervisor_id):\n\u001b[32m 6\u001b[39m url = \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mDRUID\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/druid/indexer/v1/supervisor/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msupervisor_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/terminate\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m7\u001b[39m resp = \u001b[43mrequests\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 8\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m resp.status_code == \u001b[32m200\u001b[39m:\n\u001b[32m 9\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m✅ Supervisor \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msupervisor_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m encerrado.\u001b[39m\u001b[33m\"\u001b[39m)\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/api.py:115\u001b[39m, in \u001b[36mpost\u001b[39m\u001b[34m(url, data, json, **kwargs)\u001b[39m\n\u001b[32m 103\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpost\u001b[39m(url, data=\u001b[38;5;28;01mNone\u001b[39;00m, json=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m 104\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33mr\u001b[39m\u001b[33;03m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[32m 105\u001b[39m \n\u001b[32m 106\u001b[39m \u001b[33;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 112\u001b[39m \u001b[33;03m :rtype: requests.Response\u001b[39;00m\n\u001b[32m 113\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m115\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpost\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[43m=\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/api.py:59\u001b[39m, in \u001b[36mrequest\u001b[39m\u001b[34m(method, url, **kwargs)\u001b[39m\n\u001b[32m 55\u001b[39m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[32m 56\u001b[39m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[32m 57\u001b[39m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[32m 58\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m sessions.Session() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[32m---> \u001b[39m\u001b[32m59\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m=\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/sessions.py:589\u001b[39m, in \u001b[36mSession.request\u001b[39m\u001b[34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[39m\n\u001b[32m 584\u001b[39m send_kwargs = {\n\u001b[32m 585\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mtimeout\u001b[39m\u001b[33m\"\u001b[39m: timeout,\n\u001b[32m 586\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mallow_redirects\u001b[39m\u001b[33m\"\u001b[39m: allow_redirects,\n\u001b[32m 587\u001b[39m }\n\u001b[32m 588\u001b[39m send_kwargs.update(settings)\n\u001b[32m--> \u001b[39m\u001b[32m589\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 591\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/sessions.py:703\u001b[39m, in \u001b[36mSession.send\u001b[39m\u001b[34m(self, request, **kwargs)\u001b[39m\n\u001b[32m 700\u001b[39m start = preferred_clock()\n\u001b[32m 702\u001b[39m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m703\u001b[39m r = \u001b[43madapter\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 705\u001b[39m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[32m 706\u001b[39m elapsed = preferred_clock() - start\n",
|
||||
"\u001b[36mFile \u001b[39m\u001b[32m~/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/requests/adapters.py:700\u001b[39m, in \u001b[36mHTTPAdapter.send\u001b[39m\u001b[34m(self, request, stream, timeout, verify, cert, proxies)\u001b[39m\n\u001b[32m 696\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e.reason, _SSLError):\n\u001b[32m 697\u001b[39m \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[32m 698\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request=request)\n\u001b[32m--> \u001b[39m\u001b[32m700\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n\u001b[32m 702\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m 703\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request=request)\n",
|
||||
"\u001b[31mConnectionError\u001b[39m: HTTPConnectionPool(host='localhost', port=8090): Max retries exceeded with url: /druid/indexer/v1/supervisor/raw_scouter-opcua-orchestrated-pipeline-2/terminate (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x72128094cc50>: Failed to establish a new connection: [Errno 111] Connection refused'))"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import requests\n",
|
||||
"\n",
|
||||
"DRUID = os.getenv(\"DRUID_URL\", \"http://localhost:8082\")\n",
|
||||
"def terminate_supervisor(supervisor_id):\n",
|
||||
" url = f\"{DRUID}/druid/indexer/v1/supervisor/{supervisor_id}/terminate\"\n",
|
||||
" resp = requests.post(url)\n",
|
||||
" if resp.status_code == 200:\n",
|
||||
" print(f\"✅ Supervisor '{supervisor_id}' encerrado.\")\n",
|
||||
" elif resp.status_code == 404:\n",
|
||||
" print(f\"⚠️ Supervisor '{supervisor_id}' não encontrado.\")\n",
|
||||
" else:\n",
|
||||
" print(f\"❌ Erro {resp.status_code}: {resp.text}\")\n",
|
||||
"\n",
|
||||
"for i in range(1, 30):\n",
|
||||
" terminate_supervisor(f\"raw_scouter-opcua-orchestrated-pipeline-{i+1}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✅ Document deleted successfully\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from pymongo import MongoClient\n",
|
||||
"import os\n",
|
||||
"# Get MongoDB connection details from environment variables\n",
|
||||
"MONGODB_USERNAME = os.getenv(\"MONGODB_USERNAME\", \"root\")\n",
|
||||
"MONGODB_PASSWORD = os.getenv(\"MONGODB_PASSWORD\", \"wKZDbMNU1c\") \n",
|
||||
"MONGODB_URL = os.getenv(\"MONGODB_URL\", \"localhost:27018\")\n",
|
||||
"MONGODB_DATABASE = os.getenv(\"MONGODB_DATABASE\", \"sientia\")\n",
|
||||
"\n",
|
||||
"# Create MongoDB client\n",
|
||||
"client = MongoClient(\n",
|
||||
" f\"mongodb://{MONGODB_USERNAME}:{MONGODB_PASSWORD}@{MONGODB_URL}\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Get database and collection\n",
|
||||
"db = client[MONGODB_DATABASE]\n",
|
||||
"collection = db[\"pipelines\"] # Replace with actual collection name\n",
|
||||
"\n",
|
||||
"for i in range(2, 31):\n",
|
||||
" # Delete a document matching specific criteria\n",
|
||||
" result = collection.delete_one({\"schedule_name\": f\"scouter-opcua-pipeline-{i}\"}) # Replace with actual query\n",
|
||||
"\n",
|
||||
"if result.deleted_count > 0:\n",
|
||||
" print(\"✅ Document deleted successfully\")\n",
|
||||
"else:\n",
|
||||
" print(\"⚠️ No matching document found\")\n",
|
||||
"\n",
|
||||
"# Close the connection\n",
|
||||
"client.close()\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
Reference in New Issue
Block a user