SIENTIAPDE-1110

Enhance Druid activity by adding configuration for Druid host and port in values.yaml. Refactor data loading logic to utilize SQLAlchemy for querying Druid, improving query efficiency and readability. Update timestamp handling to streamline data retrieval process.

scouter/activities/pydruid.py

@@ -6,6 +6,8 @@ with workflow.unsafe.imports_passed_through():
     from datetime import datetime, timedelta
     from pydruid.client import PyDruid
     from pydruid.query import QueryBuilder
+    from sqlalchemy.engine import create_engine
+    from sqlalchemy import MetaData, Table, select, text
     from sientia_do.temporal.activities.base import BaseActivity
     from sientia_do.notifications.handlers import NotificationHandler
     from sientia_do.temporal.utils.logger import Logger
@@ -13,15 +15,12 @@ with workflow.unsafe.imports_passed_through():
 
 class Druid(BaseActivity):
     def __init__(self, host: str, port: int,
-                 logger: Logger, notification_handler: NotificationHandler,
+                 logger: Logger, notification_handler: NotificationHandler):
-                 endpoint: str = "druid/v2"):
 
         self.host = host
         self.port = port
-        self.endpoint = endpoint
+        self.engine = create_engine(
-        self.client = PyDruid(
+            f'druid://{self.host}:{self.port}/druid/v2/sql/')
-            f"http://{self.host}:{self.port}", {self.endpoint}
-        )
         logger.info(
             f"Druid client initialized with host: {self.host}, port: {self.port}")
 
@@ -41,36 +40,25 @@ class Druid(BaseActivity):
         """
         metadata = input_data['metadata']
         datasource = f"raw_{input_data['schedule_name']}"
-        last_data_timestamp_str = input_data['last_data_timestamp']
+        last_data_timestamp = input_data['last_data_timestamp']
-        if last_data_timestamp_str is None:
-            last_data_timestamp = datetime(1970, 1, 1, 0, 0, 0)
-        else:
-            last_data_timestamp = datetime.strptime(
-                last_data_timestamp_str, "%Y-%m-%d %H:%M:%S.%f")
-
         self.debug(
             f"Loading data from Druid: {input_data}", metadata=metadata)
 
-        end_time = datetime(9999, 12, 31, 23, 59, 59)
+        query = f'"__time" > TIMESTAMP \'{last_data_timestamp}\''
-
-        interval = f"{last_data_timestamp.isoformat()}Z/{end_time.isoformat()}Z"
 
         self.info(
-            f"Loading data from Druid: {datasource} with interval: {interval}"
+            f"Loading data from Druid: {datasource} with query: {query}"
         )
 
-        builder = QueryBuilder()
+        places = Table(datasource, MetaData(), autoload_with=self.engine)
+        stmt = select(places).where(text(query))
 
-        query = builder.scan(
+        result = pd.read_sql(stmt, self.engine)
-            {
-                "datasource": datasource,
-                "intervals": interval,
-                "columns": ["timestamp", "value", "tag"],
-                "limit": 10000,
-            }
-        )
 
-        result = query.export_pandas()
+        result["inserted_at"] = pd.to_datetime(result["__time"]).dt.strftime(
+            "%Y-%m-%d %H:%M:%S.%f")
+
+        result.drop(columns=["__time"], inplace=True)
 
         self.info(
             f"Loaded {len(result)} rows from Druid"

test.ipynb

@@ -0,0 +1,267 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sqlalchemy.engine import create_engine\n",
+    "\n",
+    "engine = create_engine('druid://localhost:8082/druid/v2/sql/')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sqlalchemy import MetaData, Table\n",
+    "\n",
+    "metadata = MetaData()\n",
+    "places = Table('raw_scouter-opcua-orchestrated-pipeline', metadata, autoload_with=engine)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sqlalchemy import select\n",
+    "\n",
+    "stmt = select(places)\n",
+    "with engine.connect() as conn:\n",
+    "    result = conn.execute(stmt)\n",
+    "    for row in result:\n",
+    "        print(row)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_30057/1086103244.py:5: SADeprecationWarning: The dbapi() classmethod on dialect classes has been renamed to import_dbapi().  Implement an import_dbapi() classmethod directly on class <class 'pydruid.db.sqlalchemy.DruidDialect'> to remove this warning; the old .dbapi() classmethod may be maintained for backwards compatibility.\n",
+      "  engine = create_engine('druid://localhost:8082/druid/v2/sql/')\n",
+      "/home/grezewave/Documents/projects/sientia/sientia-dataops-scouter_temporal/venv/lib/python3.11/site-packages/pydruid/db/sqlalchemy.py:188: SAWarning: Dialect druid:rest will not make use of SQL compilation caching as it does not set the 'supports_statement_cache' attribute to ``True``.  This can have significant performance implications including some performance degradations in comparison to prior SQLAlchemy versions.  Dialect maintainers should seek to set this attribute to True after appropriate development and testing for SQLAlchemy 1.4 caching support.   Alternatively, this attribute may be set to False which will disable this warning. (Background on this warning at: https://sqlalche.me/e/20/cprf)\n",
+      "  result = connection.execute(text(query))\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>name</th>\n",
+       "      <th>kafka.topic</th>\n",
+       "      <th>tag</th>\n",
+       "      <th>value</th>\n",
+       "      <th>timestamp</th>\n",
+       "      <th>inserted_at</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Counter</td>\n",
+       "      <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
+       "      <td>ns=2;i=2</td>\n",
+       "      <td>-50.132</td>\n",
+       "      <td>2025-07-02 13:05:19</td>\n",
+       "      <td>2025-07-02 13:05:19.729000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Rollout</td>\n",
+       "      <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
+       "      <td>ns=2;i=3</td>\n",
+       "      <td>70.767</td>\n",
+       "      <td>2025-07-02 13:05:19</td>\n",
+       "      <td>2025-07-02 13:05:19.731000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Square</td>\n",
+       "      <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
+       "      <td>ns=2;i=4</td>\n",
+       "      <td>-58.448</td>\n",
+       "      <td>2025-07-02 13:05:19</td>\n",
+       "      <td>2025-07-02 13:05:19.732000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Counter</td>\n",
+       "      <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
+       "      <td>ns=2;i=2</td>\n",
+       "      <td>-50.126</td>\n",
+       "      <td>2025-07-02 13:05:24</td>\n",
+       "      <td>2025-07-02 13:05:24.728000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Rollout</td>\n",
+       "      <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
+       "      <td>ns=2;i=3</td>\n",
+       "      <td>69.199</td>\n",
+       "      <td>2025-07-02 13:05:24</td>\n",
+       "      <td>2025-07-02 13:05:24.730000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>373</th>\n",
+       "      <td>Rollout</td>\n",
+       "      <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
+       "      <td>ns=2;i=3</td>\n",
+       "      <td>85.921</td>\n",
+       "      <td>2025-07-02 13:15:40</td>\n",
+       "      <td>2025-07-02 13:15:40.230000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>374</th>\n",
+       "      <td>Square</td>\n",
+       "      <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
+       "      <td>ns=2;i=4</td>\n",
+       "      <td>-69.296</td>\n",
+       "      <td>2025-07-02 13:15:40</td>\n",
+       "      <td>2025-07-02 13:15:40.232000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>375</th>\n",
+       "      <td>Counter</td>\n",
+       "      <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
+       "      <td>ns=2;i=2</td>\n",
+       "      <td>-71.207</td>\n",
+       "      <td>2025-07-02 13:15:45</td>\n",
+       "      <td>2025-07-02 13:15:45.228000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>376</th>\n",
+       "      <td>Rollout</td>\n",
+       "      <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
+       "      <td>ns=2;i=3</td>\n",
+       "      <td>84.665</td>\n",
+       "      <td>2025-07-02 13:15:45</td>\n",
+       "      <td>2025-07-02 13:15:45.231000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>377</th>\n",
+       "      <td>Square</td>\n",
+       "      <td>raw_scouter-opcua-orchestrated-pipeline</td>\n",
+       "      <td>ns=2;i=4</td>\n",
+       "      <td>-67.592</td>\n",
+       "      <td>2025-07-02 13:15:45</td>\n",
+       "      <td>2025-07-02 13:15:45.233000</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>378 rows × 6 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        name                              kafka.topic       tag   value  \\\n",
+       "0    Counter  raw_scouter-opcua-orchestrated-pipeline  ns=2;i=2 -50.132   \n",
+       "1    Rollout  raw_scouter-opcua-orchestrated-pipeline  ns=2;i=3  70.767   \n",
+       "2     Square  raw_scouter-opcua-orchestrated-pipeline  ns=2;i=4 -58.448   \n",
+       "3    Counter  raw_scouter-opcua-orchestrated-pipeline  ns=2;i=2 -50.126   \n",
+       "4    Rollout  raw_scouter-opcua-orchestrated-pipeline  ns=2;i=3  69.199   \n",
+       "..       ...                                      ...       ...     ...   \n",
+       "373  Rollout  raw_scouter-opcua-orchestrated-pipeline  ns=2;i=3  85.921   \n",
+       "374   Square  raw_scouter-opcua-orchestrated-pipeline  ns=2;i=4 -69.296   \n",
+       "375  Counter  raw_scouter-opcua-orchestrated-pipeline  ns=2;i=2 -71.207   \n",
+       "376  Rollout  raw_scouter-opcua-orchestrated-pipeline  ns=2;i=3  84.665   \n",
+       "377   Square  raw_scouter-opcua-orchestrated-pipeline  ns=2;i=4 -67.592   \n",
+       "\n",
+       "               timestamp                 inserted_at  \n",
+       "0    2025-07-02 13:05:19  2025-07-02 13:05:19.729000  \n",
+       "1    2025-07-02 13:05:19  2025-07-02 13:05:19.731000  \n",
+       "2    2025-07-02 13:05:19  2025-07-02 13:05:19.732000  \n",
+       "3    2025-07-02 13:05:24  2025-07-02 13:05:24.728000  \n",
+       "4    2025-07-02 13:05:24  2025-07-02 13:05:24.730000  \n",
+       "..                   ...                         ...  \n",
+       "373  2025-07-02 13:15:40  2025-07-02 13:15:40.230000  \n",
+       "374  2025-07-02 13:15:40  2025-07-02 13:15:40.232000  \n",
+       "375  2025-07-02 13:15:45  2025-07-02 13:15:45.228000  \n",
+       "376  2025-07-02 13:15:45  2025-07-02 13:15:45.231000  \n",
+       "377  2025-07-02 13:15:45  2025-07-02 13:15:45.233000  \n",
+       "\n",
+       "[378 rows x 6 columns]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from sqlalchemy import create_engine, MetaData, Table, select, func, text\n",
+    "import pandas as pd\n",
+    "from datetime import datetime\n",
+    "\n",
+    "engine = create_engine('druid://localhost:8082/druid/v2/sql/')\n",
+    "metadata = MetaData()\n",
+    "places = Table('raw_scouter-opcua-orchestrated-pipeline', metadata, autoload_with=engine)\n",
+    "date_str = '2025-01-01'\n",
+    "stmt = select(places).where(text(f'\"__time\" > TIMESTAMP \\'{date_str}\\''))\n",
+    "\n",
+    "result = pd.read_sql(stmt, engine)\n",
+    "\n",
+    "result[\"inserted_at\"] = pd.to_datetime(result[\"__time\"]).dt.strftime(\n",
+    "    \"%Y-%m-%d %H:%M:%S.%f\")\n",
+    "\n",
+    "result.drop(columns=[\"__time\"], inplace=True)\n",
+    "\n",
+    "display(result)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

values.yaml

@@ -182,6 +182,11 @@ env:
   - name: MONGODB_DATABASE
     value: "sientia"
 
+  - name: DRUID_HOST
+    value: "druid-router.druid.svc.cluster.local"
+  - name: DRUID_PORT
+    value: "8081"
+
 ssh:
   enabled: true
   secretName: git-ssh-key-sientia-scouter-worker