Merge pull request #47 from Aignosi/rc/sqlalchemy-21-mongo-uri-log

Fix: Prevent MongoDB credential logging and update database dependency

ingestor/managers/data_manager.py

@@ -12,6 +12,22 @@ from sientia_do.temporal.constants import now
 import ingestor.metrics as metrics
 
 
+def redact_uri(uri: str) -> str:
+    """
+    Returns ``uri`` without the user and password, safe to log
+    (``mongodb://user:pass@host:27017/db`` -> ``mongodb://***@host:27017/db``).
+    """
+    prefix, sep, rest = uri.partition('://')
+    if not sep:
+        prefix, rest = '', uri
+    ends = [i for i in (rest.find('/'), rest.find('?')) if i != -1]
+    end = min(ends) if ends else len(rest)
+    authority, tail = rest[:end], rest[end:]
+    if '@' not in authority:
+        return uri
+    return f'{prefix}{sep}***@{authority.rpartition("@")[2]}{tail}'
+
+
 class DataManager(SientiaMonitoring):
     """
     Manages data persistence operations for the OPC Ingestor.
@@ -66,7 +82,8 @@ class DataManager(SientiaMonitoring):
         )
 
         logger.info(
-            f'Trying to initializing DataManager with MongoDB servers: {mongo_connection_string}'
+            'Trying to initializing DataManager with MongoDB servers: '
+            f'{redact_uri(mongo_connection_string)}'
         )
 
         self.connection_string = mongo_connection_string
@@ -82,7 +99,9 @@ class DataManager(SientiaMonitoring):
 
         self.metadata = metadata
 
-        logger.info(f'DataManager initialized with MongoDB servers: {self.connection_string}')
+        logger.info(
+            f'DataManager initialized with MongoDB servers: {redact_uri(self.connection_string)}'
+        )
 
     def shutdown(self):
         """

requirements.txt

@@ -1,9 +1,8 @@
 asyncua
 redis
-# TODO(SIENTIAPDE-1972): bump once sientia-dataops-library releases SIENTIAPDE-1590
+# 1.12.5 builds the Postgres engine with an explicit postgresql+psycopg2:// URL: SQLAlchemy 2.1
-# (generic Mongo find_one/distinct/aggregate methods) — not in 1.12.2 yet.
+# defaults postgresql:// to psycopg (v3), which is not installed ("No module named 'psycopg'").
-# Local dev uses an editable install of the local checkout (feature/SIENTIAPDE-1590).
+sientia_do==1.12.5
-sientia_do==1.12.2
 prometheus_client
 pymongo
 nats-py>=2.6

tests/unit/managers/test_data_manager.py

@@ -1,9 +1,10 @@
 from unittest.mock import ANY, AsyncMock, MagicMock, patch
 
+import pytest
 from pytest import fixture, mark
 from sientia_do.notifications.models import NotificationLevel
 
-from ingestor.managers.data_manager import DataManager
+from ingestor.managers.data_manager import DataManager, redact_uri
 
 metadata = {
     'metadata': {
@@ -54,6 +55,48 @@ def test___init___success(mongodb_repository):
     assert data_manager.connection_string == 'mongodb://localhost:27017'
 
 
+@patch('ingestor.managers.data_manager.MongoDBRepository')
+def test___init___does_not_log_the_credentials(mongodb_repository):
+    logger_mock = MagicMock()
+    uri = 'mongodb://root:s3cr3t@mongodb.mongodb.svc.cluster.local:27017'
+
+    data_manager = DataManager(
+        metadata=metadata['metadata'],
+        mongo_connection_string=uri,
+        mongo_database='sientia',
+        logger=logger_mock,
+        notification_handler=MagicMock(),
+        metrics_controller=MagicMock(),
+    )
+
+    logged = ' '.join(str(c) for c in logger_mock.info.call_args_list)
+    assert 's3cr3t' not in logged
+    assert 'root' not in logged
+    logger_mock.info.assert_any_call(
+        'DataManager initialized with MongoDB servers: '
+        'mongodb://***@mongodb.mongodb.svc.cluster.local:27017'
+    )
+    assert data_manager.connection_string == uri
+
+
+@pytest.mark.parametrize(
+    ('uri', 'expected'),
+    [
+        ('mongodb://localhost:27017', 'mongodb://localhost:27017'),
+        (
+            'mongodb://u:p@h1:27017,h2:27017/db?authSource=admin',
+            'mongodb://***@h1:27017,h2:27017/db?authSource=admin',
+        ),
+        ('mongodb://u:p@h:27017?authSource=admin', 'mongodb://***@h:27017?authSource=admin'),
+        ('mongodb+srv://u@cluster.example.com', 'mongodb+srv://***@cluster.example.com'),
+        ('u:p@h:27017', '***@h:27017'),
+        ('mongodb://h:27017/db?x=a@b', 'mongodb://h:27017/db?x=a@b'),
+    ],
+)
+def test_redact_uri(uri, expected):
+    assert redact_uri(uri) == expected
+
+
 def test_shutdown(data_manager):
     close_mock = MagicMock()
     data_manager.mongo_repository.close = close_mock