Compare commits
16 Commits
test-SIENT
...
d68e842e6e
| Author | SHA1 | Date | |
|---|---|---|---|
| d68e842e6e | |||
|
|
b2bbe93b7c | ||
|
|
9debff8047 | ||
| 7ee09ee743 | |||
| e189bbf085 | |||
| 78a8f5429e | |||
| 0266c1b7e5 | |||
| 4fab1e284f | |||
| f8bea68ae3 | |||
|
|
0f4c9adcf9 | ||
| 01bce62859 | |||
| f8d5a1f5a7 | |||
| 7b3da582be | |||
| 4a749bd1b6 | |||
|
|
279484b2aa | ||
|
|
ce67105c16 |
15
.github/workflows/deploy.yml
vendored
15
.github/workflows/deploy.yml
vendored
@@ -1,10 +1,15 @@
|
|||||||
name: Deploy DataOps Worker
|
name: Deploy DataOps Worker
|
||||||
|
|
||||||
on:
|
on:
|
||||||
pull_request:
|
|
||||||
types: [closed]
|
|
||||||
branches: [main, 'release/**', 'feature/**']
|
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
chart_ref:
|
||||||
|
description: >
|
||||||
|
sientia-helm-chart branch to bump GITHUB_BRANCH on AND deploy — the literal
|
||||||
|
deploy target. Validated via git ls-remote; a branch that does not exist in
|
||||||
|
sientia-helm-chart is a hard error, not a silent fallback to main.
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
contents: write
|
||||||
@@ -14,10 +19,10 @@ permissions:
|
|||||||
jobs:
|
jobs:
|
||||||
deploy:
|
deploy:
|
||||||
name: Deploy
|
name: Deploy
|
||||||
if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true
|
|
||||||
uses: Aignosi/github_workflow_templates/.github/workflows/reusable-deploy-dataops.yml@main
|
uses: Aignosi/github_workflow_templates/.github/workflows/reusable-deploy-dataops.yml@main
|
||||||
with:
|
with:
|
||||||
branch_name: ${{ github.event.pull_request.head.ref }}
|
branch_name: ${{ github.ref_name }}
|
||||||
|
chart_ref: ${{ inputs.chart_ref }}
|
||||||
project_name: 'ingestor'
|
project_name: 'ingestor'
|
||||||
chart_name: 'opc-ingestor'
|
chart_name: 'opc-ingestor'
|
||||||
env_list_path: '[".env"]'
|
env_list_path: '[".env"]'
|
||||||
|
|||||||
1
.github/workflows/quality-gate.yml
vendored
1
.github/workflows/quality-gate.yml
vendored
@@ -5,7 +5,6 @@ on:
|
|||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
- 'release/**'
|
- 'release/**'
|
||||||
- 'feature/**'
|
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
|||||||
16
.github/workflows/require-chart-ref-label.yml
vendored
16
.github/workflows/require-chart-ref-label.yml
vendored
@@ -1,16 +0,0 @@
|
|||||||
name: Require chart-ref label
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
types: [opened, synchronize, reopened, labeled, unlabeled]
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
- 'release/**'
|
|
||||||
- 'feature/**'
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
pull-requests: write
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
require-chart-ref-label:
|
|
||||||
uses: Aignosi/github_workflow_templates/.github/workflows/require-chart-ref-label.yml@main
|
|
||||||
17
.github/workflows/version-bump.yml
vendored
Normal file
17
.github/workflows/version-bump.yml
vendored
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
name: Version Bump
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
types: [closed]
|
||||||
|
branches: [main, 'release/**', 'feature/**']
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
bump:
|
||||||
|
if: github.event.pull_request.merged == true
|
||||||
|
uses: Aignosi/github_workflow_templates/.github/workflows/reusable-version-bump.yml@main
|
||||||
|
with:
|
||||||
|
update_version_in: '["pyproject"]'
|
||||||
|
secrets: inherit
|
||||||
@@ -12,6 +12,22 @@ from sientia_do.temporal.constants import now
|
|||||||
import ingestor.metrics as metrics
|
import ingestor.metrics as metrics
|
||||||
|
|
||||||
|
|
||||||
|
def redact_uri(uri: str) -> str:
|
||||||
|
"""
|
||||||
|
Returns ``uri`` without the user and password, safe to log
|
||||||
|
(``mongodb://user:pass@host:27017/db`` -> ``mongodb://***@host:27017/db``).
|
||||||
|
"""
|
||||||
|
prefix, sep, rest = uri.partition('://')
|
||||||
|
if not sep:
|
||||||
|
prefix, rest = '', uri
|
||||||
|
ends = [i for i in (rest.find('/'), rest.find('?')) if i != -1]
|
||||||
|
end = min(ends) if ends else len(rest)
|
||||||
|
authority, tail = rest[:end], rest[end:]
|
||||||
|
if '@' not in authority:
|
||||||
|
return uri
|
||||||
|
return f'{prefix}{sep}***@{authority.rpartition("@")[2]}{tail}'
|
||||||
|
|
||||||
|
|
||||||
class DataManager(SientiaMonitoring):
|
class DataManager(SientiaMonitoring):
|
||||||
"""
|
"""
|
||||||
Manages data persistence operations for the OPC Ingestor.
|
Manages data persistence operations for the OPC Ingestor.
|
||||||
@@ -66,7 +82,8 @@ class DataManager(SientiaMonitoring):
|
|||||||
)
|
)
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
f'Trying to initializing DataManager with MongoDB servers: {mongo_connection_string}'
|
'Trying to initializing DataManager with MongoDB servers: '
|
||||||
|
f'{redact_uri(mongo_connection_string)}'
|
||||||
)
|
)
|
||||||
|
|
||||||
self.connection_string = mongo_connection_string
|
self.connection_string = mongo_connection_string
|
||||||
@@ -82,7 +99,9 @@ class DataManager(SientiaMonitoring):
|
|||||||
|
|
||||||
self.metadata = metadata
|
self.metadata = metadata
|
||||||
|
|
||||||
logger.info(f'DataManager initialized with MongoDB servers: {self.connection_string}')
|
logger.info(
|
||||||
|
f'DataManager initialized with MongoDB servers: {redact_uri(self.connection_string)}'
|
||||||
|
)
|
||||||
|
|
||||||
def shutdown(self):
|
def shutdown(self):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "ingestor"
|
name = "ingestor"
|
||||||
version = "2.2.0-rc1"
|
version = "2.0.0-rc4"
|
||||||
description = "Sientia DataOps Ingestor - OPC Tag Ingestor"
|
description = "Sientia DataOps Ingestor - OPC Tag Ingestor"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.11"
|
requires-python = ">=3.11"
|
||||||
|
|||||||
@@ -1,12 +1,8 @@
|
|||||||
asyncua
|
asyncua
|
||||||
redis
|
redis
|
||||||
# TODO(SIENTIAPDE-1972): bump once sientia-dataops-library releases SIENTIAPDE-1590
|
# 1.12.5 builds the Postgres engine with an explicit postgresql+psycopg2:// URL: SQLAlchemy 2.1
|
||||||
# (generic Mongo find_one/distinct/aggregate methods) — not in 1.12.2 yet.
|
# defaults postgresql:// to psycopg (v3), which is not installed ("No module named 'psycopg'").
|
||||||
# Local dev uses an editable install of the local checkout (feature/SIENTIAPDE-1590).
|
sientia_do==1.12.5
|
||||||
sientia_do==1.12.2
|
|
||||||
# TEST ONLY (SIENTIAPDE-1975 dev test): SQLAlchemy 2.1 makes psycopg (v3) the default driver for
|
|
||||||
# postgresql:// URLs; sientia_do 1.12.2 builds that URL and ships only psycopg2-binary.
|
|
||||||
sqlalchemy<2.1
|
|
||||||
prometheus_client
|
prometheus_client
|
||||||
pymongo
|
pymongo
|
||||||
nats-py>=2.6
|
nats-py>=2.6
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
from unittest.mock import ANY, AsyncMock, MagicMock, patch
|
from unittest.mock import ANY, AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
from pytest import fixture, mark
|
from pytest import fixture, mark
|
||||||
from sientia_do.notifications.models import NotificationLevel
|
from sientia_do.notifications.models import NotificationLevel
|
||||||
|
|
||||||
from ingestor.managers.data_manager import DataManager
|
from ingestor.managers.data_manager import DataManager, redact_uri
|
||||||
|
|
||||||
metadata = {
|
metadata = {
|
||||||
'metadata': {
|
'metadata': {
|
||||||
@@ -54,6 +55,48 @@ def test___init___success(mongodb_repository):
|
|||||||
assert data_manager.connection_string == 'mongodb://localhost:27017'
|
assert data_manager.connection_string == 'mongodb://localhost:27017'
|
||||||
|
|
||||||
|
|
||||||
|
@patch('ingestor.managers.data_manager.MongoDBRepository')
|
||||||
|
def test___init___does_not_log_the_credentials(mongodb_repository):
|
||||||
|
logger_mock = MagicMock()
|
||||||
|
uri = 'mongodb://root:s3cr3t@mongodb.mongodb.svc.cluster.local:27017'
|
||||||
|
|
||||||
|
data_manager = DataManager(
|
||||||
|
metadata=metadata['metadata'],
|
||||||
|
mongo_connection_string=uri,
|
||||||
|
mongo_database='sientia',
|
||||||
|
logger=logger_mock,
|
||||||
|
notification_handler=MagicMock(),
|
||||||
|
metrics_controller=MagicMock(),
|
||||||
|
)
|
||||||
|
|
||||||
|
logged = ' '.join(str(c) for c in logger_mock.info.call_args_list)
|
||||||
|
assert 's3cr3t' not in logged
|
||||||
|
assert 'root' not in logged
|
||||||
|
logger_mock.info.assert_any_call(
|
||||||
|
'DataManager initialized with MongoDB servers: '
|
||||||
|
'mongodb://***@mongodb.mongodb.svc.cluster.local:27017'
|
||||||
|
)
|
||||||
|
assert data_manager.connection_string == uri
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
('uri', 'expected'),
|
||||||
|
[
|
||||||
|
('mongodb://localhost:27017', 'mongodb://localhost:27017'),
|
||||||
|
(
|
||||||
|
'mongodb://u:p@h1:27017,h2:27017/db?authSource=admin',
|
||||||
|
'mongodb://***@h1:27017,h2:27017/db?authSource=admin',
|
||||||
|
),
|
||||||
|
('mongodb://u:p@h:27017?authSource=admin', 'mongodb://***@h:27017?authSource=admin'),
|
||||||
|
('mongodb+srv://u@cluster.example.com', 'mongodb+srv://***@cluster.example.com'),
|
||||||
|
('u:p@h:27017', '***@h:27017'),
|
||||||
|
('mongodb://h:27017/db?x=a@b', 'mongodb://h:27017/db?x=a@b'),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_redact_uri(uri, expected):
|
||||||
|
assert redact_uri(uri) == expected
|
||||||
|
|
||||||
|
|
||||||
def test_shutdown(data_manager):
|
def test_shutdown(data_manager):
|
||||||
close_mock = MagicMock()
|
close_mock = MagicMock()
|
||||||
data_manager.mongo_repository.close = close_mock
|
data_manager.mongo_repository.close = close_mock
|
||||||
|
|||||||
Reference in New Issue
Block a user