chore: bump version to 0.6.6

Fix/sientiapde 2013

CLAUDE.md

@@ -45,7 +45,7 @@ CI quality gate (`.github/workflows/quality-gate.yml`) calls the shared reusable
 
 ### Layering
 - **`orchestrator/worker/worker.py`** — entrypoint (`python -m orchestrator.worker.worker`). Builds a single `Activities` instance, connects to Temporal, starts Prometheus (`HTTP_METRICS_PORT`) and Temporal SDK metrics (`HTTP_SDK_METRICS_PORT`) servers, then runs three `Worker`s concurrently via `asyncio.gather`, each on its own task queue: `orchestrator-queue`, `alerts-queue`, `reports-queue`. `prepare_worker.py` derives the queue name from the workflow class name (`camel_to_snake`) and reads per-queue concurrency/poller tuning from env vars prefixed with the workflow name (e.g. `ORCHESTRATOR_MAX_CONCURRENT_ACTIVITIES`).
-- **`orchestrator/workflows/`** — Temporal workflow definitions: `orchestrator.py` (pipeline/slot/schedule orchestration), `alerts.py` (real-time ERROR notifications), `reports.py` (scheduled all-level reports). `alerts.py` and `reports.py` both delegate to the same two subworkflows in `workflows/subworkflows/`: `load_notification_package.py` (load notifications + receiver configs from Mongo, incremental via Redis timestamp) and `process_notifications.py` (build HTML email, send, log to Postgres). The difference between alerts and reports is which filter activity they call (`filter_notification_alerts` vs `filter_notification_reports`) and their base data filter (`level: ERROR` by default vs all levels) — `load_notification_package.py`'s `_compute_query_time_filter` can widen either beyond that default to the union of active receiver groups' own `levels` (SIENTIAPDE-2011); the ERROR-only default only holds when no active group narrows further.
+- **`orchestrator/workflows/`** — Temporal workflow definitions: `orchestrator.py` (pipeline/slot/schedule orchestration), `alerts.py` (real-time ERROR notifications), `reports.py` (scheduled all-level reports). `alerts.py` and `reports.py` both delegate to the same two subworkflows in `workflows/subworkflows/`: `load_notification_package.py` (load notifications + receiver configs from Mongo, incremental via Redis timestamp) and `process_notifications.py` (build HTML email, send, log to Postgres). The difference between alerts and reports is which filter activity they call (`filter_notification_alerts` vs `filter_notification_reports`) and their base data filter (`level: ERROR` by default vs all levels) — `load_notification_package.py`'s `_compute_query_time_filter` can widen either beyond that default to the union of active receiver groups' own `levels` (SIENTIAPDE-2011); the ERROR-only default only holds when no active group narrows further. `load_latest_data`'s Mongo query is capped at `NOTIFICATION_QUEUE_BATCH_SIZE` documents per run (default `0`, meaning unlimited — unchanged from pre-SIENTIAPDE-2013 behavior unless a deployment opts in), sorted by `timestamp` ascending so a capped run always returns the oldest pending batch first (SIENTIAPDE-2013) — a hard cap with no drain loop, since the incremental Redis cursor picks up any remainder on the next run.
 - **`orchestrator/activities/activities.py`** — `Activities` is a single class combining every activity mixin via multiple inheritance: `TemporalManager, SlotManager, Formatters, MongoDB, Email, Postgres` (Postgres comes from `sientia_do.temporal.activities.postgres`, the external `sientia-dataops-library` dependency). One instance is constructed in `worker.py` and its bound methods are handed to each `Worker` as the activity list — activities are plain methods on this shared object, not separate classes. Each mixin's `__init__` takes its own config dict plus shared `logger`, `notification_handler`, `metrics_controller`. `Activities.shutdown()` closes every mixin's connections.
 - **`orchestrator/utils/`** — `connectors_config.py` builds each config dict from env vars (Temporal/Redis/MongoDB/Postgres/Email); `email_builder.py` renders Jinja2 templates (`utils/templates/email_template.html`, `general_template.html`); `orchestrator_functions.py` transforms pipeline documents into pipeline-type-specific configs (`scouter`, `pi_web_api_scouter`, `predictions_batch`, `minimal_retrain`, `drift`, `simple_metrics`); `converters.py` handles Temporal schedule frequency parsing.
 

e2e/conftest.py

@@ -330,6 +330,7 @@ async def test_activities(
             'connection_string': f'mongodb://localhost:{mongo_port}',
             'database_name': E2E_DATABASE,
             'ttl_index_seconds': 3600,
+            'notification_queue_batch_size': 0,
         },
         email_config={
             'sender_email': 'e2e@example.com',

e2e/helpers.py

@@ -7,6 +7,7 @@ import json
 import re
 import uuid
 from datetime import UTC, datetime, timedelta
+from email.message import EmailMessage, Message
 from pathlib import Path
 from typing import Any
 
@@ -356,3 +357,40 @@ def default_receiver_group(
         'contents': contents or ['core_alerts', 'persistent_alerts', 'reports'],
         'ignore_models_id': ignore_models_id or [],
     }
+
+
+def extract_html_body(message: EmailMessage | Message) -> str:
+    """
+    Return the text/html portion of an email message as a decoded string.
+
+    Walks every part looking for the first text/html payload, decoding it
+    according to the part's transfer encoding and charset. Falls back to
+    the message's own ``get_content``/raw payload when no HTML part is
+    present so callers can still inspect plain-text reports. Shared between
+    the Alerts and Reports e2e suites, both of which send mail through the
+    same ``build_email_html``/``send_email`` activities.
+
+    Args:
+        message: Captured email message returned by the test SMTP server.
+
+    Return:
+        str: HTML body content, or an empty string when nothing decodable
+            is found.
+    """
+    if message.is_multipart():
+        for part in message.walk():
+            if part.get_content_type() != 'text/html':
+                continue
+            payload = part.get_payload(decode=True)
+            if payload is None:
+                continue
+            charset = part.get_content_charset() or 'utf-8'
+            return payload.decode(charset, errors='replace')
+    payload = message.get_payload(decode=True)
+    if payload is not None:
+        charset = message.get_content_charset() or 'utf-8'
+        return payload.decode(charset, errors='replace')
+    try:
+        return message.get_content()
+    except (AttributeError, KeyError):
+        return str(message.get_payload() or '')

e2e/scenarios.md

@@ -6,6 +6,11 @@ Functional reference for orchestrator E2E scenarios. Tests live under `e2e/`, us
 pytest e2e/ --override-ini testpaths=e2e -m e2e
 ```
 
+**Every section below maps 1:1 to an actual test function** (named in each section, or as a bullet
+under a `Source:` heading) — this file describes what the suite verifies today, not planned or
+aspirational coverage. Adding a scenario here without a matching test (or vice versa) is a doc bug;
+fix the mismatch instead of leaving it for later.
+
 ## Execution context
 
 - MongoDB, Redis, PostgreSQL: testcontainers (session-scoped).
@@ -21,31 +26,20 @@ Source: `e2e/test_orchestrator_main_workflow.py`
 
 ### 1.1.1 Happy path
 
-Pipelines in Mongo → schedules created in correct namespace/task queue, Redis slots written, `orchestrated_schedules` updated.
-
-### 1.2.1 No-op tick
-
-Mongo, Redis, and Temporal already match desired state → no new schedules or slot writes.
+Pipelines in Mongo → Temporal schedule created in the `laborious` namespace, `orchestrated_schedules` entries written.
 
 ### 1.3.1 Create-only
 
-New pipeline only → schedules created, timestamps inserted.
-
-### 1.3.2 Update-only
-
-Existing pipeline with newer `updated_at` → schedule updated in Temporal.
+New pipeline only → Temporal schedule created for it.
 
 ### 1.3.3 Delete-only
 
-Pipeline removed from Mongo → schedule deleted from Temporal.
-
-### 1.4.1 Conflict ordering
-
-Pipeline update and slot delete on same OPC server → slot insert before delete (production ordering).
+Pipeline removed from Mongo → previously-created Temporal schedule deleted.
 
 ### 1.5.1 Empty pipelines
 
-No active pipelines → orphan schedules removed, no new orchestration writes.
+No active pipelines → no `orchestrated_schedules` documents written, even with a pre-existing
+unrelated Temporal schedule present.
 
 ### 1.6.1 TTL index bootstrap
 
@@ -65,18 +59,14 @@ ERROR notification → one SMTP message, one `log_report` row, Redis cache key.
 
 Second run with same data and cache seeded → no extra email or log row.
 
-### A.1.3 Persistent escalation
-
-Alert past `notification_ttl` with cache cleared → new email sent.
-
 ### A.2.1 Group filtering
 
-Receiver group `levels` / `ignore_models_id` honored. The query-time half
-(union of `levels`, intersection of `ignore_models_id` across active groups)
-is covered at the subworkflow level — see `LoadNotificationPackage` below.
-The per-group half (each group only receiving its own precise slice) is not
-yet implemented in production code (SIENTIAPDE-2012) and so is not yet
-exercised here.
+Receiver group `levels` / `ignore_models_id` honored, both halves now covered:
+the query-time union/intersection across active groups (SIENTIAPDE-2011, see
+`LoadNotificationPackage` below) and the per-group precise slice
+(SIENTIAPDE-2012) — a group only receives notifications matching its own
+`levels`/`ignore_models_id`, even when the query-time filter widened the fetch
+to satisfy another active group's broader configuration.
 
 ### A.3.1 Empty queue
 
@@ -110,6 +100,8 @@ No notifications → no SMTP, no Postgres row.
 - Prior timestamp → only newer notifications returned.
 - Query-time filter (SIENTIAPDE-2011) → union of active groups' `levels`, intersection of their `ignore_models_id`; a model ignored by only one of several active groups is not excluded from the fetch.
 - Empty Mongo → no Redis timestamp write.
+- Batch size cap (SIENTIAPDE-2013) → with 5 pending notifications and a batch size of 2, each run returns exactly the next-oldest 2 (sorted ascending by timestamp), and the Redis cursor advances only to what was actually returned, so three successive runs drain the backlog in order instead of skipping anything.
+- Batch size 0 means unlimited (SIENTIAPDE-2013) → the configured default (`0`) does not cap the query; 12 pending notifications all come back in one run, verified against a real MongoDB instance rather than assumed from pymongo's documented convention.
 
 ### ProcessNotifications — `e2e/test_subworkflow_process_notifications.py`
 

e2e/test_alerts_main_workflow.py

@@ -12,6 +12,7 @@ from e2e.helpers import (
     count_log_report_rows,
     default_notification,
     default_receiver_group,
+    extract_html_body,
     fetch_log_report,
     load_scenario_input,
     make_workflow_id,
@@ -92,6 +93,95 @@ async def test_scenario_a_1_2_duplicate_suppressed(
     assert count_log_report_rows(postgres_engine, 'Alerts') == 0
 
 
+@pytest.mark.e2e
+@pytest.mark.asyncio
+async def test_scenario_a_2_1_per_group_levels_and_ignore_models_id(
+    temporal_env: WorkflowEnvironment,
+    orchestrator_worker,
+    stub_workers,
+    mongo_uri: str,
+    smtp_server,
+):
+    """
+    A.2.1 (SIENTIAPDE-2012): each receiver group gets only its own precise
+    slice of notifications, on top of SIENTIAPDE-2011's query-time fetch.
+
+    Two active groups widen the query-time union beyond Alerts' ERROR-only
+    default: group-errors (levels=['ERROR']) and group-warnings
+    (levels=['WARNING'], ignore_models_id=['model-x']). Because the query-time
+    filter only unions/intersects across ALL active groups, both ERROR and
+    WARNING notifications get fetched for every group, and model-x is not
+    excluded at the query level either (group-errors has no ignore_models_id
+    restriction, so the intersection is empty). The per-group filter added in
+    this subtask is what actually narrows each group back down to what it
+    configured.
+    """
+    seed_receiver_groups(
+        mongo_uri,
+        E2E_DATABASE,
+        [
+            default_receiver_group(
+                group_name='group-errors', contents=['core_alerts'], levels=['ERROR']
+            ),
+            default_receiver_group(
+                group_name='group-warnings',
+                contents=['core_alerts'],
+                levels=['WARNING'],
+                ignore_models_id=['model-x'],
+            ),
+        ],
+    )
+    seed_notifications(
+        mongo_uri,
+        E2E_DATABASE,
+        [
+            default_notification(
+                notification_id='err-a', level='ERROR', model_name='model-a', model_id='model-a'
+            ),
+            default_notification(
+                notification_id='warn-a',
+                level='WARNING',
+                model_name='model-a',
+                model_id='model-a',
+                timestamp=datetime(2024, 6, 1, 12, 1, 0, tzinfo=UTC),
+            ),
+            default_notification(
+                notification_id='warn-x',
+                level='WARNING',
+                model_name='model-x',
+                model_id='model-x',
+                timestamp=datetime(2024, 6, 1, 12, 2, 0, tzinfo=UTC),
+            ),
+        ],
+    )
+
+    await start_and_await_workflow(
+        temporal_env.client,
+        Alerts.run,
+        load_scenario_input('alerts_happy_path.json'),
+        make_workflow_id('alerts-2012'),
+    )
+
+    # One email per group -- each group ended up with exactly one notification.
+    assert len(smtp_server.messages) == 2
+    bodies = [extract_html_body(message) for message in smtp_server.messages]
+    combined = '\n'.join(bodies)
+
+    # group-errors (levels=['ERROR']): received err-a, never warn-a/warn-x
+    # despite both being fetched by the widened query-time union.
+    errors_body = next(b for b in bodies if 'Errors detected:' in b)
+    assert 'Warnings detected:' not in errors_body
+
+    # group-warnings (levels=['WARNING'], ignore_models_id=['model-x']):
+    # received warn-a only -- warn-x was fetched (query-time intersection is
+    # empty, since group-errors doesn't ignore any model) but dropped here by
+    # the per-group ignore_models_id check.
+    warnings_body = next(b for b in bodies if 'Warnings detected:' in b)
+    assert 'Errors detected:' not in warnings_body
+    assert 'model-a' in combined
+    assert 'model-x' not in combined
+
+
 @pytest.mark.e2e
 @pytest.mark.asyncio
 async def test_scenario_a_3_1_empty_queue(

e2e/test_reports_main_workflow.py

@@ -1,7 +1,6 @@
 """E2E tests for the Reports main workflow."""
 
 from datetime import UTC, datetime
-from email.message import EmailMessage, Message
 
 import pytest
 from temporalio.testing import WorkflowEnvironment
@@ -11,6 +10,7 @@ from e2e.helpers import (
     count_log_report_rows,
     default_notification,
     default_receiver_group,
+    extract_html_body,
     load_scenario_input,
     make_workflow_id,
     seed_notifications,
@@ -20,41 +20,6 @@ from e2e.helpers import (
 from orchestrator.workflows.reports import Reports
 
 
-def _extract_html_body(message: EmailMessage | Message) -> str:
-    """
-    Return the text/html portion of an email message as a decoded string.
-
-    Walks every part looking for the first text/html payload, decoding it
-    according to the part's transfer encoding and charset. Falls back to
-    the message's own ``get_content``/raw payload when no HTML part is
-    present so callers can still inspect plain-text reports.
-
-    Args:
-        message: Captured email message returned by the test SMTP server.
-
-    Return:
-        str: HTML body content, or an empty string when nothing decodable
-            is found.
-    """
-    if message.is_multipart():
-        for part in message.walk():
-            if part.get_content_type() != 'text/html':
-                continue
-            payload = part.get_payload(decode=True)
-            if payload is None:
-                continue
-            charset = part.get_content_charset() or 'utf-8'
-            return payload.decode(charset, errors='replace')
-    payload = message.get_payload(decode=True)
-    if payload is not None:
-        charset = message.get_content_charset() or 'utf-8'
-        return payload.decode(charset, errors='replace')
-    try:
-        return message.get_content()
-    except (AttributeError, KeyError):
-        return str(message.get_payload() or '')
-
-
 @pytest.mark.e2e
 @pytest.mark.asyncio
 async def test_scenario_r_1_1_happy_path(
@@ -99,7 +64,7 @@ async def test_scenario_r_1_1_happy_path(
     )
 
     assert len(smtp_server.messages) == 1
-    body = _extract_html_body(smtp_server.messages[0])
+    body = extract_html_body(smtp_server.messages[0])
     assert 'Errors detected:' in body
     assert 'Warnings detected:' in body
     assert 'Infos detected:' in body
@@ -138,7 +103,7 @@ async def test_scenario_r_1_2_error_section_only(
     )
 
     assert smtp_server.messages, 'Expected at least one report email'
-    body = _extract_html_body(smtp_server.messages[0])
+    body = extract_html_body(smtp_server.messages[0])
     assert 'Errors detected:' in body
     assert 'Warnings detected:' not in body
     assert 'Infos detected:' not in body

e2e/test_subworkflow_load_notification_package.py

@@ -228,3 +228,114 @@ async def test_load_package_empty_mongo_no_redis_write(
     )
 
     assert redis_client.get('notification_last_timestamp:Alerts') is None
+
+
+@pytest.mark.e2e
+@pytest.mark.asyncio
+async def test_load_package_batch_size_caps_and_returns_oldest_first(
+    temporal_env: WorkflowEnvironment,
+    orchestrator_worker,
+    stub_workers,
+    mongo_uri: str,
+    redis_client: Redis,
+    test_activities,
+):
+    """
+    SIENTIAPDE-2013: a capped run returns the oldest pending documents first,
+    and the incremental cursor picks up the remainder on the next run.
+
+    Five notifications exist, but the batch size is capped at 2. Each run must
+    return exactly the next-oldest 2 (never an arbitrary subset), and the
+    Redis cursor must advance only to the max timestamp of what was actually
+    returned -- not the true max across all 5 -- or the skipped documents
+    would never be fetched again.
+    """
+    test_activities.notification_queue_batch_size = 2
+
+    seed_receiver_groups(mongo_uri, E2E_DATABASE, [default_receiver_group()])
+    seed_notifications(
+        mongo_uri,
+        E2E_DATABASE,
+        [
+            default_notification(
+                notification_id=f'n{i}',
+                timestamp=datetime(2024, 6, 1, 12, i, 0, tzinfo=UTC),
+            )
+            for i in range(1, 6)
+        ],
+    )
+
+    input_data = load_scenario_input('subworkflow_load_notification_package.json')
+    _build_metadata(input_data)
+
+    first_run = await start_and_await_workflow(
+        temporal_env.client,
+        LoadNotificationPackage.run,
+        input_data,
+        make_workflow_id('load-pkg-batch-1'),
+    )
+    assert [n['notification_id'] for n in first_run['notification_package']] == ['n1', 'n2']
+
+    second_run = await start_and_await_workflow(
+        temporal_env.client,
+        LoadNotificationPackage.run,
+        input_data,
+        make_workflow_id('load-pkg-batch-2'),
+    )
+    assert [n['notification_id'] for n in second_run['notification_package']] == ['n3', 'n4']
+
+    third_run = await start_and_await_workflow(
+        temporal_env.client,
+        LoadNotificationPackage.run,
+        input_data,
+        make_workflow_id('load-pkg-batch-3'),
+    )
+    assert [n['notification_id'] for n in third_run['notification_package']] == ['n5']
+
+
+@pytest.mark.e2e
+@pytest.mark.asyncio
+async def test_load_package_batch_size_zero_means_unlimited(
+    temporal_env: WorkflowEnvironment,
+    orchestrator_worker,
+    stub_workers,
+    mongo_uri: str,
+    test_activities,
+):
+    """
+    SIENTIAPDE-2013: notification_queue_batch_size=0 (the config default) must
+    NOT cap the query. This exercises pymongo's own documented convention --
+    `Collection.find(..., limit=0)` is equivalent to no limit at all -- against
+    a real MongoDB instance rather than trusting the convention on faith.
+
+    12 notifications is deliberately more than any other test in this file
+    seeds, so a regression that silently caps at some smaller default (e.g. if
+    `limit=0` were ever misread as "return nothing", or pymongo's own behavior
+    changed) would fail loudly here instead of passing by coincidence.
+    """
+    test_activities.notification_queue_batch_size = 0
+
+    seed_receiver_groups(mongo_uri, E2E_DATABASE, [default_receiver_group()])
+    seed_notifications(
+        mongo_uri,
+        E2E_DATABASE,
+        [
+            default_notification(
+                notification_id=f'n{i}',
+                timestamp=datetime(2024, 6, 1, 12, i, 0, tzinfo=UTC),
+            )
+            for i in range(1, 13)
+        ],
+    )
+
+    input_data = load_scenario_input('subworkflow_load_notification_package.json')
+    _build_metadata(input_data)
+
+    result = await start_and_await_workflow(
+        temporal_env.client,
+        LoadNotificationPackage.run,
+        input_data,
+        make_workflow_id('load-pkg-batch-unlimited'),
+    )
+
+    assert len(result['notification_package']) == 12

orchestrator/activities/activities.py

@@ -84,6 +84,7 @@ class Activities(TemporalManager, SlotManager, Formatters, MongoDB, Email, Postg
             connection_string=mongodb_config['connection_string'],
             database_name=mongodb_config['database_name'],
             ttl_index_seconds=mongodb_config['ttl_index_seconds'],
+            notification_queue_batch_size=mongodb_config['notification_queue_batch_size'],
             logger=logger,
             notification_handler=notification_handler,
             metrics_controller=metrics_controller,

orchestrator/activities/formatters.py

@@ -824,6 +824,8 @@ class Formatters(SientiaMonitoring):
                     - group_name (str): Name of the receiver group
                     - contents (list): Content types to include (must contain "reports")
                     - ignore (list, optional): Notification IDs to exclude
+                    - levels (list, optional): Notification levels to include
+                    - ignore_models_id (list, optional): Model IDs to exclude
 
         Returns:
             dict[str, Any]: Filtered receiver groups with their notifications, keyed by group_name.
@@ -847,6 +849,8 @@ class Formatters(SientiaMonitoring):
             already_added_keys = []
 
             ignore_list = receiver_group.get('ignore', [])
+            levels_list = receiver_group.get('levels')
+            ignore_models_id_list = receiver_group.get('ignore_models_id') or []
 
             for notification in notification_package:
                 alert_type = 'reports'
@@ -859,6 +863,11 @@ class Formatters(SientiaMonitoring):
                     alert_type in receiver_group['contents']
                     and notification_id not in ignore_list
                     and key not in already_added_keys
+                    and (not levels_list or notification.get('level') in levels_list)
+                    and (
+                        not ignore_models_id_list
+                        or notification.get('model_id') not in ignore_models_id_list
+                    )
                 ):
                     receiver_groups[group_name]['notifications'].append(notification)
                     already_added_keys.append(key)

orchestrator/activities/mongo_db.py

@@ -31,6 +31,8 @@ class MongoDB(SientiaMonitoring):
         ttl_index_seconds (int): TTL index duration in seconds
         logger (Logger): Application logger instance
         notification_handler (NotificationHandler): Notification management handler
+        notification_queue_batch_size (int): Hard cap on documents fetched per
+            `load_latest_data` run (SIENTIAPDE-2013). `0` means no limit.
     """
 
     def __init__(
@@ -41,6 +43,7 @@ class MongoDB(SientiaMonitoring):
         logger: Logger,
         notification_handler: NotificationHandler,
         metrics_controller: MetricsController,
+        notification_queue_batch_size: int = 0,
     ):
         SientiaMonitoring.__init__(
             self,
@@ -61,6 +64,7 @@ class MongoDB(SientiaMonitoring):
         )
 
         self.ttl_index_seconds = ttl_index_seconds
+        self.notification_queue_batch_size = notification_queue_batch_size
 
         # Initialize MongoDB client here (omitted for brevity)
         logger.info('MongoDB connection initialized')
@@ -447,6 +451,10 @@ class MongoDB(SientiaMonitoring):
         filtering by timestamp to enable incremental data processing. It
         handles connection management and provides comprehensive error reporting.
 
+        The query is capped at ``notification_queue_batch_size`` documents and sorted by
+        ``timestamp`` ascending, so a capped run returns the oldest pending documents first --
+        the caller advances its cursor only to what was actually returned, so nothing is skipped.
+
         Args:
             input_data (dict[str, Any]): Activity input parameters.
                 Required fields:
@@ -488,7 +496,13 @@ class MongoDB(SientiaMonitoring):
 
             self.debug(f'Data filter: {data_filter}', metadata=metadata)
 
-            data = self.mongo_db_repository.find(collection_name, data_filter, metadata)
+            data = self.mongo_db_repository.find(
+                collection_name,
+                data_filter,
+                metadata,
+                limit=self.notification_queue_batch_size,
+                sort=[('timestamp', 1)],
+            )
 
             self.debug(f'Collected: {data}', metadata=metadata)
 

orchestrator/activities/slot_manager.py

@@ -396,6 +396,8 @@ class SlotManager(SientiaMonitoring):
                     - group_name (str): Name of the receiver group
                     - contents (list): Alert types to include (core_alerts, persistent_alerts)
                     - ignore (list, optional): Notification IDs to exclude
+                    - levels (list, optional): Notification levels to include
+                    - ignore_models_id (list, optional): Model IDs to exclude
                 - notification_ttl (int): Seconds before considering notification persistent
 
         Returns:
@@ -415,48 +417,118 @@ class SlotManager(SientiaMonitoring):
 
         for receiver_group in sending_configs:
             group_name = receiver_group['group_name']
-            receiver_groups[group_name] = {**receiver_group, 'notifications': []}
-            receiver_groups[group_name]['notifications'] = []
-
-            already_added_keys = []
-
-            ignore_list = receiver_group.get('ignore', [])
-
-            for notification in notification_package:
-                alert_type = 'do_nothing'
-                notification_id = notification['notification_id']
-                # Check if notification was recently sent
-                key = f'{notification["trigger"]}:{notification_id}'
-
-                last_sent = self.redis_repository.get(key)
-
-                if last_sent is None:
-                    alert_type = 'core_alerts'
-
-                else:
-                    last_sent = datetime.strptime(last_sent, DATETIME_FORMAT_MS_WITH_TZ)
-
-                    # Check if "notification_ttl" seconds has passed since last sent
-                    if (now() - last_sent) > timedelta(seconds=notification_ttl):
-                        alert_type = 'persistent_alerts'
-
-                # Check if this group must be notified
-                if (
-                    alert_type in receiver_group['contents']
-                    and notification_id not in ignore_list
-                    and key not in already_added_keys
-                ):
-                    receiver_groups[group_name]['notifications'].append(notification)
-                    already_added_keys.append(key)
+            notifications = self._filter_group_notifications(
+                receiver_group, notification_package, notification_ttl
+            )
+            receiver_groups[group_name] = {**receiver_group, 'notifications': notifications}
 
         # Remove groups with no notifications
-        receiver_groups = {
+        return {
             group_name: group
             for group_name, group in receiver_groups.items()
             if group['notifications']
         }
 
-        return receiver_groups
+    def _filter_group_notifications(
+        self,
+        receiver_group: dict[str, Any],
+        notification_package: list[dict[str, Any]],
+        notification_ttl: int,
+    ) -> list[dict[str, Any]]:
+        """
+        Select the notifications this receiver group should get.
+
+        Args:
+            receiver_group (dict[str, Any]): The receiver group configuration.
+            notification_package (list[dict[str, Any]]): Candidate notifications.
+            notification_ttl (int): Seconds before considering a notification persistent.
+
+        Returns:
+            list[dict[str, Any]]: Notifications matching this group's filters.
+        """
+        already_added_keys: list[str] = []
+        ignore_list = receiver_group.get('ignore', [])
+        levels_list = receiver_group.get('levels')
+        ignore_models_id_list = receiver_group.get('ignore_models_id') or []
+
+        notifications = []
+        for notification in notification_package:
+            notification_id = notification['notification_id']
+            key = f'{notification["trigger"]}:{notification_id}'
+            alert_type = self._determine_alert_type(key, notification_ttl)
+
+            if (
+                alert_type in receiver_group['contents']
+                and self._check_keys_and_notif_id(
+                    notification_id, ignore_list, key, already_added_keys
+                )
+                and self._check_levels_and_ignore_models_id(
+                    notification, levels_list, ignore_models_id_list
+                )
+            ):
+                notifications.append(notification)
+                already_added_keys.append(key)
+
+        return notifications
+
+    def _determine_alert_type(self, key: str, notification_ttl: int) -> str:
+        """
+        Classify a notification as new, persistent, or not-yet-due.
+
+        Args:
+            key (str): The Redis cache key for this notification.
+            notification_ttl (int): Seconds before considering a notification persistent.
+
+        Returns:
+            str: 'core_alerts', 'persistent_alerts', or 'do_nothing'.
+        """
+        last_sent = self.redis_repository.get(key)
+        if last_sent is None:
+            return 'core_alerts'
+
+        last_sent = datetime.strptime(last_sent, DATETIME_FORMAT_MS_WITH_TZ)
+        if (now() - last_sent) > timedelta(seconds=notification_ttl):
+            return 'persistent_alerts'
+
+        return 'do_nothing'
+
+    @staticmethod
+    def _check_levels_and_ignore_models_id(notification, levels_list, ignore_models_id_list):
+        """
+        Check if the notification's level is in the levels_list and its model_id is not in the ignore_models_id_list.
+
+        Args:
+            notification (dict): The notification to check.
+            levels_list (list, optional): List of levels to include. If None, all levels are included.
+            ignore_models_id_list (list, optional): List of model_ids to exclude. If None, no models are excluded.
+
+        Returns:
+            bool: True if the notification passes the checks, False otherwise.
+        """
+        level_check = not levels_list or notification.get('level') in levels_list
+        model_id_check = (
+            not ignore_models_id_list or notification.get('model_id') not in ignore_models_id_list
+        )
+        return level_check and model_id_check
+
+    @staticmethod
+    def _check_keys_and_notif_id(
+        notification_id, ignore_list, key, already_added_keys: list[str]
+    ) -> bool:
+        """
+        Check if the key has already been added to the list of already added keys.
+
+        Args:
+            notification_id (str): The ID of the notification.
+            ignore_list (list): List of notification IDs to ignore.
+            key (str): The key to check.
+            already_added_keys (list[str]): List of keys that have already been added.
+
+        Returns:
+            bool: True if the key has not been added yet, False otherwise.
+
+        """
+        return notification_id not in ignore_list and key not in already_added_keys
 
     @activity.defn(name='store_notification_cache')
     def store_notification_cache(self, input_data: dict[str, Any]) -> None:

orchestrator/utils/connectors_config.py

@@ -21,7 +21,8 @@ def build_mongodb_config():
     Build MongoDB configuration from environment variables.
 
     Returns:
-        dict: MongoDB configuration with connection string, database name, and TTL index seconds.
+        dict: MongoDB configuration with connection string, database name, TTL index seconds,
+            and the notification queue's per-run batch size cap.
     """
     username = getenv('MONGODB_USERNAME', 'root')
     password = getenv('MONGODB_PASSWORD', 'wKZDbMNU1c')
@@ -32,6 +33,7 @@ def build_mongodb_config():
         'connection_string': connection_string,
         'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
         'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600,
+        'notification_queue_batch_size': int(getenv('NOTIFICATION_QUEUE_BATCH_SIZE', '0')),
     }
 
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "orchestrator"
-version = "0.6.4"
+version = "0.6.6"
 description = "Sientia DataOps Orchestrator - ML Model Orchestration System"
 readme = "README.md"
 requires-python = ">=3.11"

requirements-local.txt

@@ -4,5 +4,5 @@ sqlalchemy
 redis
 pymongo
 jinja2
-git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.2
+git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.3
 prometheus-client

requirements.txt

@@ -4,5 +4,5 @@ sqlalchemy
 redis
 pymongo
 jinja2
-sientia_do>=1.12.2
+sientia_do>=1.12.3
 prometheus-client

tests/orchestrator/activities/test_activities.py

@@ -27,6 +27,7 @@ def test___init__(
         'connection_string': 'mongodb://localhost:27017',
         'database_name': 'test_db',
         'ttl_index_seconds': 3600,
+        'notification_queue_batch_size': 5000,
     }
 
     redis_config = {'host': 'localhost', 'port': 6379, 'username': 'admin', 'password': 'password'}
@@ -89,6 +90,7 @@ def test___init__(
         connection_string='mongodb://localhost:27017',
         database_name='test_db',
         ttl_index_seconds=3600,
+        notification_queue_batch_size=5000,
         logger=logger,
         notification_handler=notification_handler,
         metrics_controller=mock_metrics_controller.return_value,

tests/orchestrator/activities/test_formatters.py

@@ -766,3 +766,143 @@ def test_filter_notification_reports(formatters):
             ],
         }
     }
+
+
+def test_filter_notification_reports_levels_restriction(formatters):
+    # SIENTIAPDE-2012: a group with `levels` set only receives notifications
+    # matching one of those levels.
+    input_data = {
+        **metadata,
+        'notification_package': [
+            {
+                'trigger': 'test_trigger_1',
+                'notification_id': 'test_notification_id_1',
+                'level': 'ERROR',
+            },
+            {
+                'trigger': 'test_trigger_2',
+                'notification_id': 'test_notification_id_2',
+                'level': 'INFO',
+            },
+        ],
+        'sending_configs': [
+            {'group_name': 'test_group_1', 'contents': ['reports'], 'levels': ['ERROR']},
+        ],
+    }
+
+    response = formatters.filter_notification_reports(input_data)
+
+    assert response == {
+        'test_group_1': {
+            'group_name': 'test_group_1',
+            'contents': ['reports'],
+            'levels': ['ERROR'],
+            'notifications': [
+                {
+                    'trigger': 'test_trigger_1',
+                    'notification_id': 'test_notification_id_1',
+                    'level': 'ERROR',
+                },
+            ],
+        },
+    }
+
+
+def test_filter_notification_reports_ignore_models_id_restriction(formatters):
+    # SIENTIAPDE-2012: a group with `ignore_models_id` set does not receive
+    # notifications from those models, but still receives everything else.
+    input_data = {
+        **metadata,
+        'notification_package': [
+            {
+                'trigger': 'test_trigger_1',
+                'notification_id': 'test_notification_id_1',
+                'model_id': 'model_x',
+            },
+            {
+                'trigger': 'test_trigger_2',
+                'notification_id': 'test_notification_id_2',
+                'model_id': 'model_y',
+            },
+        ],
+        'sending_configs': [
+            {
+                'group_name': 'test_group_1',
+                'contents': ['reports'],
+                'ignore_models_id': ['model_x'],
+            },
+        ],
+    }
+
+    response = formatters.filter_notification_reports(input_data)
+
+    assert response == {
+        'test_group_1': {
+            'group_name': 'test_group_1',
+            'contents': ['reports'],
+            'ignore_models_id': ['model_x'],
+            'notifications': [
+                {
+                    'trigger': 'test_trigger_2',
+                    'notification_id': 'test_notification_id_2',
+                    'model_id': 'model_y',
+                },
+            ],
+        },
+    }
+
+
+def test_filter_notification_reports_no_levels_or_ignore_models_id_passes_everything(
+    formatters,
+):
+    # SIENTIAPDE-2012: a group configuring neither field is unaffected -- same
+    # behavior as before this subtask (only `contents`/`ignore` apply).
+    input_data = {
+        **metadata,
+        'notification_package': [
+            {
+                'trigger': 'test_trigger_1',
+                'notification_id': 'test_notification_id_1',
+                'level': 'ERROR',
+                'model_id': 'model_x',
+            },
+            {
+                'trigger': 'test_trigger_2',
+                'notification_id': 'test_notification_id_2',
+                'level': 'INFO',
+                'model_id': 'model_y',
+            },
+        ],
+        'sending_configs': [
+            {'group_name': 'test_group_1', 'contents': ['reports']},
+        ],
+    }
+
+    response = formatters.filter_notification_reports(input_data)
+
+    assert response['test_group_1']['notifications'] == input_data['notification_package']
+
+
+def test_filter_notification_reports_ignore_models_id_explicit_none_does_not_raise(
+    formatters,
+):
+    # receiver_groups has no schema validation -- ignore_models_id can be
+    # explicitly null (key present, value None) instead of omitted. Must not
+    # crash (see the equivalent SIENTIAPDE-2011 fix in load_notification_package).
+    input_data = {
+        **metadata,
+        'notification_package': [
+            {
+                'trigger': 'test_trigger_1',
+                'notification_id': 'test_notification_id_1',
+                'model_id': 'model_x',
+            },
+        ],
+        'sending_configs': [
+            {'group_name': 'test_group_1', 'contents': ['reports'], 'ignore_models_id': None},
+        ],
+    }
+
+    response = formatters.filter_notification_reports(input_data)
+
+    assert response['test_group_1']['notifications'] == input_data['notification_package']

tests/orchestrator/activities/test_mongo_db.py

@@ -31,13 +31,14 @@ def test___init__(mongo_mock):
     notification_handler = MagicMock()
     metrics_controller = AsyncMock()
 
-    MongoDB(
+    mongo = MongoDB(
         connection_string='mongodb://localhost:27017',
         database_name='test_db',
         ttl_index_seconds=3600,
         logger=logger,
         notification_handler=notification_handler,
         metrics_controller=metrics_controller,
+        notification_queue_batch_size=5000,
     )
 
     mongo_mock.assert_called_once_with(
@@ -47,6 +48,22 @@ def test___init__(mongo_mock):
         notification_handler=notification_handler,
         metrics_controller=metrics_controller,
     )
+    assert mongo.notification_queue_batch_size == 5000
+
+
+@patch('orchestrator.activities.mongo_db.MongoDBRepository')
+def test___init___notification_queue_batch_size_defaults_to_zero(mongo_mock):
+    """0 means no limit, matching the underlying find()'s own convention."""
+    mongo = MongoDB(
+        connection_string='mongodb://localhost:27017',
+        database_name='test_db',
+        ttl_index_seconds=3600,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+
+    assert mongo.notification_queue_batch_size == 0
 
 
 @patch('orchestrator.activities.mongo_db.SientiaMonitoring')
@@ -480,6 +497,8 @@ def test_load_latest_data_none_last_data_timestamp(mongo_db):
         'test_collection',
         {'level': 'ERROR'},
         {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+        limit=0,
+        sort=[('timestamp', 1)],
     )
 
     assert result == [
@@ -487,6 +506,29 @@ def test_load_latest_data_none_last_data_timestamp(mongo_db):
     ]
 
 
+def test_load_latest_data_passes_configured_batch_size_as_limit(mongo_db):
+    """SIENTIAPDE-2013: the configured notification_queue_batch_size caps the query."""
+    mongo_db.notification_queue_batch_size = 100
+    mongo_db.mongo_db_repository.find = MagicMock(return_value=[])
+
+    mongo_db.load_latest_data(
+        {
+            'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+            'collection_name': 'test_collection',
+            'last_data_timestamp': None,
+            'base_data_filter': {'level': 'ERROR'},
+        }
+    )
+
+    mongo_db.mongo_db_repository.find.assert_called_once_with(
+        'test_collection',
+        {'level': 'ERROR'},
+        {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+        limit=100,
+        sort=[('timestamp', 1)],
+    )
+
+
 def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
     """Test load_latest_data compares native datetime against native datetime in $gt"""
 
@@ -516,6 +558,8 @@ def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
             'timestamp': {'$gt': datetime(2023, 1, 1, 12, 0, 0, tzinfo=UTC)},
         },
         {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+        limit=0,
+        sort=[('timestamp', 1)],
     )
 
     assert result == [
@@ -529,7 +573,7 @@ def test_load_latest_data_run_boundary_no_skip_or_duplicate(mongo_db):
 
     boundary_timestamp = datetime(2023, 1, 1, 12, 0, 0, tzinfo=UTC)
 
-    def fake_find(collection_name, data_filter, metadata):
+    def fake_find(collection_name, data_filter, metadata, limit=0, sort=None):
         gt = data_filter.get('timestamp', {}).get('$gt')
         all_docs = [
             {

tests/orchestrator/activities/test_slot_manager.py

@@ -389,6 +389,162 @@ def test_filter_notification_alerts(slot_manager):
     }
 
 
+def test_filter_notification_alerts_levels_restriction(slot_manager):
+    # SIENTIAPDE-2012: a group with `levels` set only receives notifications
+    # matching one of those levels.
+    slot_manager.redis_repository.get = MagicMock(side_effect=[None, None])
+
+    input_data = {
+        **metadata,
+        'notification_package': [
+            {
+                'trigger': 'test_trigger_1',
+                'notification_id': 'test_notification_id_1',
+                'level': 'ERROR',
+            },
+            {
+                'trigger': 'test_trigger_2',
+                'notification_id': 'test_notification_id_2',
+                'level': 'WARNING',
+            },
+        ],
+        'sending_configs': [
+            {'group_name': 'test_group_1', 'contents': ['core_alerts'], 'levels': ['ERROR']},
+        ],
+        'notification_ttl': 300,
+        'mail_type': 'test_mail_type',
+    }
+
+    response = slot_manager.filter_notification_alerts(input_data)
+
+    assert response == {
+        'test_group_1': {
+            'group_name': 'test_group_1',
+            'contents': ['core_alerts'],
+            'levels': ['ERROR'],
+            'notifications': [
+                {
+                    'trigger': 'test_trigger_1',
+                    'notification_id': 'test_notification_id_1',
+                    'level': 'ERROR',
+                },
+            ],
+        },
+    }
+
+
+def test_filter_notification_alerts_ignore_models_id_restriction(slot_manager):
+    # SIENTIAPDE-2012: a group with `ignore_models_id` set does not receive
+    # notifications from those models, but still receives everything else.
+    slot_manager.redis_repository.get = MagicMock(side_effect=[None, None])
+
+    input_data = {
+        **metadata,
+        'notification_package': [
+            {
+                'trigger': 'test_trigger_1',
+                'notification_id': 'test_notification_id_1',
+                'model_id': 'model_x',
+            },
+            {
+                'trigger': 'test_trigger_2',
+                'notification_id': 'test_notification_id_2',
+                'model_id': 'model_y',
+            },
+        ],
+        'sending_configs': [
+            {
+                'group_name': 'test_group_1',
+                'contents': ['core_alerts'],
+                'ignore_models_id': ['model_x'],
+            },
+        ],
+        'notification_ttl': 300,
+        'mail_type': 'test_mail_type',
+    }
+
+    response = slot_manager.filter_notification_alerts(input_data)
+
+    assert response == {
+        'test_group_1': {
+            'group_name': 'test_group_1',
+            'contents': ['core_alerts'],
+            'ignore_models_id': ['model_x'],
+            'notifications': [
+                {
+                    'trigger': 'test_trigger_2',
+                    'notification_id': 'test_notification_id_2',
+                    'model_id': 'model_y',
+                },
+            ],
+        },
+    }
+
+
+def test_filter_notification_alerts_no_levels_or_ignore_models_id_passes_everything(
+    slot_manager,
+):
+    # SIENTIAPDE-2012: a group configuring neither field is unaffected -- same
+    # behavior as before this subtask (only `contents`/`ignore` apply).
+    slot_manager.redis_repository.get = MagicMock(side_effect=[None, None])
+
+    input_data = {
+        **metadata,
+        'notification_package': [
+            {
+                'trigger': 'test_trigger_1',
+                'notification_id': 'test_notification_id_1',
+                'level': 'ERROR',
+                'model_id': 'model_x',
+            },
+            {
+                'trigger': 'test_trigger_2',
+                'notification_id': 'test_notification_id_2',
+                'level': 'INFO',
+                'model_id': 'model_y',
+            },
+        ],
+        'sending_configs': [
+            {'group_name': 'test_group_1', 'contents': ['core_alerts']},
+        ],
+        'notification_ttl': 300,
+        'mail_type': 'test_mail_type',
+    }
+
+    response = slot_manager.filter_notification_alerts(input_data)
+
+    assert response['test_group_1']['notifications'] == input_data['notification_package']
+
+
+def test_filter_notification_alerts_ignore_models_id_explicit_none_does_not_raise(
+    slot_manager,
+):
+    # receiver_groups has no schema validation -- ignore_models_id can be
+    # explicitly null (key present, value None) instead of omitted. Must not
+    # crash (see the equivalent SIENTIAPDE-2011 fix in load_notification_package).
+    slot_manager.redis_repository.get = MagicMock(side_effect=[None])
+
+    input_data = {
+        **metadata,
+        'notification_package': [
+            {
+                'trigger': 'test_trigger_1',
+                'notification_id': 'test_notification_id_1',
+                'model_id': 'model_x',
+            },
+        ],
+        'sending_configs': [
+            {'group_name': 'test_group_1', 'contents': ['core_alerts'], 'ignore_models_id': None},
+        ],
+        'notification_ttl': 300,
+        'mail_type': 'test_mail_type',
+    }
+
+    response = slot_manager.filter_notification_alerts(input_data)
+
+    assert response['test_group_1']['notifications'] == input_data['notification_package']
+
+
 def test_store_notification_cache(slot_manager):
     """Test store_notification_cache"""
     test_data = {

tests/orchestrator/utils/test_connectors_config.py

@@ -64,10 +64,12 @@ def test_build_mongo_db_config_with_env_vars():
     environ['MONGODB_URL'] = 'localhost:27018'
     environ['MONGODB_DATABASE_NAME'] = 'test_db'
     environ['MONGODB_TTL_INDEX_HOURS'] = '2'
+    environ['NOTIFICATION_QUEUE_BATCH_SIZE'] = '2000'
     assert build_mongodb_config() == {
         'connection_string': 'mongodb://sientia1:sientia1@localhost:27018',
         'database_name': 'test_db',
         'ttl_index_seconds': 7200,
+        'notification_queue_batch_size': 2000,
     }
 
 
@@ -77,10 +79,12 @@ def test_build_mongo_db_config_with_defaults():
     environ.pop('MONGODB_DATABASE_NAME', None)
     environ.pop('MONGODB_URL', None)
     environ.pop('MONGODB_TTL_INDEX_HOURS', None)
+    environ.pop('NOTIFICATION_QUEUE_BATCH_SIZE', None)
     assert build_mongodb_config() == {
         'connection_string': 'mongodb://root:wKZDbMNU1c@localhost:27018',
         'database_name': 'sientia',
         'ttl_index_seconds': 3600,
+        'notification_queue_batch_size': 0,
     }