SIENTIAPDE-1974

Add e2e tag-database scenarios and cover worker/email/formatter gaps

- Add e2e scenarios 1.9.1-1.9.4 for PI Web API through the tag
  database: scouter and output tag resolution, mixed-connector schedule
  hold, and OPC slot unchanged by PI connectors. Seed connectors and
  connector_tags from a new scenario input.
- Tighten the happy-path e2e to assert both schedules and that no PI
  output config is produced when the tag database is unused.
- Drop dead `if self.server` / `if notification_handler` guards in the
  email reconnect and worker shutdown paths, since both are always set.
- Log the traceback inside the message on worker failure, as the
  sientia_do Logger.error does not accept exc_info.
- Add unit tests for the worker failure exit, Prometheus startup
  failure, email reconnect without a password, and formatter, mongo,
  temporal manager and orchestrator edge cases.

tests/orchestrator/activities/test_email.py

@@ -326,3 +326,37 @@ def test_send_email(mimemultipart, mimetext, email):
     )
 
     assert email.try_send_email.call_count == 2
+
+
+@patch('orchestrator.activities.email.EmailBuilder')
+@patch('orchestrator.activities.email.smtplib')
+def test___init___without_smtp_server(smtplib, email_builder):
+    email = Email(
+        sender_email='test@test.com',
+        sender_password='test',
+        smtp_server=None,
+        smtp_port=587,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+
+    smtplib.SMTP.assert_not_called()
+    assert not hasattr(email, 'server')
+
+
+@patch('orchestrator.activities.email.smtplib.SMTP')
+def test_try_send_email_reconnect_without_password(smtp, email):
+    email.sender_password = None
+    email.server.sendmail = MagicMock(side_effect=SMTPServerDisconnected('test'))
+    email.server.quit = MagicMock()
+
+    msg = MagicMock()
+
+    email.try_send_email(msg, 'test')
+
+    smtp.return_value.starttls.assert_not_called()
+    smtp.return_value.login.assert_not_called()
+    smtp.return_value.sendmail.assert_called_once_with(
+        'test@test.com', 'test', msg.as_string.return_value
+    )

tests/orchestrator/activities/test_formatters.py

@@ -1649,3 +1649,148 @@ def test_create_schedule_config_leaves_held_schedule_untouched(formatters):
         'to_create': {'scouter': {}, 'laborious': {}},
         'to_delete': {'scouter': [], 'laborious': []},
     }
+
+
+def test_format_schedule_config_groups_schedules_of_one_namespace(formatters):
+    result = formatters.format_schedule_config(
+        {
+            'schedule_config': [
+                {'namespace': 'laborious', 'schedule_name': 'a', 'updated_at': '2021-01-01'},
+                {'namespace': 'laborious', 'schedule_name': 'b', 'updated_at': '2021-01-02'},
+            ],
+            **metadata,
+        }
+    )
+
+    assert result == {'laborious': {'a': '2021-01-01', 'b': '2021-01-02'}}
+
+
+def test_report_schedule_orchestration_success_only_and_empty_types(formatters):
+    formatters.send_success_report = MagicMock()
+    formatters.send_error_report = MagicMock()
+
+    formatters.report_schedule_orchestration(
+        {
+            **metadata,
+            'created_schedules': [
+                {'namespace': 'laborious', 'schedule_name': 'a', 'success': True}
+            ],
+            'updated_schedules': [],
+            'deleted_schedules': [],
+        }
+    )
+
+    formatters.send_success_report.assert_called_once()
+    assert formatters.send_success_report.call_args.kwargs['notification_id'] == (
+        'REPORT_ORCHESTRATION_CREATED_SCHEDULES'
+    )
+    formatters.send_error_report.assert_not_called()
+
+
+def test_report_schedule_orchestration_errors_only(formatters):
+    formatters.send_success_report = MagicMock()
+    formatters.send_error_report = MagicMock()
+
+    formatters.report_schedule_orchestration(
+        {
+            **metadata,
+            'created_schedules': [],
+            'updated_schedules': [],
+            'deleted_schedules': [
+                {
+                    'namespace': 'scouter',
+                    'schedule_name': 'b',
+                    'success': False,
+                    'message': 'boom',
+                    'attachment': None,
+                }
+            ],
+        }
+    )
+
+    formatters.send_success_report.assert_not_called()
+    formatters.send_error_report.assert_called_once()
+    assert formatters.send_error_report.call_args.kwargs['notification_id'] == (
+        'REPORT_ORCHESTRATION_DELETED_SCHEDULES_ERROR'
+    )
+
+
+def test_report_slot_orchestration_success_only_inserts(formatters):
+    formatters.send_success_report = MagicMock()
+    formatters.send_error_report = MagicMock()
+
+    formatters.report_slot_orchestration(
+        {**metadata, 'inserted_slots': {'1': {'success': True}}, 'deleted_slots': {}}
+    )
+
+    formatters.send_success_report.assert_called_once_with(
+        metadata=metadata['metadata'],
+        message='Inserted slots: \n 1',
+        notification_id='REPORT_ORCHESTRATION_INSERTED_SLOTS',
+    )
+    formatters.send_error_report.assert_not_called()
+
+
+def test_report_slot_orchestration_errors_only_deletes(formatters):
+    formatters.send_success_report = MagicMock()
+    formatters.send_error_report = MagicMock()
+    deleted = {'2': {'success': False, 'error': 'boom'}}
+
+    formatters.report_slot_orchestration(
+        {**metadata, 'inserted_slots': {}, 'deleted_slots': deleted}
+    )
+
+    formatters.send_success_report.assert_not_called()
+    formatters.send_error_report.assert_called_once_with(
+        metadata=metadata['metadata'],
+        message='Failed to delete slots: \n 2',
+        notification_id='REPORT_ORCHESTRATION_DELETED_SLOTS',
+        attachment=deleted,
+    )
+
+
+def test_report_slot_orchestration_errors_only_inserts_success_only_deletes(formatters):
+    formatters.send_success_report = MagicMock()
+    formatters.send_error_report = MagicMock()
+
+    formatters.report_slot_orchestration(
+        {
+            **metadata,
+            'inserted_slots': {'1': {'success': False, 'error': 'boom'}},
+            'deleted_slots': {'2': {'success': True}},
+        }
+    )
+
+    assert [c.kwargs['notification_id'] for c in formatters.send_error_report.call_args_list] == [
+        'REPORT_ORCHESTRATION_INSERTED_SLOTS'
+    ]
+    assert [c.kwargs['notification_id'] for c in formatters.send_success_report.call_args_list] == [
+        'REPORT_ORCHESTRATION_DELETED_SLOTS'
+    ]
+
+
+def test_format_log_report_same_notification_twice_in_one_group(formatters):
+    notification = {
+        'notification_id': 'n1',
+        'trigger': 't1',
+        'timestamp': '2021-01-01',
+        'message': 'm',
+        'level': 'ERROR',
+        'block': 'b',
+        'pipeline': 'p',
+        'project': 'pr',
+        'model_name': 'mn',
+        'model_id': 'mi',
+    }
+
+    result = formatters.format_log_report(
+        {
+            **metadata,
+            'receiver_groups': {
+                'g1': {'notifications': [notification, dict(notification)], 'status': 'sent'}
+            },
+            'mail_type': 'alerts',
+        }
+    )
+
+    assert [row['groups'] for row in DataFrame(result).to_dict('records')] == [['g1']]

tests/orchestrator/activities/test_mongo_db.py

@@ -704,3 +704,48 @@ def test_load_latest_data_error(mongo_db):
             level=NotificationLevel.ERROR,
             attachment_content=ANY,
         )
+
+
+def test_find_documents_in_mongodb_leaves_documents_without_timestamp_field(mongo_db):
+    mongo_db.mongo_db_repository.find = MagicMock(return_value=[{'name': 'no-timestamp'}])
+
+    result = mongo_db.find_documents_in_mongodb(
+        {'query': {'collection': 'test_collection'}, 'timestamp_fields': ['timestamp']}
+    )
+
+    assert result == [{'name': 'no-timestamp'}]
+
+
+def test_aggregate_documents_in_mongodb_leaves_documents_without_timestamp_field(mongo_db):
+    mongo_db.mongo_db_repository.aggregate = MagicMock(return_value=[{'name': 'no-timestamp'}])
+
+    result = mongo_db.aggregate_documents_in_mongodb(
+        {
+            'query': {'collection': 'test_collection', 'aggregation': [{'$match': {}}]},
+            'timestamp_fields': ['timestamp'],
+        }
+    )
+
+    assert result == [{'name': 'no-timestamp'}]
+
+
+@patch('orchestrator.activities.mongo_db.now')
+def test_create_pipelines_timestamps_all_failed_inserts_nothing(_now_mock, mongo_db):
+    mongo_db.mongo_db_repository.insert_many = MagicMock()
+
+    mongo_db.create_pipelines_timestamps(
+        {'created_pipelines': [{'schedule_name': 'a', 'namespace': 'n', 'success': False}]}
+    )
+
+    mongo_db.mongo_db_repository.insert_many.assert_not_called()
+
+
+def test_create_collection_with_ttl_index_skips_pipelines_without_topic(mongo_db):
+    database = mongo_db.mongo_db_repository.database
+    database.list_collection_names = MagicMock(return_value=[])
+
+    mongo_db.create_collection_with_ttl_index(
+        {**metadata, 'pipelines': {'pi-scouter': {'model_tags': {}}, 'held': {'held': True}}}
+    )
+
+    database.create_collection.assert_not_called()

tests/orchestrator/activities/test_temporal_manager.py

@@ -723,3 +723,23 @@ async def test_create_schedules_drift_keeps_default_skip_overlap(
     mock_schedule.assert_called_once()
     policy = mock_schedule.call_args.kwargs.get('policy')
     assert policy is None or policy.overlap == ScheduleOverlapPolicy.SKIP
+
+
+@mark.asyncio
+async def test_normalize_schedules_keeps_schedules_known_in_mongo(temporal_manager):
+    """An orchestrated schedule still tracked in orchestrated_schedules is not deleted"""
+    input_data = {
+        'orchestrated_schedules': {
+            'scouter': {'test-schedule-id': '2021-01-01'},
+            'laborious': {'test-schedule-id': '2021-01-01'},
+        },
+        **metadata,
+    }
+    for client in temporal_manager.temporal_clients.values():
+        client.list_schedules = AsyncMock(return_value=async_iter())
+        client.get_schedule_handle = MagicMock()
+
+    await temporal_manager.normalize_schedules(input_data)
+
+    for client in temporal_manager.temporal_clients.values():
+        client.get_schedule_handle.assert_not_called()

tests/orchestrator/utils/test_orchestrator_functions.py

@@ -1515,3 +1515,44 @@ def test_pi_web_api_scouter_connection_carries_only_the_secret_reference():
 
     assert result['connection']['auth'] == {'type': 'bearer', 'secret': PI_SECRET}
     assert 'leaked-credential' not in str(result)
+
+
+def test_predictions_batch_groups_write_tags_by_server_and_type():
+    config = {
+        'workflow_type': 'predictions_batch',
+        'schedule_name': 'batch',
+        'model_id': 'model_id',
+        'model': {'name': 'model_name'},
+        'query': 'SELECT 1',
+        'write_tags': [
+            {'server_id': '1', 'type': 'prediction', 'addr': 'ns=2;s=P1'},
+            {'server_id': '1', 'type': 'prediction', 'addr': 'ns=2;s=P2', 'data_type': 'int'},
+            # Neither prediction nor confidence: the server is known, the tag is ignored
+            {'server_id': '2', 'type': 'other', 'addr': 'ns=2;s=O1'},
+        ],
+    }
+
+    result = predictions_batch(config)
+
+    assert result['opc_output_config'] == {
+        '1': {
+            'prediction_tags': {
+                'ns=2;s=P1': {'data_type': 'float'},
+                'ns=2;s=P2': {'data_type': 'int'},
+            }
+        },
+        '2': {},
+    }
+
+
+def test_referenced_pi_tag_ids_ignores_malformed_read_tags():
+    assert referenced_pi_tag_ids([_pi_scouter_pipeline([{'tag_id': 42}])]) == []
+
+
+def test_resolve_pi_output_tags_ignores_non_dict_entries():
+    resolved, issues = resolve_pi_output_tags(
+        {'prediction_tags': ['F1DP-P', None, {'PRED-101': 'F1DP-P'}]}, {}, {}
+    )
+
+    assert resolved == {'prediction_tags': {'PRED-101': 'F1DP-P'}}
+    assert not any(issues.values())

tests/orchestrator/worker/test_worker.py

@@ -3,7 +3,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
 
 from pytest import mark
 
-from orchestrator.worker.worker import main
+from orchestrator.worker.worker import main, start_prometheus_server
 from orchestrator.workflows.alerts import Alerts
 from orchestrator.workflows.orchestrator import Orchestrator
 from orchestrator.workflows.reports import Reports
@@ -47,3 +47,43 @@ async def test_main_starts_three_workers_for_expected_workflows(
     assert main_workflows == [Orchestrator, Alerts, Reports]
 
     assert worker_mock.run.call_count == 3
+
+
+@mark.asyncio
+@patch('orchestrator.worker.worker.Runtime')
+@patch('orchestrator.worker.worker.sys')
+@patch('orchestrator.worker.worker.start_http_server')
+@patch('orchestrator.worker.worker.NotificationHandler')
+@patch('orchestrator.worker.worker.Activities')
+@patch('orchestrator.worker.worker.prepare_worker')
+@patch('orchestrator.worker.worker.client.Client.connect', new_callable=AsyncMock)
+async def test_main_exits_with_error_when_a_worker_fails(
+    _connect_mock,
+    prepare_worker_mock,
+    activities_mock,
+    notification_handler_mock,
+    _start_http_server,
+    sys_mock,
+    _runtime_mock,
+):
+    """A worker failure shuts everything down and exits with code 1."""
+    activities_instance = activities_mock.return_value
+    activities_instance.connect_to_temporal = AsyncMock()
+
+    worker_mock = MagicMock()
+    worker_mock.run = AsyncMock(side_effect=RuntimeError('worker crashed'))
+    prepare_worker_mock.return_value = worker_mock
+
+    await main()
+
+    notification_handler_mock.return_value.shutdown.assert_called_once()
+    activities_instance.shutdown.assert_called_once()
+    sys_mock.exit.assert_called_once_with(1)
+
+
+@patch('orchestrator.worker.worker.os._exit')
+@patch('orchestrator.worker.worker.start_http_server', side_effect=OSError('port in use'))
+def test_start_prometheus_server_exits_on_failure(_start_http_server, exit_mock):
+    start_prometheus_server()
+
+    exit_mock.assert_called_once_with(1)

tests/orchestrator/workflows/test_orchestrator.py

@@ -632,3 +632,30 @@ async def test_run_replay_before_pi_web_api_patch(workflow_mock, _ids, _pi_ids,
     schedules_input = _process_schedules_input(workflow_mock)
     assert schedules_input['pi_connectors'] == []
     assert schedules_input['pi_connector_tags'] == []
+
+
+@mark.asyncio
+@patch('orchestrator.workflows.orchestrator.referenced_tag_ids', return_value=[])
+@patch('orchestrator.workflows.orchestrator.workflow', new_callable=AsyncMock)
+async def test_run_without_changes_skips_reports_and_timestamps(workflow_mock, _ids, orchestrator):
+    """Nothing created, updated or deleted: no report and no pipeline timestamp activity"""
+    workflow_mock.patched = MagicMock(return_value=True)
+    workflow_mock.start_activity_method = AsyncMock(return_value=[])
+    input_data = {
+        'pipelines_query': 'SELECT * FROM bucket',
+        'opc_servers_query': 'SELECT * FROM servers',
+        'schedule_name': 'test-schedule-name',
+    }
+
+    await orchestrator.run(input_data)
+
+    started = [c.args[0] for c in workflow_mock.start_activity_method.call_args_list]
+    assert started == [
+        Activities.normalize_schedules,
+        Activities.create_collection_with_ttl_index,
+        Activities.delete_slots,
+        Activities.update_slots,
+        Activities.delete_schedules,
+        Activities.create_schedules,
+        Activities.update_schedules,
+    ]