SIENTIAPDE-1974

Add e2e tag-database scenarios and cover worker/email/formatter gaps

- Add e2e scenarios 1.9.1-1.9.4 for PI Web API through the tag
  database: scouter and output tag resolution, mixed-connector schedule
  hold, and OPC slot unchanged by PI connectors. Seed connectors and
  connector_tags from a new scenario input.
- Tighten the happy-path e2e to assert both schedules and that no PI
  output config is produced when the tag database is unused.
- Drop dead `if self.server` / `if notification_handler` guards in the
  email reconnect and worker shutdown paths, since both are always set.
- Log the traceback inside the message on worker failure, as the
  sientia_do Logger.error does not accept exc_info.
- Add unit tests for the worker failure exit, Prometheus startup
  failure, email reconnect without a password, and formatter, mongo,
  temporal manager and orchestrator edge cases.

orchestrator/activities/email.py

@@ -177,14 +177,13 @@ class Email(SientiaMonitoring):
             self.logger.error(f'SMTP server disconnected: {e}')
             self.logger.info(f'Reconnecting to {self.smtp_server}:{self.smtp_port}')
 
-            if self.server:
-                try:
-                    self.server.quit()
-                except SMTPServerDisconnected as e:
-                    self.logger.info(f'Server already disconnected: {e}')
-                except Exception as e:
-                    self.logger.error(f'Failed to quit server: {e}')
-                    raise e
+            try:
+                self.server.quit()
+            except SMTPServerDisconnected as e:
+                self.logger.info(f'Server already disconnected: {e}')
+            except Exception as e:
+                self.logger.error(f'Failed to quit server: {e}')
+                raise e
 
             self.server = smtplib.SMTP(self.smtp_server, self.smtp_port, timeout=20)
             if self.sender_password:

orchestrator/worker/worker.py

@@ -5,6 +5,7 @@ with workflow.unsafe.imports_passed_through():
     import asyncio
     import os
     import sys
+    import traceback
 
     from prometheus_client import start_http_server
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
@@ -192,13 +193,12 @@ async def main():
     try:
         await asyncio.gather(*handlers)
     except BaseException as e:
-        logger.error(f'An unhandled exception occurred: {e}', exc_info=True)
+        # sientia_do's Logger.error takes only a message: the traceback goes in it
+        logger.error(f'An unhandled exception occurred: {e}\n{traceback.format_exc()}')
         exit_code = 1
     finally:
-        if notification_handler:
-            notification_handler.shutdown()
-        if activities:
-            activities.shutdown()
+        notification_handler.shutdown()
+        activities.shutdown()
         metrics.APP_UP.labels(pod_id=POD_ID).set(0)
         sys.exit(exit_code)