SIENTIAPDE-1975: align the gateway write path with the Laborious

- GW1: write commands (C2) carry only the server_id; ServerConfigResolver
  resolves the connection (command override, live pooled connection,
  connectors, opc-servers) with a 60 s cache and a 10 s negative cache.
- GW2: write_status keeps the worst outcome (ok < error < session_bad,
  pending/NULL replaced), same rule as the Laborious, built with SQLAlchemy
  Core (no SQL formatting).
- GW3: update_write_status never raises; outcome (updated, kept,
  row_not_found, invalid, error) logged and counted
  (opc_write_status_update_total); runs off the event loop.
- GW4: the reason of a failed write is logged; opc_write_status_by_model
  shows the last outcome; a bad timestamp is a failed write, not an
  exception.
- GW5: the NATS first connect gives up after 5 s (nats-py's defaults block
  the main loop for about 4 minutes while NATS is down); reconnection after
  it is unbounded; idle_sweep restarts a connection closed for good; stop()
  is safe on a closed connection.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

ingestor/io/data_sink.py

@@ -1,6 +1,7 @@
+import asyncio
 import re
 
-from sqlalchemy import text
+from sqlalchemy import Select, Update, case, column, select, table, update
 
 import ingestor.metrics as metrics
 
@@ -8,6 +9,34 @@ import ingestor.metrics as metrics
 # match before a trailing '\n', which is not what we want for a SQL identifier.
 _IDENTIFIER_RE = re.compile(r'\w+', re.ASCII)
 
+# Worst-wins precedence of predictions.write_status (SIENTIAPDE-1975), from best to worst;
+# NULL, 'pending' and unknown values rank 0. Same rule as the Laborious
+# (laborious/utils/write_status.py): keep both in sync.
+STATUS_RANK = {'ok': 1, 'error': 2, 'session_bad': 3}
+
+
+def write_status_statements(
+    schema: str, table_name: str, status: str, timestamp, model_id
+) -> tuple[Update, Select]:
+    """The UPDATE that records `status` unless the row has a worse one, and the SELECT that
+    tells whether the row exists. Built with SQLAlchemy Core: identifiers and values are
+    never formatted into the SQL text."""
+    predictions = table(
+        table_name,
+        column('write_status'),
+        column('timestamp'),
+        column('model_id'),
+        schema=schema,
+    )
+    row = (predictions.c.timestamp == timestamp) & (predictions.c.model_id == model_id)
+    current_rank = case(STATUS_RANK, value=predictions.c.write_status, else_=0)
+    return (
+        update(predictions)
+        .where(row, current_rank <= STATUS_RANK.get(status, 0))
+        .values(write_status=status),
+        select(1).select_from(predictions).where(row),
+    )
+
 
 class DataSink:
     """Gateway egress wrapper: fans a reading out to every configured name/collection."""
@@ -43,33 +72,71 @@ class DataSink:
                 },
             )
 
-    async def update_write_status(self, cmd: dict, result) -> None:  # NOSONAR
-        """UPDATE the per-model prediction row's write_status (C3), by timestamp+model_id.
+    async def update_write_status(self, cmd: dict, result) -> str:
+        """Record a write outcome on the prediction row (C3), keeping the worst one.
 
-        schema/table_name are per-model SQL identifiers, not values — they can't be
-        bound params, so they're checked against an allowlist before interpolation.
-        status/timestamp/model_id ARE bound params, never f-stringed.
+        The row is found by timestamp + model_id. A prediction is written to several
+        tags, each answered by its own command, so a better outcome never replaces a
+        worse one: ``ok`` < ``error`` < ``session_bad``, and ``pending``/NULL are
+        replaced by any answer. The Laborious runs the same statement for the outcomes it
+        records itself (SIENTIAPDE-1975), so the order of the answers does not matter.
+
+        Never raises: a failure here must not reach the NATS callback nor hide the OPC
+        write itself, which already happened. The outcome is logged and counted instead.
+
+        Return:
+            str: ``updated``, ``kept`` (the row already has a worse outcome),
+            ``row_not_found`` (e.g. the Laborious INSERT failed), ``invalid`` (bad or
+            missing schema/table/timestamp/model_id) or ``error``.
         """
-        schema = self._validate_identifier(cmd['schema'])
-        table = self._validate_identifier(cmd['table_name'])
-        # schema/table are allowlist-validated identifiers above; status/timestamp/model_id
-        # are bound params below, never interpolated — not a real injection vector.
-        stmt = text(
-            f'UPDATE {schema}.{table} SET write_status = :status '  # noqa: S608 # nosec B608
-            'WHERE timestamp = :timestamp AND model_id = :model_id'
+        try:
+            schema = self._validate_identifier(cmd.get('schema'))
+            table = self._validate_identifier(cmd.get('table_name'))
+            if cmd.get('timestamp') is None or cmd.get('model_id') is None:
+                raise ValueError('missing timestamp or model_id')
+        except ValueError as e:
+            return await self._status_outcome('invalid', cmd, f'write_status not recorded: {e}')
+
+        statements = write_status_statements(
+            schema, table, result.status, cmd['timestamp'], cmd['model_id']
         )
-        params = {
-            'status': result.status,
-            'timestamp': cmd['timestamp'],
-            'model_id': cmd['model_id'],
-        }
-        # ponytail: sync engine call inside async def — same pattern sientia_do's own
-        # Postgres activities use (e.g. export_data_to_postgres); no async pg driver here.
+        try:
+            # The engine is synchronous (no async pg driver here): run it off the event
+            # loop, which also serves the subscriptions of the read path.
+            outcome = await asyncio.to_thread(self._update_sync, *statements)
+        except Exception as e:
+            return await self._status_outcome('error', cmd, f'write_status UPDATE failed: {e}')
+        detail = {
+            'updated': f'write_status={result.status} recorded',
+            'kept': f'write_status={result.status} not recorded: the row has a worse outcome',
+            'row_not_found': f'write_status={result.status} not recorded: no prediction row',
+        }[outcome]
+        return await self._status_outcome(outcome, cmd, detail)
+
+    def _update_sync(self, update_stmt: Update, exists_stmt: Select) -> str:
         with self.postgres.engine.begin() as connection:
-            connection.execute(stmt, params)
+            if connection.execute(update_stmt).rowcount:
+                return 'updated'
+            row = connection.execute(exists_stmt).first()
+        return 'kept' if row is not None else 'row_not_found'
+
+    async def _status_outcome(self, outcome: str, cmd: dict, detail: str) -> str:
+        message = (
+            f'{detail} (server={cmd.get("server_id")} addr={cmd.get("addr")} '
+            f'model_id={cmd.get("model_id")} timestamp={cmd.get("timestamp")})'
+        )
+        if outcome in ('updated', 'kept'):
+            self.data_manager.logger.debug(message)
+        else:
+            self.data_manager.logger.warning(message)
+        await self.emit_metric(
+            metric_object=metrics.OPC_WRITE_STATUS_UPDATE_TOTAL,
+            tags={'pod_id': self.data_manager.pod_id, 'result': outcome},
+        )
+        return outcome
 
     @staticmethod
-    def _validate_identifier(name: str) -> str:
+    def _validate_identifier(name: object) -> str:
         if not isinstance(name, str) or not _IDENTIFIER_RE.fullmatch(name):
             raise ValueError(f'invalid SQL identifier: {name!r}')
         return name