QTZPOC-20

QTZPOC-20: Improve structured logging and reporting for all model import pipeline failures by introducing a dedicated activity and centralizing error classification. This ensures consistent observability, with all workflow failures now reported before cleanup. Also, adjusted import step timeouts and refined manual run script cleanup logic.

laborious/workflows/import_model.py

@@ -31,23 +31,27 @@ with workflow.unsafe.imports_passed_through():
         IMPORT_STATUS_RETRY_POLICY,
         derive_run_name,
     )
-    from laborious.activities.model_import_errors import ImportInputError, failure_location
     from laborious.utils.bundle.steps import ImportStep
+    from laborious.utils.models.import_errors import (
+        ImportInputError,
+        describe_failure_chain,
+        failure_location,
+    )
 
 # `start_to_close_timeout` is per attempt. The terminal write additionally gets a
-# `schedule_to_close_timeout` larger than its retry envelope (10 attempts, ≈151 s of backoff, ≈251 s
+# `schedule_to_close_timeout` larger than its retry envelope (10 attempts, ≈151 s of backoff, ≈451 s
 # if every attempt also burns its timeout) — set it smaller and Temporal cuts the retries short,
 # which is the one configuration mistake that would silently undo design D20.
-STATUS_WRITE_TIMEOUT = timedelta(seconds=10)
-STATUS_WRITE_ENVELOPE = timedelta(seconds=300)
-DOWNLOAD_TIMEOUT = timedelta(seconds=600)
-OPEN_TIMEOUT = timedelta(seconds=900)
-EXPERIMENT_TIMEOUT = timedelta(seconds=60)
-ARTIFACT_UPLOAD_TIMEOUT = timedelta(seconds=900)
-REGISTRATION_TIMEOUT = timedelta(seconds=120)
-DOCUMENT_TIMEOUT = timedelta(seconds=60)
-CLEANUP_TIMEOUT = timedelta(seconds=120)
-RETENTION_TIMEOUT = timedelta(seconds=60)
+STATUS_WRITE_TIMEOUT = timedelta(seconds=30)
+STATUS_WRITE_ENVELOPE = timedelta(seconds=900)
+DOWNLOAD_TIMEOUT = timedelta(seconds=1800)
+OPEN_TIMEOUT = timedelta(seconds=1800)
+EXPERIMENT_TIMEOUT = timedelta(seconds=300)
+ARTIFACT_UPLOAD_TIMEOUT = timedelta(seconds=1800)
+REGISTRATION_TIMEOUT = timedelta(seconds=600)
+DOCUMENT_TIMEOUT = timedelta(seconds=300)
+CLEANUP_TIMEOUT = timedelta(seconds=600)
+RETENTION_TIMEOUT = timedelta(seconds=300)
 
 # Fields no import may carry. The project is a frontend concern: the model listing is the MongoDB
 # `models` document, no migrated database holds a project or model table, and nothing in this
@@ -226,6 +230,7 @@ class ImportModel:
             located_step, located_gate = failure_location(error)
             failed_step = located_step or current_step.value
             gate = located_gate
+            await self._report_failure(base, error, step=failed_step, gate=gate)
             raise
         finally:
             cleanup = await self._cleanup(base)
@@ -365,6 +370,36 @@ class ImportModel:
                 verdict=f'succeeded={succeeded} step={step} gate={gate}',
             )
 
+    @staticmethod
+    async def _report_failure(
+        base: dict[str, Any],
+        error: BaseException,
+        *,
+        step: str | None,
+        gate: int | None,
+    ) -> None:
+        """Route the failure that ended this import through the one failure channel.
+
+        Runs from the `except`, before cleanup and before the terminal write, so the log reads in the
+        order things happened: what failed, then what was cleaned, then what was recorded. The step
+        and the gate are the ones already located from the failure, so the log line and the row can
+        never name different ones.
+
+        Its own failure is swallowed. The channel is the thing that logs, so a report that cannot be
+        made has nowhere to say so, and the failure it was reporting is re-raised either way — the
+        event history and the record both still carry it.
+        """
+        error_type, detail = describe_failure_chain(error)
+        try:
+            await workflow.execute_activity_method(
+                Activities.report_import_pipeline_failure,
+                {**base, 'step': step, 'gate': gate, 'detail': detail, 'error_type': error_type},
+                retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY,
+                start_to_close_timeout=STATUS_WRITE_TIMEOUT,
+            )
+        except BaseException:  # noqa: S110  # NOSONAR - the report of a report is where this stops
+            pass
+
     async def _report_status_write_failure(
         self,
         base: dict[str, Any],