gitea_admin/sientia-dataops-laborious-legacy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

QTZPOC-20

Browse Source 
QTZPOC-20: Improve structured logging and reporting for all model import pipeline failures by introducing a dedicated activity and centralizing error classification. This ensures consistent observability, with all workflow failures now reported before cleanup. Also, adjusted import step timeouts and refined manual run script cleanup logic.
This commit is contained in:
Vitor Pimentel dos Santos
2026-08-27 14:44:57 -03:00
parent e95e65b72b
commit 853c83e662
18 changed files with 857 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
laborious/activities/model_import.py
View File

@@ -41,14 +41,6 @@ with workflow.unsafe.imports_passed_through():
from temporalio.common import RetryPolicy
from laborious import metrics
from laborious.activities.model_import_errors import (
ImportErrorCode,
ImportGateRejectedError,
ImportLogRowNotClaimableError,
ImportStatusRejectedError,
classify_import_failure,
report_import_failure,
)
from laborious.utils.bundle import format as fmt
from laborious.utils.bundle.reader import BundleGateError, BundleLimits, BundleReader
from laborious.utils.bundle.steps import ImportStep
@@ -57,6 +49,14 @@ with workflow.unsafe.imports_passed_through():
decrypt_password_envelope,
load_envelope_key,
)
from laborious.utils.models.import_errors import (
ImportErrorCode,
ImportGateRejectedError,
ImportLogRowNotClaimableError,
ImportStatusRejectedError,
classify_import_failure,
report_import_failure,
)
from laborious.utils.repository.model_repository import MLFlowRepository
# The table is a constant of the code, never an input: there is exactly one import log on this
@@ -631,6 +631,50 @@ class ModelImport(SientiaMonitoring):
)
return {'reported': True}
@activity.defn(name='report_import_pipeline_failure')
async def report_import_pipeline_failure(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""Report the failure that ended the import — the one every step shares.
Until this existed, a failure at `download` … `model_document` reached the observability
surface only as whatever the Temporal SDK printed for the failing attempt: a bare traceback,
with no `import_run_id`, no step, no code, and no pod or runtime label, so it could not be
joined to the record it failed. It also never incremented the import error metric and never
notified, which the two failures that *are* reported (a row that cannot be claimed, a verdict
that cannot be written) always did.
The exception object is gone by the time the workflow gets here — it crossed an activity
boundary — so the failure travels as text and its type as a name, and the classification comes
from the step, exactly as the record's does. Reporting is never a reason to fail: an
unrecognised step is reported as unclassified rather than raising over a spelling.
Args (input_data):
metadata (dict): workflow metadata.
import_run_id (int): the record's primary key.
workflow_id (str): the identifier a user quotes to support.
step (str, optional): the step that failed. Absent means unclassified.
gate (int, optional): the gate that rejected the bundle, when one did.
detail (str): the failure as text, root cause first.
error_type (str, optional): the exception type name that travelled with it.
"""
raw_step = input_data.get('step')
try:
step = ImportStep(raw_step) if raw_step else None
except ValueError:
step = None
await report_import_failure(
self,
step,
None,
import_run_id=input_data.get('import_run_id'),
workflow_id=input_data['workflow_id'],
metadata=input_data.get('metadata', {}),
marker=str(raw_step) if raw_step else None,
detail_text=input_data.get('detail'),
error_type=input_data.get('error_type'),
gate=input_data.get('gate'),
)
return {'reported': True}
# ------------------------------------------------------------- bundle helpers
def _bundle_reader(self) -> BundleReader:

162
laborious/activities/model_import_errors.py → laborious/utils/models/import_errors.py
View File

@@ -236,6 +236,142 @@ class ImportGateRejectedError(ApplicationError):
self.gate = gate
_SUMMARY_LIMIT = 400
_CHAIN_LIMIT = 10
_UNKNOWN_ERROR_TYPE = 'unavailable'
def _root_cause(exception: BaseException) -> BaseException:
"""The deepest `__cause__` in the chain: the failure that actually happened.
A `MlflowException` reaches this channel wrapped — by an `ActivityError`, by an
`ApplicationError`, by `requests`' own re-raises — and the outermost type is the wrapper's, which
says nothing an operator can act on. The chain is walked to its end, and bounded, because a cycle
in `__cause__` would otherwise hang the reporting of a failure.
"""
candidate = exception
for _ in range(_CHAIN_LIMIT):
if candidate.__cause__ is None:
break
candidate = candidate.__cause__
return candidate
def _type_name(exception: BaseException) -> str:
"""The name to report a failure under: `ApplicationError.type` when there is one.
An activity's own exception type is the one thing worth grouping failures by, and Temporal keeps
it in `ApplicationError.type` rather than in the Python class — a `MlflowException` raised in an
activity reaches the workflow as an `ApplicationError` whose `type` still says `MlflowException`.
Reporting the class name here would say `ApplicationError` for every activity failure there is.
"""
return str(getattr(exception, 'type', None) or type(exception).__name__)
def _one_line(text: str) -> str:
"""The first non-empty line, bounded — a log *field* has to stay one field.
The traceback still reaches the log in full, in the message. This is the value a dashboard groups
by and an alert quotes, so a 40-frame `urllib3` chain would make it useless.
"""
for line in text.splitlines():
stripped = line.strip()
if stripped:
return stripped[:_SUMMARY_LIMIT]
return ''
def summarize_failure(
exception: BaseException | None,
detail_text: str | None = None,
error_type: str | None = None,
) -> tuple[str, str]:
"""`(error_type, error_summary)` — the two technical fields every failure log carries.
Args:
exception: the failure, when the object itself is to hand.
detail_text: the failure as text, for a caller whose exception crossed an activity boundary.
error_type: the type name that travelled with such a text, when it is known.
Returns:
tuple: the exception type name and a single-line summary of what it said.
"""
if exception is not None:
cause = _root_cause(exception)
return _type_name(cause), _one_line(str(cause))
return error_type or _UNKNOWN_ERROR_TYPE, _one_line(detail_text or '')
def describe_failure_chain(exception: BaseException) -> tuple[str, str]:
"""`(error_type, detail)` for a failure that is about to lose its exception object.
The workflow is the one place that holds the whole chain — `ActivityError` wrapping the
`ApplicationError` that carries the activity's own type and message — and it cannot hand that
object to the channel, because the channel runs in an activity. So the chain is flattened here,
**root cause first**, one link per line: the first line is what actually broke, which is what
becomes `error_summary`, and the wrappers follow for whoever reads further.
`ApplicationError.type` is preferred over the Python class name because it is the only place the
activity's original exception type survives the boundary: `MlflowException` arrives as an
`ApplicationError` whose `type` says so.
"""
links: list[str] = []
candidate: BaseException | None = exception
for _ in range(_CHAIN_LIMIT):
if candidate is None:
break
kind = _type_name(candidate)
message = _one_line(str(candidate))
# `ApplicationError.__str__` already prefixes its own type, and `MlflowException:
# MlflowException: …` reads like two failures rather than one.
if not message:
links.append(kind)
elif message.startswith(f'{kind}:'):
links.append(message)
else:
links.append(f'{kind}: {message}')
candidate = candidate.__cause__
links.reverse()
root = links[0].split(':', 1)[0] if links else _UNKNOWN_ERROR_TYPE
return root, '\n'.join(links)
def _log_fields(
*,
failure: ImportFailure,
where: str,
import_run_id: Any,
workflow_id: str,
gate: int | None,
error_type: str,
error_summary: str,
verdict: str | None,
metadata: dict[str, Any] | None,
) -> dict[str, Any]:
"""The workflow metadata plus this failure's own fields, as `logfmt` keys.
Every key is prefixed `import_` or `error_` so none of them can collide with the metadata the
rest of the runtime already logs (`model_id`, `model_name`, `schedule_name`, `workflow_name`) or
with the formatter's own (`at`, `msg`, `timestamp`). Values are single-line by construction: the
traceback stays in the message, where a newline costs nothing.
"""
fields: dict[str, Any] = {
**(metadata or {}),
'import_run_id': import_run_id,
'import_workflow_id': workflow_id,
'import_step': where,
'import_code': failure.code.value,
'error_type': error_type,
}
if error_summary:
fields['error_summary'] = error_summary
if gate is not None:
fields['import_gate'] = gate
if verdict:
fields['import_verdict_not_recorded'] = verdict
return fields
def failure_location(exception: BaseException | None) -> tuple[str | None, int | None]:
"""Read the step and gate an activity's failure carries, if it carried any.
@@ -294,6 +430,8 @@ async def report_import_failure(
marker: str | None = None,
verdict: str | None = None,
detail_text: str | None = None,
error_type: str | None = None,
gate: int | None = None,
) -> ImportFailure:
"""Report one failure through every sink, exactly once.
@@ -303,6 +441,12 @@ async def report_import_failure(
reaches Temporal by being raised, and the record is the fifth, optional sink — the terminal
write, when the row can be written at all.
Both log lines this produces — the technical one here and the user-facing sentence the
notification logs — carry the same `logfmt` fields, built once by `_log_fields`. That is what
makes the two greppable and joinable: `import_run_id=29` finds the pair, `import_code=` groups
them, and `error_type=` says what actually broke without reading a traceback. The message text is
kept as it was, ids and all, so a query written against it still matches.
Args:
sinks: the activity providing logger, metrics and notifications.
step: the pipeline step, or `None` for a failure that is not one (see `marker`).
@@ -316,6 +460,8 @@ async def report_import_failure(
survives in text even when it cannot survive in the row.
detail_text: the failure as text, for the callers whose exception object is already gone —
anything that crossed an activity boundary. Used in place of a traceback.
error_type: the exception type name that travelled with such a text, when it is known.
gate: the gate that rejected the bundle, when one did — a field, never a sentence.
Returns:
ImportFailure: the classification that was reported.
@@ -329,17 +475,29 @@ async def report_import_failure(
detail = ''.join(traceback.format_exception(exception)).strip()
else:
detail = detail_text or 'no exception object available'
kind, summary = summarize_failure(exception, detail_text, error_type)
fields = _log_fields(
failure=failure,
where=where,
import_run_id=import_run_id,
workflow_id=workflow_id,
gate=gate,
error_type=kind,
error_summary=summary,
verdict=verdict,
metadata=metadata,
)
verdict_line = f' verdict_not_recorded={verdict}' if verdict else ''
sinks.error(
f'model import failed: step={where} code={failure.code.value} '
f'import_run_id={import_run_id} workflow_id={workflow_id}{verdict_line}\n{detail}',
metadata or {},
fields,
)
labels = sinks.get_core_labels(metadata or {}, operation_type=failure.code.value)
await sinks.emit_metric(metric_object=metrics.MODEL_IMPORT_ERROR_COUNT, tags=labels)
await sinks.send_notification_async(
metadata=metadata or {},
metadata=fields,
notification_id=failure.code.value,
message=failure.reason,
block=where,

1
laborious/worker/worker.py
View File

@@ -277,6 +277,7 @@ async def main():
activities.record_import_names,
activities.record_import_terminal_status,
activities.report_import_status_write_failure,
activities.report_import_pipeline_failure,
# Bundle
activities.download_import_bundle,
activities.open_import_bundle,

59
laborious/workflows/import_model.py
View File

@@ -31,23 +31,27 @@ with workflow.unsafe.imports_passed_through():
IMPORT_STATUS_RETRY_POLICY,
derive_run_name,
)
from laborious.activities.model_import_errors import ImportInputError, failure_location
from laborious.utils.bundle.steps import ImportStep
from laborious.utils.models.import_errors import (
ImportInputError,
describe_failure_chain,
failure_location,
)
# `start_to_close_timeout` is per attempt. The terminal write additionally gets a
# `schedule_to_close_timeout` larger than its retry envelope (10 attempts, ≈151 s of backoff, ≈251 s
# `schedule_to_close_timeout` larger than its retry envelope (10 attempts, ≈151 s of backoff, ≈451 s
# if every attempt also burns its timeout) — set it smaller and Temporal cuts the retries short,
# which is the one configuration mistake that would silently undo design D20.
STATUS_WRITE_TIMEOUT = timedelta(seconds=10)
STATUS_WRITE_ENVELOPE = timedelta(seconds=300)
DOWNLOAD_TIMEOUT = timedelta(seconds=600)
OPEN_TIMEOUT = timedelta(seconds=900)
EXPERIMENT_TIMEOUT = timedelta(seconds=60)
ARTIFACT_UPLOAD_TIMEOUT = timedelta(seconds=900)
REGISTRATION_TIMEOUT = timedelta(seconds=120)
DOCUMENT_TIMEOUT = timedelta(seconds=60)
CLEANUP_TIMEOUT = timedelta(seconds=120)
RETENTION_TIMEOUT = timedelta(seconds=60)
STATUS_WRITE_TIMEOUT = timedelta(seconds=30)
STATUS_WRITE_ENVELOPE = timedelta(seconds=900)
DOWNLOAD_TIMEOUT = timedelta(seconds=1800)
OPEN_TIMEOUT = timedelta(seconds=1800)
EXPERIMENT_TIMEOUT = timedelta(seconds=300)
ARTIFACT_UPLOAD_TIMEOUT = timedelta(seconds=1800)
REGISTRATION_TIMEOUT = timedelta(seconds=600)
DOCUMENT_TIMEOUT = timedelta(seconds=300)
CLEANUP_TIMEOUT = timedelta(seconds=600)
RETENTION_TIMEOUT = timedelta(seconds=300)
# Fields no import may carry. The project is a frontend concern: the model listing is the MongoDB
# `models` document, no migrated database holds a project or model table, and nothing in this
@@ -226,6 +230,7 @@ class ImportModel:
located_step, located_gate = failure_location(error)
failed_step = located_step or current_step.value
gate = located_gate
await self._report_failure(base, error, step=failed_step, gate=gate)
raise
finally:
cleanup = await self._cleanup(base)
@@ -365,6 +370,36 @@ class ImportModel:
verdict=f'succeeded={succeeded} step={step} gate={gate}',
)
@staticmethod
async def _report_failure(
base: dict[str, Any],
error: BaseException,
*,
step: str | None,
gate: int | None,
) -> None:
"""Route the failure that ended this import through the one failure channel.
Runs from the `except`, before cleanup and before the terminal write, so the log reads in the
order things happened: what failed, then what was cleaned, then what was recorded. The step
and the gate are the ones already located from the failure, so the log line and the row can
never name different ones.
Its own failure is swallowed. The channel is the thing that logs, so a report that cannot be
made has nowhere to say so, and the failure it was reporting is re-raised either way — the
event history and the record both still carry it.
"""
error_type, detail = describe_failure_chain(error)
try:
await workflow.execute_activity_method(
Activities.report_import_pipeline_failure,
{**base, 'step': step, 'gate': gate, 'detail': detail, 'error_type': error_type},
retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY,
start_to_close_timeout=STATUS_WRITE_TIMEOUT,
)
except BaseException: # noqa: S110 # NOSONAR - the report of a report is where this stops
pass
async def _report_status_write_failure(
self,
base: dict[str, Any],