QTZPOC-20
QTZPOC-20: Refactor model import workflow, introduce atomic model ID generation, and clarify MinIO expiry. Fuses bundle download, decryption, MLflow experiment creation, and artifact upload into a single activity to prevent cross-worker issues. Implements an atomic counter for MongoDB model IDs, replacing the max(id) + 1 scan to avoid collisions during concurrent imports. Adds comprehensive documentation for MinIO object expiry mechanisms and refines lifecycle rule application. Improves error handling with clearer responsibility classification and centralizes MLflow repository injection.
This commit is contained in:
@@ -17,7 +17,8 @@ from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
import traceback
|
||||
from collections.abc import Mapping
|
||||
from collections import deque
|
||||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from types import MappingProxyType
|
||||
@@ -58,74 +59,95 @@ class ImportErrorCode(StrEnum):
|
||||
CLEANUP_INCOMPLETE = 'IMPORT_CLEANUP_INCOMPLETE'
|
||||
|
||||
|
||||
class ImportResponsibility(StrEnum):
|
||||
"""Who can act on a step's failure: the user (re-export, re-type, re-submit) or only support."""
|
||||
|
||||
USER = 'user'
|
||||
SUPPORT = 'support'
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportFailure:
|
||||
"""One classification of one failure: the step, its code and the sentence it stores."""
|
||||
"""One classification of one failure: the step, its code, its sentence and whose to fix."""
|
||||
|
||||
step: ImportStep | None
|
||||
code: ImportErrorCode
|
||||
reason: str
|
||||
responsibility: ImportResponsibility = ImportResponsibility.SUPPORT
|
||||
|
||||
|
||||
_CONTACT_SUPPORT = "Contact support with this import's identifier."
|
||||
_RE_EXPORT = 'Export the model again from the origin platform.'
|
||||
|
||||
IMPORT_FAILURE_CATALOG: Final[Mapping[ImportStep, tuple[ImportErrorCode, str]]] = MappingProxyType(
|
||||
IMPORT_FAILURE_CATALOG: Final[
|
||||
Mapping[ImportStep, tuple[ImportErrorCode, str, ImportResponsibility]]
|
||||
] = MappingProxyType(
|
||||
{
|
||||
ImportStep.RECEIVED: (
|
||||
ImportErrorCode.REQUEST_INVALID,
|
||||
'The import could not be started because the request was incomplete or '
|
||||
'contradictory. Nothing was created. Submit the import again.',
|
||||
ImportResponsibility.USER,
|
||||
),
|
||||
ImportStep.DOWNLOAD: (
|
||||
ImportErrorCode.FILE_UNREADABLE,
|
||||
'The uploaded file could not be read, or it is not the file that was sent. '
|
||||
'Upload it again.',
|
||||
ImportResponsibility.USER,
|
||||
),
|
||||
ImportStep.DECRYPTION: (
|
||||
ImportErrorCode.BUNDLE_CANNOT_BE_OPENED,
|
||||
'This file could not be opened: the password is wrong, or the file is damaged or '
|
||||
'was altered. Confirm the password with whoever exported the model, then try '
|
||||
'again.',
|
||||
ImportResponsibility.USER,
|
||||
),
|
||||
ImportStep.ARCHIVE_INSPECTION: (
|
||||
ImportErrorCode.BUNDLE_REJECTED,
|
||||
"The file's contents did not pass the platform's safety checks, so it was not "
|
||||
f'opened. {_RE_EXPORT}',
|
||||
ImportResponsibility.USER,
|
||||
),
|
||||
ImportStep.EXTRACTION: (
|
||||
ImportErrorCode.BUNDLE_UNPACK_FAILED,
|
||||
f'The file could not be unpacked safely and nothing from it was kept. {_RE_EXPORT}',
|
||||
ImportResponsibility.USER,
|
||||
),
|
||||
ImportStep.STRUCTURE_VALIDATION: (
|
||||
ImportErrorCode.BUNDLE_INCOMPLETE,
|
||||
'This file is not a complete model export — part of what the platform needs is '
|
||||
f'missing from it. {_RE_EXPORT}',
|
||||
ImportResponsibility.USER,
|
||||
),
|
||||
ImportStep.CONTENT_POLICY: (
|
||||
ImportErrorCode.BUNDLE_UNEXPECTED_CONTENT,
|
||||
'The file contains something a model export should not contain, so it was '
|
||||
f'rejected. {_RE_EXPORT}',
|
||||
ImportResponsibility.USER,
|
||||
),
|
||||
ImportStep.EXPERIMENT_CREATION: (
|
||||
ImportErrorCode.STORAGE_PREPARATION_FAILED,
|
||||
'The platform could not prepare a place to keep this model. The import stopped '
|
||||
f'and no model was created. {_CONTACT_SUPPORT}',
|
||||
ImportResponsibility.SUPPORT,
|
||||
),
|
||||
ImportStep.ARTIFACT_UPLOAD: (
|
||||
ImportErrorCode.MODEL_FILES_NOT_STORED,
|
||||
"The model's files could not be stored on this platform. The import stopped and "
|
||||
f'the model is not available. {_CONTACT_SUPPORT}',
|
||||
ImportResponsibility.SUPPORT,
|
||||
),
|
||||
ImportStep.REGISTRATION: (
|
||||
ImportErrorCode.MODEL_NOT_REGISTERED,
|
||||
"The model's files were stored, but the model itself could not be registered, so "
|
||||
f'it cannot be used. {_CONTACT_SUPPORT}',
|
||||
ImportResponsibility.SUPPORT,
|
||||
),
|
||||
ImportStep.MODEL_DOCUMENT: (
|
||||
ImportErrorCode.MODEL_SETTINGS_NOT_SAVED,
|
||||
'The model was registered, but the settings that tell the platform how to run it '
|
||||
f'could not be saved. {_CONTACT_SUPPORT}',
|
||||
ImportResponsibility.SUPPORT,
|
||||
),
|
||||
}
|
||||
)
|
||||
@@ -135,36 +157,44 @@ UNCLASSIFIED_FAILURE: Final[tuple[ImportErrorCode, str]] = (
|
||||
f'The import stopped for an unexpected reason and no model was created. {_CONTACT_SUPPORT}',
|
||||
)
|
||||
|
||||
# The three explicit-code failures are not pipeline steps, so `classify_import_failure` falls
|
||||
# through to `UNCLASSIFIED_FAILURE`'s reason for all of them. That sentence says "no model was
|
||||
# created", which is false for a cleanup-only failure (the model was imported) and misleading for
|
||||
# the other two (neither ever attempted the pipeline). Each gets its own reason here instead.
|
||||
_OVERRIDE_REASONS: Final[Mapping[ImportErrorCode, str]] = MappingProxyType(
|
||||
{
|
||||
ImportErrorCode.CLEANUP_INCOMPLETE: (
|
||||
'The model was imported successfully; only its temporary files could not be removed '
|
||||
'from the worker. No action needed.'
|
||||
),
|
||||
ImportErrorCode.LOG_ROW_NOT_CLAIMABLE: 'The import request could not be started.',
|
||||
ImportErrorCode.STATUS_NOT_RECORDED: 'The import outcome could not be recorded.',
|
||||
}
|
||||
)
|
||||
|
||||
# `cleanup` deletes temporary files on the worker. It is never the reported failure: when every
|
||||
# other step succeeded and only cleanup failed, the import is a success and the leak is a log line.
|
||||
# So it deliberately has no user-facing sentence, and the catalog-completeness test knows it.
|
||||
STEPS_WITHOUT_A_USER_FACING_REASON: Final[frozenset[ImportStep]] = frozenset({ImportStep.CLEANUP})
|
||||
|
||||
|
||||
def classify_import_failure(
|
||||
step: ImportStep | None, exception: BaseException | None = None
|
||||
) -> ImportFailure:
|
||||
def classify_import_failure(step: ImportStep | None) -> ImportFailure:
|
||||
"""Turn a failure into the one code and the one sentence that will be recorded.
|
||||
|
||||
The exception's text is **ignored entirely** — it is what carries paths, table names and type
|
||||
names, and it belongs in the log. What decides the outcome is the step.
|
||||
The step alone decides the outcome — an exception's text carries paths, table names and type
|
||||
names, and belongs in the log, never in the record.
|
||||
|
||||
Args:
|
||||
step: the step that was being attempted, or `None` when none had been entered.
|
||||
exception: the failure, when there is one to hand. It is accepted so callers read as
|
||||
"classify this failure" and never absent because the classification needs it: the
|
||||
catalog is keyed by step, and an exception cannot cross an activity boundary with its
|
||||
type intact anyway — the terminal write classifies from the step alone.
|
||||
|
||||
Returns:
|
||||
ImportFailure: step, code and the reason to store.
|
||||
"""
|
||||
del exception # the catalog is keyed by step; the exception's text never reaches the record
|
||||
if step is None or step not in IMPORT_FAILURE_CATALOG:
|
||||
code, reason = UNCLASSIFIED_FAILURE
|
||||
return ImportFailure(step=step, code=code, reason=reason)
|
||||
code, reason = IMPORT_FAILURE_CATALOG[step]
|
||||
return ImportFailure(step=step, code=code, reason=reason)
|
||||
code, reason, responsibility = IMPORT_FAILURE_CATALOG[step]
|
||||
return ImportFailure(step=step, code=code, reason=reason, responsibility=responsibility)
|
||||
|
||||
|
||||
class ImportLogRowNotClaimableError(ApplicationError):
|
||||
@@ -198,6 +228,13 @@ class ImportStatusRejectedError(ApplicationError):
|
||||
super().__init__(message, type='ImportStatusRejectedError', non_retryable=True)
|
||||
|
||||
|
||||
class ImportMisconfiguredError(ApplicationError):
|
||||
"""A required repository was not injected into this worker. A deploy defect, never transient."""
|
||||
|
||||
def __init__(self, message: str) -> None:
|
||||
super().__init__(message, type='ImportMisconfiguredError', non_retryable=True)
|
||||
|
||||
|
||||
class ImportInputError(ApplicationError):
|
||||
"""The request is incomplete or contradictory. Fails at `received`, on the claimed row."""
|
||||
|
||||
@@ -241,6 +278,18 @@ _CHAIN_LIMIT = 10
|
||||
_UNKNOWN_ERROR_TYPE = 'unavailable'
|
||||
|
||||
|
||||
def walk_cause_chain(
|
||||
exception: BaseException, limit: int = _CHAIN_LIMIT
|
||||
) -> Iterator[BaseException]:
|
||||
"""Yield `exception`, then each `__cause__`, up to `limit` links. Bounded against a cycle."""
|
||||
candidate: BaseException | None = exception
|
||||
for _ in range(limit):
|
||||
if candidate is None:
|
||||
return
|
||||
yield candidate
|
||||
candidate = candidate.__cause__
|
||||
|
||||
|
||||
def _root_cause(exception: BaseException) -> BaseException:
|
||||
"""The deepest `__cause__` in the chain: the failure that actually happened.
|
||||
|
||||
@@ -249,12 +298,7 @@ def _root_cause(exception: BaseException) -> BaseException:
|
||||
says nothing an operator can act on. The chain is walked to its end, and bounded, because a cycle
|
||||
in `__cause__` would otherwise hang the reporting of a failure.
|
||||
"""
|
||||
candidate = exception
|
||||
for _ in range(_CHAIN_LIMIT):
|
||||
if candidate.__cause__ is None:
|
||||
break
|
||||
candidate = candidate.__cause__
|
||||
return candidate
|
||||
return deque(walk_cause_chain(exception), maxlen=1)[-1]
|
||||
|
||||
|
||||
def _type_name(exception: BaseException) -> str:
|
||||
@@ -316,10 +360,7 @@ def describe_failure_chain(exception: BaseException) -> tuple[str, str]:
|
||||
`ApplicationError` whose `type` says so.
|
||||
"""
|
||||
links: list[str] = []
|
||||
candidate: BaseException | None = exception
|
||||
for _ in range(_CHAIN_LIMIT):
|
||||
if candidate is None:
|
||||
break
|
||||
for candidate in walk_cause_chain(exception):
|
||||
kind = _type_name(candidate)
|
||||
message = _one_line(str(candidate))
|
||||
# `ApplicationError.__str__` already prefixes its own type, and `MlflowException:
|
||||
@@ -330,7 +371,6 @@ def describe_failure_chain(exception: BaseException) -> tuple[str, str]:
|
||||
links.append(message)
|
||||
else:
|
||||
links.append(f'{kind}: {message}')
|
||||
candidate = candidate.__cause__
|
||||
links.reverse()
|
||||
root = links[0].split(':', 1)[0] if links else _UNKNOWN_ERROR_TYPE
|
||||
return root, '\n'.join(links)
|
||||
@@ -379,16 +419,14 @@ def failure_location(exception: BaseException | None) -> tuple[str | None, int |
|
||||
Returns `(None, None)` when the failure did not say where it happened, and the caller then falls
|
||||
back to the step it was attempting.
|
||||
"""
|
||||
seen = 0
|
||||
candidate: BaseException | None = exception
|
||||
while candidate is not None and seen < 10:
|
||||
if exception is None:
|
||||
return None, None
|
||||
for candidate in walk_cause_chain(exception):
|
||||
details = getattr(candidate, 'details', None)
|
||||
if details and isinstance(details[0], dict) and 'step' in details[0]:
|
||||
location = details[0]
|
||||
gate = location.get('gate')
|
||||
return location.get('step'), int(gate) if isinstance(gate, int) else None
|
||||
candidate = candidate.__cause__
|
||||
seen += 1
|
||||
return None, None
|
||||
|
||||
|
||||
@@ -466,9 +504,11 @@ async def report_import_failure(
|
||||
Returns:
|
||||
ImportFailure: the classification that was reported.
|
||||
"""
|
||||
failure = classify_import_failure(step, exception)
|
||||
failure = classify_import_failure(step)
|
||||
if code is not None:
|
||||
failure = ImportFailure(step=step, code=code, reason=failure.reason)
|
||||
failure = ImportFailure(
|
||||
step=step, code=code, reason=_OVERRIDE_REASONS.get(code, failure.reason)
|
||||
)
|
||||
|
||||
where = marker or (step.value if step else 'unknown')
|
||||
if exception is not None:
|
||||
|
||||
Reference in New Issue
Block a user