QTZPOC-20
QTZPOC-20: Improve structured logging and reporting for all model import pipeline failures by introducing a dedicated activity and centralizing error classification. This ensures consistent observability, with all workflow failures now reported before cleanup. Also, adjusted import step timeouts and refined manual run script cleanup logic.
This commit is contained in:
@@ -15,18 +15,19 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
from pytest import fixture, mark, raises
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from temporalio.exceptions import ApplicationError
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.activities.model_import import (
|
||||
IMPORT_STATUS_HINT_RETRY_POLICY,
|
||||
IMPORT_STATUS_RETRY_POLICY,
|
||||
)
|
||||
from laborious.activities.model_import_errors import (
|
||||
from laborious.utils.bundle.steps import ImportStep
|
||||
from laborious.utils.models.import_errors import (
|
||||
ImportGateRejectedError,
|
||||
ImportInputError,
|
||||
ImportLogRowNotClaimableError,
|
||||
)
|
||||
from laborious.utils.bundle.steps import ImportStep
|
||||
from laborious.workflows.import_model import (
|
||||
REJECTED_INPUT_FIELDS,
|
||||
STATUS_WRITE_ENVELOPE,
|
||||
@@ -224,6 +225,24 @@ async def test_the_names_are_recorded_before_mlflow_is_touched(
|
||||
assert names['run_name'] == 'import-factory_model-v2'
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_model_is_named_from_the_bundles_model_name(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
"""`metadata.model_name` names the model, verbatim, in both destinations.
|
||||
|
||||
The registered model and the model document take the bundle's own `model_name` and nothing else:
|
||||
no other metadata field substitutes for it, and nothing is appended to it.
|
||||
"""
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
registration = workflow_env.recorder.payload_of(Activities.register_import_model_version)
|
||||
document = workflow_env.recorder.payload_of(Activities.write_import_model_document)
|
||||
assert registration['model_name'] == BUNDLE_METADATA['model_name']
|
||||
assert document['model_name'] == BUNDLE_METADATA['model_name']
|
||||
assert document['model_name'] == registration['model_name']
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_run_is_created_with_exactly_the_recorded_name(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
@@ -660,6 +679,168 @@ async def test_reporting_the_report_is_where_it_stops(import_workflow: ImportMod
|
||||
assert result['run_id'] == RUN_ID
|
||||
|
||||
|
||||
# --------------------------------------------------------------- the failure report
|
||||
|
||||
|
||||
def activity_failure(message: str, kind: str) -> BaseException:
|
||||
"""What a failed activity looks like from inside the workflow.
|
||||
|
||||
The activity's own exception type survives the boundary in `ApplicationError.type` and nowhere
|
||||
else; the outer error is the one Temporal raises for the failed activity.
|
||||
"""
|
||||
wrapper = RuntimeError('Activity task failed')
|
||||
wrapper.__cause__ = ApplicationError(message, type=kind)
|
||||
return wrapper
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_failure_that_ends_an_import_is_reported_through_the_channel(
|
||||
import_workflow: ImportModel,
|
||||
):
|
||||
"""The gap this closes: a failure at any pipeline step used to reach the structured logs as
|
||||
nothing at all — only the SDK's own traceback for the attempt, with no id and no step."""
|
||||
recorder = Recorder(
|
||||
failures={
|
||||
Activities.create_import_experiment: activity_failure(
|
||||
'API request to http://localhost:5080/api/2.0/mlflow/experiments/get-by-name '
|
||||
'failed with exception Connection refused',
|
||||
'MlflowException',
|
||||
)
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(RuntimeError, match='Activity task failed'):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
reported = recorder.payload_of(Activities.report_import_pipeline_failure)
|
||||
assert reported['step'] == ImportStep.EXPERIMENT_CREATION.value
|
||||
assert reported['error_type'] == 'MlflowException'
|
||||
assert reported['detail'].splitlines()[0].startswith('MlflowException: API request to')
|
||||
assert reported['import_run_id'] == 77
|
||||
assert reported['workflow_id'] == WORKFLOW_ID
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_report_comes_before_the_cleanup_and_the_terminal_write(
|
||||
import_workflow: ImportModel,
|
||||
):
|
||||
"""So the log reads in the order things happened, and the row's step matches the line's."""
|
||||
recorder = Recorder(
|
||||
failures={Activities.upload_import_artifacts: activity_failure('bucket gone', 'S3Error')}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(RuntimeError):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
order = recorder.order
|
||||
assert (
|
||||
order.index(Activities.report_import_pipeline_failure)
|
||||
< order.index(Activities.cleanup_import_files)
|
||||
< order.index(Activities.record_import_terminal_status)
|
||||
)
|
||||
reported = recorder.payload_of(Activities.report_import_pipeline_failure)
|
||||
terminal = recorder.payload_of(Activities.record_import_terminal_status)
|
||||
assert reported['step'] == terminal['step'] == ImportStep.ARTIFACT_UPLOAD.value
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_gate_that_rejected_the_bundle_travels_into_the_report(
|
||||
import_workflow: ImportModel,
|
||||
):
|
||||
recorder = Recorder(
|
||||
failures={
|
||||
Activities.open_import_bundle: ImportGateRejectedError(
|
||||
'gate 3 rejected the bundle', step=ImportStep.DECRYPTION, gate=3
|
||||
)
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(ImportGateRejectedError):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
reported = recorder.payload_of(Activities.report_import_pipeline_failure)
|
||||
assert reported['step'] == ImportStep.DECRYPTION.value
|
||||
assert reported['gate'] == 3
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_claim_that_matches_nothing_is_not_reported_twice(import_workflow: ImportModel):
|
||||
"""The claim reports itself, and it runs outside the try — so the channel is hit once."""
|
||||
recorder = Recorder(
|
||||
failures={
|
||||
Activities.claim_import_status: ImportLogRowNotClaimableError(
|
||||
'row 29 could not be claimed', import_run_id=29
|
||||
)
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(ImportLogRowNotClaimableError):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert Activities.report_import_pipeline_failure not in recorder.order
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_report_that_cannot_be_made_does_not_mask_the_failure(
|
||||
import_workflow: ImportModel,
|
||||
):
|
||||
recorder = Recorder(
|
||||
failures={
|
||||
Activities.write_import_model_document: activity_failure('mongo down', 'PyMongoError'),
|
||||
Activities.report_import_pipeline_failure: RuntimeError('no channel'),
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(RuntimeError, match='Activity task failed'):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
terminal = recorder.payload_of(Activities.record_import_terminal_status)
|
||||
assert terminal['succeeded'] is False
|
||||
assert terminal['step'] == ImportStep.MODEL_DOCUMENT.value
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_successful_import_reports_no_failure(import_workflow: ImportModel, workflow_env):
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
assert Activities.report_import_pipeline_failure not in workflow_env.recorder.order
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_report_gives_up_inside_the_short_bound(import_workflow: ImportModel):
|
||||
"""A failure already happened; spending 151 seconds telling the log about it would be worse."""
|
||||
recorder = Recorder(
|
||||
failures={
|
||||
Activities.create_import_experiment: activity_failure('refused', 'MlflowException')
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(RuntimeError):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
call = recorder.calls_to(Activities.report_import_pipeline_failure)[0]
|
||||
assert call.kwargs['retry_policy'] is IMPORT_STATUS_HINT_RETRY_POLICY
|
||||
assert call.kwargs['start_to_close_timeout'] == STATUS_WRITE_TIMEOUT
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- the retry split
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user