QTZPOC-20
QTZPOC-20: Implement model import workflow for .sientia bundles Introduces the import_model workflow and its associated activities to enable importing .sientia bundles into MLflow and MongoDB. This includes: * New core workflow and activity logic for handling bundle download, decryption, validation, MLflow provisioning (experiments, runs, registered versions), and MongoDB model document creation. * Comprehensive E2E and unit test suites, including a dedicated test harness and a manual run script for development. * Updated .env.example and README.md with new configurations and usage instructions. * Ensures correct MLflow model source resolution for legacy loader compatibility and robust MongoDB document management. * Refactors existing utility scripts into a scripts/ directory.
This commit is contained in:
781
tests/laborious/workflows/test_import_model.py
Normal file
781
tests/laborious/workflows/test_import_model.py
Normal file
@@ -0,0 +1,781 @@
|
||||
"""The `import_model` workflow: order, fail-fast, and which calls are allowed to retry.
|
||||
|
||||
The workflow itself holds no logic worth testing in isolation except the two things that can only be
|
||||
wrong here: **the order** — the claim before anything else, the names before MLflow, the document
|
||||
last, cleanup and then one terminal write in the `finally` — and **the retry split**, where a single
|
||||
misplaced policy either replays a registry call or spends 151 seconds on a hint nobody reads.
|
||||
|
||||
Everything is asserted through the recorded activity calls, because that is exactly what the
|
||||
workflow is: a sequence of activity invocations with policies attached.
|
||||
"""
|
||||
|
||||
from datetime import timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from pytest import fixture, mark, raises
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.activities.model_import import (
|
||||
IMPORT_STATUS_HINT_RETRY_POLICY,
|
||||
IMPORT_STATUS_RETRY_POLICY,
|
||||
)
|
||||
from laborious.activities.model_import_errors import (
|
||||
ImportGateRejectedError,
|
||||
ImportInputError,
|
||||
ImportLogRowNotClaimableError,
|
||||
)
|
||||
from laborious.utils.bundle.steps import ImportStep
|
||||
from laborious.workflows.import_model import (
|
||||
REJECTED_INPUT_FIELDS,
|
||||
STATUS_WRITE_ENVELOPE,
|
||||
STATUS_WRITE_TIMEOUT,
|
||||
ImportModel,
|
||||
)
|
||||
|
||||
WORKFLOW_ID = 'import_model-77-abc'
|
||||
RUN_ID = '0123456789abcdef0123456789abcdef'
|
||||
DIGEST = 'a' * 64
|
||||
OBJECT_KEY = 'imported_models/factory_model_v2.sientia'
|
||||
|
||||
BUNDLE_METADATA = {
|
||||
'model_name': 'factory_model',
|
||||
'experiment_name': 'factory_experiment',
|
||||
'run_id': 'aa11bb22cc33dd44ee55ff6677889900',
|
||||
'model_version': '2',
|
||||
'model_project': 'factory_project',
|
||||
'export_timestamp': '2026-08-25T12:00:00+00:00',
|
||||
}
|
||||
BUNDLE_PARAMETERS = {'model_name': 'factory_model', 'target_variable': 'SE'}
|
||||
|
||||
# The activities that write the import log row. These four are the only calls allowed to retry.
|
||||
STATUS_ACTIVITIES = (
|
||||
Activities.claim_import_status,
|
||||
Activities.record_import_status,
|
||||
Activities.record_import_names,
|
||||
Activities.record_import_terminal_status,
|
||||
)
|
||||
CLAIM_AND_TERMINAL = (Activities.claim_import_status, Activities.record_import_terminal_status)
|
||||
HINTS = (Activities.record_import_status, Activities.record_import_names)
|
||||
|
||||
# What a correct import does, in order.
|
||||
EXPECTED_ORDER = [
|
||||
Activities.claim_import_status,
|
||||
Activities.record_import_status,
|
||||
Activities.ensure_import_bundle_retention,
|
||||
Activities.download_import_bundle,
|
||||
Activities.record_import_status,
|
||||
Activities.open_import_bundle,
|
||||
Activities.record_import_names,
|
||||
Activities.record_import_status,
|
||||
Activities.create_import_experiment,
|
||||
Activities.record_import_status,
|
||||
Activities.upload_import_artifacts,
|
||||
Activities.record_import_status,
|
||||
Activities.register_import_model_version,
|
||||
Activities.record_import_status,
|
||||
Activities.write_import_model_document,
|
||||
Activities.cleanup_import_files,
|
||||
Activities.record_import_terminal_status,
|
||||
]
|
||||
|
||||
PROVISIONING = (
|
||||
Activities.create_import_experiment,
|
||||
Activities.upload_import_artifacts,
|
||||
Activities.register_import_model_version,
|
||||
Activities.write_import_model_document,
|
||||
)
|
||||
|
||||
HAPPY_RESULTS = {
|
||||
Activities.claim_import_status: {
|
||||
'claimed': True,
|
||||
'bucket_name': None,
|
||||
'file_name': None,
|
||||
'request_digest': None,
|
||||
},
|
||||
Activities.download_import_bundle: {
|
||||
'path': '/work/bundle.sientia',
|
||||
'size': 4096,
|
||||
'digest': DIGEST,
|
||||
},
|
||||
Activities.open_import_bundle: {
|
||||
'extracted_dir': '/work/extract',
|
||||
'metadata': BUNDLE_METADATA,
|
||||
'parameters': BUNDLE_PARAMETERS,
|
||||
'digest': DIGEST,
|
||||
},
|
||||
Activities.create_import_experiment: {'experiment_id': '17'},
|
||||
Activities.upload_import_artifacts: {'run_id': RUN_ID, 'run_name': 'import-factory_model-v2'},
|
||||
Activities.register_import_model_version: {'model_name': 'factory_model', 'version': '4'},
|
||||
Activities.write_import_model_document: {'written': True, 'id': '8'},
|
||||
Activities.cleanup_import_files: {'cleaned': True, 'existed': True},
|
||||
Activities.record_import_terminal_status: {'status': 'IMPORT_SUCCESS'},
|
||||
}
|
||||
|
||||
|
||||
class Recorder:
|
||||
"""Stands in for `workflow.execute_activity_method`, recording every invocation."""
|
||||
|
||||
def __init__(self, failures: dict | None = None, results: dict | None = None) -> None:
|
||||
self.results = {**HAPPY_RESULTS, **(results or {})}
|
||||
self.failures = dict(failures or {})
|
||||
self.calls: list[SimpleNamespace] = []
|
||||
|
||||
async def __call__(self, method, payload, **kwargs):
|
||||
self.calls.append(SimpleNamespace(method=method, payload=payload, kwargs=kwargs))
|
||||
if method in self.failures:
|
||||
raise self.failures[method]
|
||||
return self.results.get(method, {})
|
||||
|
||||
@property
|
||||
def order(self) -> list:
|
||||
return [call.method for call in self.calls]
|
||||
|
||||
def payload_of(self, method):
|
||||
return next(call.payload for call in self.calls if call.method is method)
|
||||
|
||||
def last_payload_of(self, method):
|
||||
return [call.payload for call in self.calls if call.method is method][-1]
|
||||
|
||||
def calls_to(self, method) -> list[SimpleNamespace]:
|
||||
return [call for call in self.calls if call.method is method]
|
||||
|
||||
|
||||
def valid_input(**overrides) -> dict:
|
||||
return {
|
||||
'import_run_id': 77,
|
||||
'object_key': OBJECT_KEY,
|
||||
'expected_digest': DIGEST,
|
||||
'password_envelope': 'ZW52ZWxvcGU=',
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
@fixture
|
||||
def import_workflow() -> ImportModel:
|
||||
return ImportModel()
|
||||
|
||||
|
||||
def run_with(recorder: Recorder):
|
||||
"""Patch the workflow module's `workflow` object and hand back the patch context."""
|
||||
patcher = patch('laborious.workflows.import_model.workflow')
|
||||
workflow_mock = patcher.start()
|
||||
workflow_mock.info.return_value = MagicMock(workflow_id=WORKFLOW_ID)
|
||||
# Assigned directly rather than wrapped in an `AsyncMock`: a mock whose side effect is an
|
||||
# instance with an `async def __call__` hands back the coroutine instead of awaiting it.
|
||||
workflow_mock.execute_activity_method = recorder
|
||||
return patcher, workflow_mock
|
||||
|
||||
|
||||
@fixture
|
||||
def recorder():
|
||||
return Recorder()
|
||||
|
||||
|
||||
@fixture
|
||||
def workflow_env(recorder: Recorder):
|
||||
patcher, workflow_mock = run_with(recorder)
|
||||
yield SimpleNamespace(recorder=recorder, workflow=workflow_mock)
|
||||
patcher.stop()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------- order
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_successful_import_calls_its_activities_in_order(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
result = await import_workflow.run(valid_input())
|
||||
|
||||
assert workflow_env.recorder.order == EXPECTED_ORDER
|
||||
assert result == {
|
||||
'import_run_id': 77,
|
||||
'run_id': RUN_ID,
|
||||
'model_name': 'factory_model',
|
||||
'version': '4',
|
||||
'document_id': '8',
|
||||
}
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_claim_is_the_first_thing_that_happens(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
assert workflow_env.recorder.order[0] is Activities.claim_import_status
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_names_are_recorded_before_mlflow_is_touched(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
"""So a person watching the import list sees which model is being imported, while it runs."""
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
order = workflow_env.recorder.order
|
||||
assert order.index(Activities.record_import_names) < order.index(
|
||||
Activities.create_import_experiment
|
||||
)
|
||||
names = workflow_env.recorder.payload_of(Activities.record_import_names)
|
||||
assert names['experiment_name'] == 'factory_experiment'
|
||||
assert names['run_name'] == 'import-factory_model-v2'
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_run_is_created_with_exactly_the_recorded_name(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
recorded = workflow_env.recorder.payload_of(Activities.record_import_names)['run_name']
|
||||
used = workflow_env.recorder.payload_of(Activities.upload_import_artifacts)['run_name']
|
||||
assert used == recorded
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_model_document_is_the_last_thing_provisioned(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
order = workflow_env.recorder.order
|
||||
for step in PROVISIONING[:-1]:
|
||||
assert order.index(step) < order.index(Activities.write_import_model_document)
|
||||
assert order[-2:] == [Activities.cleanup_import_files, Activities.record_import_terminal_status]
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_document_target_comes_from_the_bundle_s_parameters(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
assert (
|
||||
workflow_env.recorder.payload_of(Activities.write_import_model_document)['target'] == 'SE'
|
||||
)
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_terminal_write_reports_a_success_with_its_last_step(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
terminal = workflow_env.recorder.payload_of(Activities.record_import_terminal_status)
|
||||
assert terminal['succeeded'] is True
|
||||
assert terminal['step'] == ImportStep.MODEL_DOCUMENT.value
|
||||
assert terminal['cleanup_failed'] is False
|
||||
assert terminal['source'] == BUNDLE_METADATA
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_download_is_handed_the_digest_the_request_carried(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
payload = workflow_env.recorder.payload_of(Activities.download_import_bundle)
|
||||
assert payload['expected_digest'] == DIGEST
|
||||
assert payload['object_key'] == OBJECT_KEY
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- fail-fast
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@mark.parametrize(
|
||||
('failing', 'step'),
|
||||
[
|
||||
(Activities.download_import_bundle, ImportStep.DOWNLOAD),
|
||||
(Activities.open_import_bundle, ImportStep.DECRYPTION),
|
||||
(Activities.create_import_experiment, ImportStep.EXPERIMENT_CREATION),
|
||||
(Activities.upload_import_artifacts, ImportStep.ARTIFACT_UPLOAD),
|
||||
(Activities.register_import_model_version, ImportStep.REGISTRATION),
|
||||
(Activities.write_import_model_document, ImportStep.MODEL_DOCUMENT),
|
||||
],
|
||||
ids=lambda value: getattr(value, '__name__', str(value)),
|
||||
)
|
||||
async def test_a_failing_step_is_recorded_as_that_step_and_stops_the_import(
|
||||
import_workflow: ImportModel, failing, step: ImportStep
|
||||
):
|
||||
recorder = Recorder(failures={failing: RuntimeError('dependency unavailable')})
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(RuntimeError, match='dependency unavailable'):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
terminal = recorder.payload_of(Activities.record_import_terminal_status)
|
||||
assert terminal['succeeded'] is False
|
||||
assert terminal['step'] == step.value
|
||||
# Nothing after the failing step was attempted, and cleanup still ran before the verdict.
|
||||
order = recorder.order
|
||||
assert order[-2:] == [Activities.cleanup_import_files, Activities.record_import_terminal_status]
|
||||
later = EXPECTED_ORDER[EXPECTED_ORDER.index(failing) + 1 :]
|
||||
for method in later:
|
||||
if method in (Activities.cleanup_import_files, Activities.record_import_terminal_status):
|
||||
continue
|
||||
if method in STATUS_ACTIVITIES:
|
||||
continue
|
||||
assert method not in order
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_nothing_is_provisioned_before_the_gates_pass(import_workflow: ImportModel):
|
||||
recorder = Recorder(
|
||||
failures={
|
||||
Activities.open_import_bundle: ImportGateRejectedError(
|
||||
'rejected', step=ImportStep.ARCHIVE_INSPECTION, gate=4
|
||||
)
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(ImportGateRejectedError):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
for method in PROVISIONING:
|
||||
assert method not in recorder.order
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_gate_rejection_is_recorded_with_the_gate_that_rejected_it(
|
||||
import_workflow: ImportModel,
|
||||
):
|
||||
"""Gates 2 to 7 share one activity, so the step travels in the error's details, not in the
|
||||
step the workflow was attempting."""
|
||||
recorder = Recorder(
|
||||
failures={
|
||||
Activities.open_import_bundle: ImportGateRejectedError(
|
||||
'rejected', step=ImportStep.CONTENT_POLICY, gate=7
|
||||
)
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(ImportGateRejectedError):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
terminal = recorder.payload_of(Activities.record_import_terminal_status)
|
||||
assert terminal['step'] == ImportStep.CONTENT_POLICY.value
|
||||
assert terminal['gate'] == 7
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_an_ambiguous_decryption_failure_keeps_its_step(import_workflow: ImportModel):
|
||||
recorder = Recorder(
|
||||
failures={
|
||||
Activities.open_import_bundle: ImportGateRejectedError(
|
||||
'the bundle could not be opened', step=ImportStep.DECRYPTION, gate=3
|
||||
)
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(ImportGateRejectedError):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
terminal = recorder.payload_of(Activities.record_import_terminal_status)
|
||||
assert terminal['step'] == ImportStep.DECRYPTION.value
|
||||
assert terminal['gate'] == 3
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- the input contract
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@mark.parametrize('field', list(REJECTED_INPUT_FIELDS))
|
||||
async def test_an_unknown_field_is_rejected_rather_than_ignored(
|
||||
import_workflow: ImportModel, field: str
|
||||
):
|
||||
"""A caller that believes it recorded a project link must be told it did not."""
|
||||
recorder = Recorder()
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(ImportInputError, match='does not accept'):
|
||||
await import_workflow.run(valid_input(**{field: 'whatever'}))
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
# After the claim, so the refusal lands on the record instead of only in the logs.
|
||||
assert recorder.order[0] is Activities.claim_import_status
|
||||
assert Activities.download_import_bundle not in recorder.order
|
||||
terminal = recorder.payload_of(Activities.record_import_terminal_status)
|
||||
assert terminal['step'] == ImportStep.RECEIVED.value
|
||||
assert terminal['succeeded'] is False
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@mark.parametrize(
|
||||
'digest',
|
||||
[None, '', 'not-a-digest', 'a' * 63, 'a' * 65, 'g' * 64],
|
||||
ids=['missing', 'empty', 'short-text', 'too-short', 'too-long', 'not-hex'],
|
||||
)
|
||||
async def test_a_missing_or_malformed_digest_fails_before_anything_is_downloaded(
|
||||
import_workflow: ImportModel, digest
|
||||
):
|
||||
recorder = Recorder()
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(ImportInputError):
|
||||
await import_workflow.run(valid_input(expected_digest=digest))
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert Activities.download_import_bundle not in recorder.order
|
||||
assert (
|
||||
recorder.payload_of(Activities.record_import_terminal_status)['step']
|
||||
== ImportStep.RECEIVED.value
|
||||
)
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@mark.parametrize('missing', ['object_key', 'password_envelope'])
|
||||
async def test_a_request_missing_a_required_field_fails_at_received(
|
||||
import_workflow: ImportModel, missing: str
|
||||
):
|
||||
recorder = Recorder()
|
||||
patcher, _ = run_with(recorder)
|
||||
payload = valid_input()
|
||||
payload.pop(missing)
|
||||
try:
|
||||
with raises(ImportInputError, match=missing):
|
||||
await import_workflow.run(payload)
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert Activities.download_import_bundle not in recorder.order
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_record_naming_another_object_is_a_received_failure(
|
||||
import_workflow: ImportModel,
|
||||
):
|
||||
recorder = Recorder(
|
||||
results={
|
||||
Activities.claim_import_status: {
|
||||
'claimed': True,
|
||||
'file_name': 'imported_models/somebody_elses.sientia',
|
||||
'request_digest': None,
|
||||
}
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(ImportInputError, match='different object'):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert Activities.download_import_bundle not in recorder.order
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_record_naming_another_digest_is_a_received_failure(
|
||||
import_workflow: ImportModel,
|
||||
):
|
||||
recorder = Recorder(
|
||||
results={
|
||||
Activities.claim_import_status: {
|
||||
'claimed': True,
|
||||
'file_name': OBJECT_KEY,
|
||||
'request_digest': 'b' * 64,
|
||||
}
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(ImportInputError, match='different digest'):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert Activities.download_import_bundle not in recorder.order
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@mark.parametrize('value', [None, 0, -1, '77', 7.0, True], ids=lambda v: f'{type(v).__name__}-{v}')
|
||||
async def test_an_unusable_record_id_fails_before_any_activity(import_workflow: ImportModel, value):
|
||||
"""There is no record to claim and none to write: the run ends FAILED and nothing is touched."""
|
||||
recorder = Recorder()
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(ImportInputError, match='positive integer'):
|
||||
await import_workflow.run(valid_input(import_run_id=value))
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert recorder.calls == []
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_claim_that_matches_nothing_writes_nothing_at_all(import_workflow: ImportModel):
|
||||
recorder = Recorder(
|
||||
failures={
|
||||
Activities.claim_import_status: ImportLogRowNotClaimableError(
|
||||
'import log row 77 could not be claimed: no such import log row'
|
||||
)
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(ImportLogRowNotClaimableError):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
# The claim is outside the try/finally on purpose: no row is owned, so there is nothing to clean
|
||||
# up and nothing to record.
|
||||
assert recorder.order == [Activities.claim_import_status]
|
||||
|
||||
|
||||
# ----------------------------------------------------------- failures that must not fail
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_lost_hint_does_not_fail_a_working_import(import_workflow: ImportModel):
|
||||
recorder = Recorder(failures={Activities.record_import_status: RuntimeError('db down')})
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
result = await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert result['run_id'] == RUN_ID
|
||||
assert Activities.report_import_status_write_failure in recorder.order
|
||||
assert recorder.payload_of(Activities.record_import_terminal_status)['succeeded'] is True
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_lost_names_write_does_not_fail_a_working_import(import_workflow: ImportModel):
|
||||
recorder = Recorder(failures={Activities.record_import_names: RuntimeError('db down')})
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
result = await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert result['version'] == '4'
|
||||
assert Activities.upload_import_artifacts in recorder.order
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_retention_rule_that_cannot_be_set_does_not_fail_an_import(
|
||||
import_workflow: ImportModel,
|
||||
):
|
||||
recorder = Recorder(
|
||||
failures={Activities.ensure_import_bundle_retention: RuntimeError('access denied')}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
result = await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert result['run_id'] == RUN_ID
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_cleanup_failure_is_carried_into_the_verdict_not_raised(
|
||||
import_workflow: ImportModel,
|
||||
):
|
||||
recorder = Recorder(failures={Activities.cleanup_import_files: RuntimeError('busy')})
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
result = await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
terminal = recorder.payload_of(Activities.record_import_terminal_status)
|
||||
assert result['run_id'] == RUN_ID
|
||||
assert terminal['succeeded'] is True
|
||||
assert terminal['cleanup_failed'] is True
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_terminal_write_that_cannot_land_does_not_invent_a_failed_import(
|
||||
import_workflow: ImportModel,
|
||||
):
|
||||
recorder = Recorder(
|
||||
failures={Activities.record_import_terminal_status: RuntimeError('still down')}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
result = await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert result['run_id'] == RUN_ID
|
||||
reported = recorder.payload_of(Activities.report_import_status_write_failure)
|
||||
assert reported['verdict'] == 'succeeded=True step=model_document gate=None'
|
||||
assert 'still down' in reported['detail']
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_a_terminal_write_failure_does_not_mask_a_real_failure(
|
||||
import_workflow: ImportModel,
|
||||
):
|
||||
recorder = Recorder(
|
||||
failures={
|
||||
Activities.register_import_model_version: RuntimeError('registry refused'),
|
||||
Activities.record_import_terminal_status: RuntimeError('status db down'),
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
with raises(RuntimeError, match='registry refused'):
|
||||
await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
reported = recorder.payload_of(Activities.report_import_status_write_failure)
|
||||
assert reported['verdict'] == 'succeeded=False step=registration gate=None'
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_reporting_the_report_is_where_it_stops(import_workflow: ImportModel):
|
||||
"""A failure in the failure channel cannot become the import's outcome."""
|
||||
recorder = Recorder(
|
||||
failures={
|
||||
Activities.record_import_terminal_status: RuntimeError('down'),
|
||||
Activities.report_import_status_write_failure: RuntimeError('also down'),
|
||||
}
|
||||
)
|
||||
patcher, _ = run_with(recorder)
|
||||
try:
|
||||
result = await import_workflow.run(valid_input())
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert result['run_id'] == RUN_ID
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- the retry split
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_only_the_status_writes_retry(import_workflow: ImportModel, workflow_env):
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
for call in workflow_env.recorder.calls:
|
||||
if call.method in STATUS_ACTIVITIES:
|
||||
continue
|
||||
assert call.kwargs['retry_policy'] is retry_policy
|
||||
assert retry_policy.maximum_attempts == 1
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_claim_and_the_terminal_write_use_the_long_envelope(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
for method in CLAIM_AND_TERMINAL:
|
||||
for call in workflow_env.recorder.calls_to(method):
|
||||
assert call.kwargs['retry_policy'] is IMPORT_STATUS_RETRY_POLICY
|
||||
assert call.kwargs['start_to_close_timeout'] == STATUS_WRITE_TIMEOUT
|
||||
assert call.kwargs['schedule_to_close_timeout'] == STATUS_WRITE_ENVELOPE
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_hints_use_the_short_policy(import_workflow: ImportModel, workflow_env):
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
for method in HINTS:
|
||||
for call in workflow_env.recorder.calls_to(method):
|
||||
assert call.kwargs['retry_policy'] is IMPORT_STATUS_HINT_RETRY_POLICY
|
||||
# No long envelope: a hint that cannot land must not hold the import open.
|
||||
assert 'schedule_to_close_timeout' not in call.kwargs
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_no_provisioning_call_references_a_retrying_policy(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
"""A repeated registry call or Mongo insert writes twice; that is why they cannot retry."""
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
for call in workflow_env.recorder.calls:
|
||||
if call.method in PROVISIONING or call.method is Activities.open_import_bundle:
|
||||
assert call.kwargs['retry_policy'] not in (
|
||||
IMPORT_STATUS_RETRY_POLICY,
|
||||
IMPORT_STATUS_HINT_RETRY_POLICY,
|
||||
)
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_the_only_retrying_calls_are_the_four_status_writes(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
retrying = {
|
||||
call.method
|
||||
for call in workflow_env.recorder.calls
|
||||
if call.kwargs['retry_policy']
|
||||
in (IMPORT_STATUS_RETRY_POLICY, IMPORT_STATUS_HINT_RETRY_POLICY)
|
||||
}
|
||||
assert retrying == set(STATUS_ACTIVITIES)
|
||||
|
||||
|
||||
def test_the_retry_policies_are_the_values_the_design_chose():
|
||||
assert IMPORT_STATUS_RETRY_POLICY.maximum_attempts == 10
|
||||
assert IMPORT_STATUS_RETRY_POLICY.initial_interval == timedelta(seconds=1)
|
||||
assert IMPORT_STATUS_RETRY_POLICY.backoff_coefficient == 2.0
|
||||
assert IMPORT_STATUS_RETRY_POLICY.maximum_interval == timedelta(seconds=30)
|
||||
assert IMPORT_STATUS_HINT_RETRY_POLICY.maximum_attempts == 3
|
||||
assert IMPORT_STATUS_HINT_RETRY_POLICY.maximum_interval == timedelta(seconds=2)
|
||||
|
||||
|
||||
def test_the_terminal_write_s_envelope_outlasts_its_own_retries():
|
||||
"""Set this smaller and Temporal cuts the retries short, silently undoing the retry split."""
|
||||
backoff = sum(
|
||||
min(
|
||||
IMPORT_STATUS_RETRY_POLICY.initial_interval.total_seconds()
|
||||
* IMPORT_STATUS_RETRY_POLICY.backoff_coefficient**attempt,
|
||||
IMPORT_STATUS_RETRY_POLICY.maximum_interval.total_seconds(),
|
||||
)
|
||||
for attempt in range(IMPORT_STATUS_RETRY_POLICY.maximum_attempts - 1)
|
||||
)
|
||||
|
||||
assert backoff == 151
|
||||
assert STATUS_WRITE_ENVELOPE.total_seconds() > backoff
|
||||
|
||||
|
||||
def test_a_hint_gives_up_inside_a_short_bound():
|
||||
"""Three attempts capped at two seconds: a lost hint costs the import seconds, not minutes."""
|
||||
backoff = sum(
|
||||
min(
|
||||
IMPORT_STATUS_HINT_RETRY_POLICY.initial_interval.total_seconds()
|
||||
* IMPORT_STATUS_HINT_RETRY_POLICY.backoff_coefficient**attempt,
|
||||
IMPORT_STATUS_HINT_RETRY_POLICY.maximum_interval.total_seconds(),
|
||||
)
|
||||
for attempt in range(IMPORT_STATUS_HINT_RETRY_POLICY.maximum_attempts - 1)
|
||||
)
|
||||
|
||||
assert backoff <= 3
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_no_retrying_call_carries_a_payload_that_would_insert(
|
||||
import_workflow: ImportModel, workflow_env
|
||||
):
|
||||
"""What retries must be idempotent, so the payload may not name a row to create."""
|
||||
await import_workflow.run(valid_input())
|
||||
|
||||
for call in workflow_env.recorder.calls:
|
||||
if call.method not in STATUS_ACTIVITIES:
|
||||
continue
|
||||
assert call.payload['import_run_id'] == 77
|
||||
for owned in ('username', 'request_data', 'bucket_name', 'file_name', 'run_type'):
|
||||
assert owned not in call.payload
|
||||
Reference in New Issue
Block a user