vitor-aignosi
853c83e662
QTZPOC-20: Improve structured logging and reporting for all model import pipeline failures by introducing a dedicated activity and centralizing error classification. This ensures consistent observability, with all workflow failures now reported before cleanup. Also, adjusted import step timeouts and refined manual run script cleanup logic.
265 lines
9.3 KiB
Python
265 lines
9.3 KiB
Python
"""What the import deliberately does **not** do.
|
|
|
|
Every absence here was a decision, and every one of them is the kind of thing a later well-meaning
|
|
edit adds back: a rollback for a failed import, a resume for a half-finished one, a signal to retry,
|
|
a `joblib.load` to "check the model is valid", a reason string built from the exception. None of
|
|
those are missing features — a failed import stays failed, and trying again is a new upload — so the
|
|
tests are written against the source itself, because there is no behaviour to observe when the
|
|
behaviour is the point of not existing.
|
|
"""
|
|
|
|
import ast
|
|
from pathlib import Path
|
|
|
|
from pytest import mark
|
|
|
|
from laborious.workflows.import_model import ImportModel
|
|
|
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
WORKFLOW_MODULE = REPOSITORY_ROOT / 'laborious/workflows/import_model.py'
|
|
ACTIVITIES_MODULE = REPOSITORY_ROOT / 'laborious/activities/model_import.py'
|
|
ERRORS_MODULE = REPOSITORY_ROOT / 'laborious/utils/models/import_errors.py'
|
|
READER_MODULE = REPOSITORY_ROOT / 'laborious/utils/bundle/reader.py'
|
|
FORMAT_MODULE = REPOSITORY_ROOT / 'laborious/utils/bundle/format.py'
|
|
|
|
IMPORT_PATH_MODULES = (
|
|
WORKFLOW_MODULE,
|
|
ACTIVITIES_MODULE,
|
|
ERRORS_MODULE,
|
|
READER_MODULE,
|
|
FORMAT_MODULE,
|
|
)
|
|
|
|
# The vocabulary of recovery. A partial import is not repaired; it is reported and abandoned.
|
|
RECOVERY_WORDS = ('rollback', 'compensate', 'resume', 'reconcile', 'undo')
|
|
|
|
# Loading a model is the *execution* side, owned by another card. The importer moves bytes and
|
|
# writes metadata; it never unpickles what it just accepted from a user.
|
|
MODEL_LOADING_CALLS = (
|
|
'mlflow.pyfunc.load_model',
|
|
'mlflow.sklearn.load_model',
|
|
'joblib.load',
|
|
'pickle.load',
|
|
'pickle.loads',
|
|
)
|
|
|
|
# Decryption lives in the activity that uses the password, never in the workflow, which would put
|
|
# the plaintext into the event history.
|
|
DECRYPTION_PRIMITIVES = (
|
|
'AESGCM',
|
|
'crypto_pwhash',
|
|
'crypto_secretstream',
|
|
'decrypt_password_envelope',
|
|
'load_envelope_key',
|
|
'IMPORT_PASSWORD_KEY',
|
|
'Argon2',
|
|
)
|
|
|
|
|
|
def parse(path: Path) -> ast.Module:
|
|
return ast.parse(path.read_text())
|
|
|
|
|
|
def called_names(tree: ast.Module) -> list[str]:
|
|
"""Every called name, as written: `a.b.c(...)` becomes `a.b.c`."""
|
|
names = []
|
|
for node in ast.walk(tree):
|
|
if not isinstance(node, ast.Call):
|
|
continue
|
|
names.append(ast.unparse(node.func))
|
|
return names
|
|
|
|
|
|
def defined_names(tree: ast.Module) -> list[str]:
|
|
return [
|
|
node.name
|
|
for node in ast.walk(tree)
|
|
if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef)
|
|
]
|
|
|
|
|
|
# ------------------------------------------------------------- no retry beyond the record
|
|
|
|
|
|
def test_the_workflow_holds_no_recovery_routine():
|
|
tree = parse(WORKFLOW_MODULE)
|
|
|
|
for name in defined_names(tree) + called_names(tree):
|
|
for word in RECOVERY_WORDS:
|
|
assert word not in name.lower(), f'{name} looks like recovery'
|
|
|
|
|
|
def test_the_activities_hold_no_recovery_routine():
|
|
tree = parse(ACTIVITIES_MODULE)
|
|
|
|
for name in defined_names(tree) + called_names(tree):
|
|
for word in RECOVERY_WORDS:
|
|
assert word not in name.lower(), f'{name} looks like recovery'
|
|
|
|
|
|
def test_the_workflow_never_continues_as_new():
|
|
"""A single import is one workflow execution, start to finish."""
|
|
assert not [name for name in called_names(parse(WORKFLOW_MODULE)) if 'continue_as_new' in name]
|
|
|
|
|
|
# ------------------------------------------------------------------ no handlers, no waiting
|
|
|
|
|
|
def test_the_workflow_declares_no_signal_or_query_handler():
|
|
"""Nothing outside can steer an import once it starts: no retry signal, no progress query."""
|
|
for name in dir(ImportModel):
|
|
member = getattr(ImportModel, name, None)
|
|
for marker in (
|
|
'__temporal_signal_definition',
|
|
'__temporal_query_definition',
|
|
'__temporal_update_definition',
|
|
):
|
|
assert not hasattr(member, marker), f'{name} declares {marker}'
|
|
|
|
|
|
def test_the_workflow_module_carries_no_handler_decorator():
|
|
tree = parse(WORKFLOW_MODULE)
|
|
decorators = [
|
|
ast.unparse(decorator)
|
|
for node in ast.walk(tree)
|
|
if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef)
|
|
for decorator in node.decorator_list
|
|
]
|
|
|
|
assert set(decorators) == {'workflow.run', 'staticmethod'}
|
|
assert decorators.count('workflow.run') == 1
|
|
|
|
|
|
def test_the_workflow_never_waits_for_a_condition():
|
|
"""No `wait_condition`: there is no external event an import waits on."""
|
|
assert not [name for name in called_names(parse(WORKFLOW_MODULE)) if 'wait_condition' in name]
|
|
|
|
|
|
# --------------------------------------------------------------------- nothing is loaded
|
|
|
|
|
|
@mark.parametrize('module_path', IMPORT_PATH_MODULES, ids=lambda path: path.name)
|
|
def test_no_module_in_the_import_path_loads_a_model(module_path: Path):
|
|
source = module_path.read_text()
|
|
|
|
for call in MODEL_LOADING_CALLS:
|
|
assert call not in source
|
|
|
|
|
|
def test_the_reader_knows_nothing_about_mlflow():
|
|
"""It stays liftable into a shared package: the reader is about the file, not the platform."""
|
|
source = READER_MODULE.read_text()
|
|
|
|
assert 'mlflow' not in source
|
|
assert 'temporalio' not in source
|
|
|
|
|
|
# ----------------------------------------------------------- the workflow holds no secret
|
|
|
|
|
|
@mark.parametrize('primitive', DECRYPTION_PRIMITIVES)
|
|
def test_the_workflow_module_holds_no_decryption_primitive(primitive: str):
|
|
assert primitive not in WORKFLOW_MODULE.read_text()
|
|
|
|
|
|
def test_the_workflow_only_ever_holds_the_envelope():
|
|
"""The envelope crosses the workflow; the password it wraps never does."""
|
|
source = WORKFLOW_MODULE.read_text()
|
|
|
|
assert 'password_envelope' in source
|
|
# The only other mention of a password is the input field the workflow refuses to accept.
|
|
assert source.count("'password'") == 1
|
|
|
|
|
|
def test_the_plaintext_password_field_is_refused_by_name():
|
|
from laborious.workflows.import_model import REJECTED_INPUT_FIELDS
|
|
|
|
assert 'password' in REJECTED_INPUT_FIELDS
|
|
|
|
|
|
# ------------------------------------------------------- every reason comes from the catalog
|
|
|
|
|
|
def _error_message_values(tree: ast.Module) -> list[ast.expr]:
|
|
"""Every expression written into the record's `error_message`."""
|
|
values = []
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Dict):
|
|
for key, value in zip(node.keys, node.values, strict=True):
|
|
if isinstance(key, ast.Constant) and key.value == 'error_message':
|
|
values.append(value)
|
|
if isinstance(node, ast.keyword) and node.arg == 'error_message':
|
|
values.append(node.value)
|
|
return values
|
|
|
|
|
|
def test_no_recorded_reason_is_built_from_an_exception():
|
|
"""The catalog is the only source of a sentence a user reads."""
|
|
values = _error_message_values(parse(ACTIVITIES_MODULE))
|
|
|
|
assert values, 'no error_message write found — has the terminal statement moved?'
|
|
for value in values:
|
|
assert isinstance(value, ast.Name | ast.Constant), ast.unparse(value)
|
|
assert not isinstance(value, ast.JoinedStr | ast.BinOp)
|
|
|
|
|
|
def test_the_reason_written_is_the_classified_one():
|
|
values = _error_message_values(parse(ACTIVITIES_MODULE))
|
|
|
|
assert {ast.unparse(value) for value in values} == {'reason'}
|
|
|
|
|
|
def test_the_classification_ignores_the_exception_by_construction():
|
|
"""`classify_import_failure` deletes its exception argument; nothing can leak from it."""
|
|
source = ERRORS_MODULE.read_text()
|
|
|
|
assert 'del exception' in source
|
|
|
|
|
|
@mark.parametrize('module_path', (WORKFLOW_MODULE, ACTIVITIES_MODULE), ids=lambda p: p.name)
|
|
def test_no_status_detail_interpolates_an_exception(module_path: Path):
|
|
"""The JSON detail is built from a code and a step, both of them enum values."""
|
|
tree = parse(module_path)
|
|
|
|
for node in ast.walk(tree):
|
|
if not isinstance(node, ast.Call) or ast.unparse(node.func) != 'build_import_detail':
|
|
continue
|
|
for keyword in node.keywords:
|
|
rendered = ast.unparse(keyword.value)
|
|
assert 'error' not in rendered
|
|
assert 'exception' not in rendered
|
|
assert not isinstance(keyword.value, ast.JoinedStr)
|
|
|
|
|
|
# ------------------------------------------------------------------ no relational target
|
|
|
|
|
|
def test_the_import_writes_to_one_table_and_names_it_once():
|
|
"""D22: no project table, no model table, no second database. One `UPDATE`, one relation."""
|
|
statements = []
|
|
for node in ast.walk(parse(ACTIVITIES_MODULE)):
|
|
if not isinstance(node, ast.Call) or ast.unparse(node.func) != 'text':
|
|
continue
|
|
argument = node.args[0]
|
|
if isinstance(argument, ast.Constant):
|
|
statements.append(argument.value)
|
|
|
|
assert statements, 'no SQL statement found in the import activities'
|
|
for statement in statements:
|
|
words = statement.split()
|
|
relations = {
|
|
words[index + 1]
|
|
for index, word in enumerate(words)
|
|
if word.upper() in ('UPDATE', 'FROM', 'INTO', 'JOIN')
|
|
}
|
|
assert relations == {'public.experiment_run'}
|
|
|
|
|
|
def test_the_import_never_inserts_a_relational_row():
|
|
source = ACTIVITIES_MODULE.read_text().upper()
|
|
|
|
assert 'INSERT INTO' not in source
|
|
assert 'DELETE FROM' not in source
|
|
assert 'CREATE TABLE' not in source
|