gitea_admin/sientia-dataops-laborious-legacy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

QTZPOC-20

Browse Source 
QTZPOC-20: Implement model import workflow for .sientia bundles

Introduces the import_model workflow and its associated activities to enable importing .sientia bundles into MLflow and MongoDB. This includes:

*   New core workflow and activity logic for handling bundle download, decryption, validation, MLflow provisioning (experiments, runs, registered versions), and MongoDB model document creation.
*   Comprehensive E2E and unit test suites, including a dedicated test harness and a manual run script for development.
*   Updated .env.example and README.md with new configurations and usage instructions.
*   Ensures correct MLflow model source resolution for legacy loader compatibility and robust MongoDB document management.
*   Refactors existing utility scripts into a scripts/ directory.
This commit is contained in:
Vitor Pimentel dos Santos
2026-08-25 17:07:42 -03:00
parent 0e445de386
commit dac4f1718f
41 changed files with 8168 additions and 1045 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
e2e/conftest.py
View File

@@ -33,6 +33,11 @@ from laborious.workflows.predictions_batch import PredictionsBatch
from laborious.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction
from laborious.workflows.sub_workflows.prediction_process import PredictionProcess
# The `.sientia` import fixtures live in their own module rather than here: four containers' worth
# of setup, an MLflow server subprocess and an ImportModel worker would double this file. Every
# fixture it adds is `import_`-prefixed, so nothing below changes meaning.
pytest_plugins = ['e2e.import_harness']
# Test constants
TEST_MONGODB_CONNECTION_STRING = 'mongodb://localhost:27017'
TEST_DATABASE_NAME = 'test_db'
@@ -60,8 +65,12 @@ def postgres_container():
This fixture creates a real PostgreSQL database in a Docker container
that will be used for all tests in the session.
`log_statement=all` is the one thing the import suite needs from this fixture: proving an import
issues no statement at all against the predictions database requires a log that would have
shown one. It changes what the server writes to its own log and nothing a test observes.
"""
postgres = PostgresContainer('postgres:15')
postgres = PostgresContainer('postgres:15').with_command('postgres -c log_statement=all')
postgres.start()
yield postgres
postgres.stop()

22
e2e/fixtures/experiment_run.sql
View File

@@ -1,4 +1,4 @@
-- public.experiment_run as migrations V1 … V12 leave it, for the e2e Postgres container.
-- public.experiment_run as migrations V1 … V11 leave it, for the e2e Postgres container.
--
-- Source of truth: sientia-dataops-database-migrations,
-- postgres/sientia-core-mlops-bff/migrations/ — this file mirrors the *final state* of:
@@ -14,7 +14,11 @@
-- V9 remove_file_upload_statuses (status vocabulary)
-- V10 remove_unused_statuses (status vocabulary)
-- V11 add_run_type_to_experiment_run (run_type + reindexed)
-- V12 add_import_statuses_to_experiment_run (IMPORT_SUCCESS / IMPORT_ERROR + pairing)
--
-- V12 (add_import_statuses_to_experiment_run) lives in `experiment_run_v12.sql`, applied on top of
-- this file. It is separate because one e2e test — the evidence for why the migration gates
-- enabling the import worker — has to run against a table that does *not* have it, and a second
-- whole-schema copy with V12 removed would rot the moment either copy were edited.
--
-- The same drift trade already accepted for the bundle fixtures: this is a copy, and the tests
-- that read it are what make the copy auditable. If a migration lands there, update this file and
@@ -48,19 +52,13 @@ CREATE TABLE IF NOT EXISTS public.experiment_run (
CONSTRAINT chk_experiment_run_file_name_pattern
CHECK (file_name ~* '^[a-zA-Z0-9 /._-]+$'),
-- V12: the two shared stage values plus the training and import terminals.
-- The V1…V11 status vocabulary: the two shared stage values plus the training terminals.
-- V12 adds IMPORT_SUCCESS and IMPORT_ERROR to this list.
CONSTRAINT chk_experiment_run_status CHECK (status IN ('PENDING', 'ORCHESTRATOR_REQUEST_ERROR',
'ORCHESTRATOR_WAITING_PROC',
'ORCHESTRATOR_VALIDATION_ERROR',
'TRAINING_SUCCESS', 'TRAINING_ERROR',
'IMPORT_SUCCESS', 'IMPORT_ERROR')),
CONSTRAINT chk_experiment_run_run_type CHECK (run_type IN ('TRAIN', 'IMPORT')),
-- V12: each pair of terminal values belongs to its own run_type. The shared values stay out.
CONSTRAINT chk_experiment_run_terminal_status_run_type CHECK (
(run_type = 'IMPORT' OR status NOT IN ('IMPORT_SUCCESS', 'IMPORT_ERROR'))
AND (run_type = 'TRAIN' OR status NOT IN ('TRAINING_SUCCESS', 'TRAINING_ERROR'))
)
'TRAINING_SUCCESS', 'TRAINING_ERROR')),
CONSTRAINT chk_experiment_run_run_type CHECK (run_type IN ('TRAIN', 'IMPORT'))
);
CREATE INDEX IF NOT EXISTS idx_experiment_run_user_exp_status_date_type

36
e2e/fixtures/experiment_run_v12.sql Normal file
View File

@@ -0,0 +1,36 @@
-- V12 add_import_statuses_to_experiment_run, as a delta applied on top of `experiment_run.sql`.
--
-- Source of truth: sientia-dataops-database-migrations,
-- postgres/sientia-core-mlops-bff/migrations/V12__add_import_statuses_to_experiment_run.sql
--
-- Two things, and nothing else:
--
-- 1. the status vocabulary gains IMPORT_SUCCESS and IMPORT_ERROR;
-- 2. chk_experiment_run_terminal_status_run_type pairs each terminal value with its own
-- run_type, so a TRAIN row can never end IMPORT_SUCCESS and an IMPORT row can never end
-- TRAINING_ERROR. The two shared stage values (PENDING, ORCHESTRATOR_WAITING_PROC) stay out
-- of the pairing — both run types legitimately pass through them.
--
-- This file is applied separately because one e2e test runs an import against a table *without*
-- it: that failure shape — provisioned, then unrecordable, refused on the first attempt — is the
-- evidence for why this migration has to land before IMPORT_MODEL_ENABLED is turned on anywhere.
ALTER TABLE public.experiment_run
DROP CONSTRAINT IF EXISTS chk_experiment_run_status;
ALTER TABLE public.experiment_run
ADD CONSTRAINT chk_experiment_run_status CHECK (status IN ('PENDING',
'ORCHESTRATOR_REQUEST_ERROR',
'ORCHESTRATOR_WAITING_PROC',
'ORCHESTRATOR_VALIDATION_ERROR',
'TRAINING_SUCCESS', 'TRAINING_ERROR',
'IMPORT_SUCCESS', 'IMPORT_ERROR'));
ALTER TABLE public.experiment_run
DROP CONSTRAINT IF EXISTS chk_experiment_run_terminal_status_run_type;
ALTER TABLE public.experiment_run
ADD CONSTRAINT chk_experiment_run_terminal_status_run_type CHECK (
(run_type = 'IMPORT' OR status NOT IN ('IMPORT_SUCCESS', 'IMPORT_ERROR'))
AND (run_type = 'TRAIN' OR status NOT IN ('TRAINING_SUCCESS', 'TRAINING_ERROR'))
);

950
e2e/import_harness.py Normal file
View File

@@ -0,0 +1,950 @@
"""Fixtures for the `.sientia` model-import e2e suite. Registered from `e2e/conftest.py`.
**Why this is a module of its own.** `e2e/conftest.py` is shared by every existing e2e suite and
long enough already; four containers' worth of setup, an MLflow server subprocess, the import
environment variables and an `ImportModel` worker do not belong in it. Every fixture here is
prefixed `import_`, so nothing collides with the predictions harness, and no existing fixture is
edited — the one exception being `log_statement=all` on the shared `postgres_container`, which the
absence assertions need and which changes no test's behaviour.
**The fidelity contract, which is the whole point of this file.** On the import path *nothing is
mocked*:
| Dependency | Here | Why real |
|---|---|---|
| MinIO | the shared `minio_container`, plus an import bucket | gate 1 reads a real object |
| BFF Postgres | a container of its own, `sientia-core-mlops-bff` | the real `CHECK` constraints are half of what is under test, and two tests stop the server |
| MongoDB | a container, `models` seeded with gapped ids and a unique index on `id` | `max(int(id)) + 1` against a real unique index |
| MLflow | `mlflow server` from *this* venv, sqlite-backed, artifacts proxied | experiment/run/artifact/registry semantics are what the provisioning activities are tested for |
| the crypto | the production reader, real Argon2id, real `secretstream` | a bundle the producer wrote must open |
What stays stubbed, deliberately: `CoreNotificationHandler`'s Mongo client keeps the existing
`mock_mongo_client`. Notifications are a side channel this suite counts, not a dependency it tests,
and a real notification collection would add a second write path to every absence assertion for
nothing.
The import `Activities` is **never** composed with `patch_mlflow`, `patch_minio_repository` or
`patch_create_engine`. `mlflow.set_tracking_uri` is process-global, so a leak would silently turn a
real assertion into an assertion about a `MagicMock`; `import_activities` asserts the live tracking
URI names the test server before it hands the object over.
**Bundles come from the unit suite.** `tests/helpers/bundle_factory.py` writes them and
`tests/fixtures/bundle/golden_model_v3.sientia` is the producer-emitted one. No third mirrored copy
of the producer's crypto is introduced here.
**Isolation** (the containers are session-scoped): per test the status table is rebuilt, the `models`
collection is dropped and reseeded, and the import prefix is emptied. MLflow state is *not* reset —
experiment and model names are made unique per test instead, because deleted registered models
leave tombstones `get_registered_model` still finds, which would make "second import of the same
model" unreadable.
"""
from __future__ import annotations
import contextlib
import hashlib
import json
import os
import signal
import socket
import subprocess
import sys
import time
import uuid
from collections.abc import Callable, Iterator
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import mlflow
import pytest
import pytest_asyncio
import requests
from pymongo import MongoClient
from sqlalchemy import Engine, create_engine, text
from temporalio.client import WorkflowHandle
from temporalio.worker import Worker
from testcontainers.mongodb import MongoDbContainer
from testcontainers.postgres import PostgresContainer
from laborious.activities.activities import Activities
from laborious.workflows.import_model import ImportModel
from tests.helpers.bundle_factory import (
DEFAULT_ENVELOPE_KEY,
DEFAULT_PASSWORD,
ZipSpec,
build_envelope,
build_metadata_document,
default_tree,
envelope_key_b64,
make_bundle,
)
# --- constants -----------------------------------------------------------------------------------
REPO_ROOT = Path(__file__).resolve().parent.parent
FIXTURES_DIR = Path(__file__).resolve().parent / 'fixtures'
STATUS_TABLE_DDL = FIXTURES_DIR / 'experiment_run.sql'
STATUS_TABLE_DDL_V12 = FIXTURES_DIR / 'experiment_run_v12.sql'
GOLDEN_BUNDLE = REPO_ROOT / 'tests' / 'fixtures' / 'bundle' / 'golden_model_v3.sientia'
GOLDEN_BUNDLE_PASSWORD = 'Import1Model!Fixture'
BFF_DATABASE = 'sientia-core-mlops-bff'
BFF_USER = 'bff'
BFF_PASSWORD = 'bff'
# Lowercase and hyphens only: `chk_experiment_run_bucket_name_pattern` on the real table.
IMPORT_BUCKET = 'sientia-import'
IMPORT_PREFIX = 'imported_models/'
MODELS_COLLECTION = 'models'
MONGO_DATABASE = 'sientia'
IMPORT_TASK_QUEUE = 'import-model-e2e-queue'
# Deliberately gapped, and deliberately strings: the baseline migration stores `id` as a string and
# `max(int(id)) + 1` must not be confused with a count or with a lexicographic maximum ("9" > "30").
SEEDED_MODEL_IDS = ('1', '2', '7')
MLFLOW_HEALTH_TIMEOUT_SECONDS = 90.0
MLFLOW_HEALTH_POLL_SECONDS = 0.5
IMPORT_ACTIVITY_NAMES = (
'claim_import_status',
'record_import_status',
'record_import_names',
'record_import_terminal_status',
'report_import_status_write_failure',
'download_import_bundle',
'open_import_bundle',
'create_import_experiment',
'upload_import_artifacts',
'register_import_model_version',
'write_import_model_document',
'cleanup_import_files',
'ensure_import_bundle_retention',
)
# --- the BFF Postgres container ------------------------------------------------------------------
@pytest.fixture(scope='session')
def import_bff_container() -> Iterator[PostgresContainer]:
"""The import log database, on a server of its own.
Separate from the shared `postgres_container` for two independent reasons, either sufficient:
the retry-recovery tests **stop and start this server**, which must not disturb the predictions
database every other e2e suite shares session scope with; and "the import touches no Postgres
database other than `sientia-core-mlops-bff`" is a stronger statement when the other database
lives on a different server.
`log_statement=all` is what makes the absence assertions possible: the server log is the only
vantage point that sees every client, including the psycopg pool `Storage` holds.
"""
container = PostgresContainer(
'postgres:15',
username=BFF_USER,
password=BFF_PASSWORD,
dbname=BFF_DATABASE,
).with_command('postgres -c log_statement=all')
container.start()
try:
yield container
finally:
container.stop()
@pytest.fixture(scope='session')
def import_bff_engine(import_bff_container: PostgresContainer) -> Iterator[Engine]:
"""A SQLAlchemy engine on the BFF container.
`pool_pre_ping` matters here rather than being habit: two tests stop the container underneath
this engine, and a stale pooled connection would fail the *fixture* instead of the assertion.
"""
engine = create_engine(import_bff_container.get_connection_url(), pool_pre_ping=True)
try:
yield engine
finally:
engine.dispose()
def _apply_ddl(engine: Engine, files: tuple[Path, ...]) -> None:
"""Drop `public.experiment_run` and rebuild it from `files`, in order."""
with engine.begin() as connection:
connection.execute(text('DROP TABLE IF EXISTS public.experiment_run CASCADE'))
for path in files:
# One transaction per file, because each file is one migration's worth of statements.
with engine.begin() as connection:
connection.execute(text(path.read_text(encoding='utf-8')))
@pytest.fixture
def import_status_table(import_bff_engine: Engine) -> Engine:
"""`public.experiment_run` as `V1`…`V12` leave it, rebuilt for this test.
Rebuilt rather than truncated: the pre-`V12` variant below has to be able to alternate with
this one inside a single session, and `TRUNCATE` cannot take a constraint away. The effect a
test sees is the one the spec asks for — the table holds only the row that test inserted.
"""
_apply_ddl(import_bff_engine, (STATUS_TABLE_DDL, STATUS_TABLE_DDL_V12))
return import_bff_engine
@pytest.fixture
def import_status_table_pre_v12(import_bff_engine: Engine) -> Engine:
"""The same table without `V12`: `run_type` present, the import statuses and pairing absent.
There is no second copy of the base schema — this applies the same file, and simply does not
apply the delta.
"""
_apply_ddl(import_bff_engine, (STATUS_TABLE_DDL,))
return import_bff_engine
InsertStatusRow = Callable[..., int]
@pytest.fixture
def import_insert_status_row(import_bff_engine: Engine) -> InsertStatusRow:
"""Insert one import log row the way the BFF will, and return its `id`.
Defaults mirror what the frontend sends: `PENDING`, `run_type='IMPORT'`, the object key in
`file_name`, and the uploader's digest inside `request_data` — which is where
`claim_import_status` looks for it.
"""
def insert(
*,
object_key: str,
expected_digest: str | None = None,
username: str = 'import.e2e@example.com',
experiment_name: str = 'import-e2e-placeholder',
status: str = 'PENDING',
run_type: str = 'IMPORT',
bucket_name: str = IMPORT_BUCKET,
request_data: dict[str, Any] | None = None,
orchestrator_response_data: dict[str, Any] | None = None,
) -> int:
payload = request_data if request_data is not None else {}
if expected_digest is not None and request_data is None:
payload = {'expected_digest': expected_digest}
now = datetime.now(UTC).replace(tzinfo=None)
statement = text(
'INSERT INTO public.experiment_run '
'(experiment_name, username, status, created_at, updated_at, bucket_name, file_name, '
' request_data, orchestrator_response_data, run_type) '
'VALUES (:experiment_name, :username, :status, :created_at, :updated_at, '
' :bucket_name, :file_name, CAST(:request_data AS JSONB), '
' CAST(:detail AS JSONB), :run_type) '
'RETURNING id'
)
with import_bff_engine.begin() as connection:
return int(
connection.execute(
statement,
{
'experiment_name': experiment_name,
'username': username,
'status': status,
'created_at': now,
'updated_at': now,
'bucket_name': bucket_name,
'file_name': object_key,
'request_data': json.dumps(payload),
'detail': (
json.dumps(orchestrator_response_data)
if orchestrator_response_data is not None
else None
),
'run_type': run_type,
},
).scalar_one()
)
return insert
ReadStatusRow = Callable[[int], dict[str, Any] | None]
@pytest.fixture
def import_read_status_row(import_bff_engine: Engine) -> ReadStatusRow:
"""Read one import log row as a plain mapping, or `None` when it is gone."""
def read(import_run_id: int) -> dict[str, Any] | None:
with import_bff_engine.begin() as connection:
row = (
connection.execute(
text('SELECT * FROM public.experiment_run WHERE id = :id'),
{'id': import_run_id},
)
.mappings()
.one_or_none()
)
return dict(row) if row is not None else None
return read
# --- MongoDB ------------------------------------------------------------------------------------
@pytest.fixture(scope='session')
def import_mongo_container() -> Iterator[MongoDbContainer]:
"""A real MongoDB for the `models` collection the import writes its document into."""
container = MongoDbContainer('mongo:7')
container.start()
try:
yield container
finally:
container.stop()
@pytest.fixture(scope='session')
def import_mongo_client(import_mongo_container: MongoDbContainer) -> Iterator[MongoClient]:
"""A client on the Mongo container, for seeding and for reading assertions back."""
client: MongoClient = MongoClient(import_mongo_container.get_connection_url())
try:
yield client
finally:
client.close()
@pytest.fixture
def import_models_collection(import_mongo_client: MongoClient) -> Any:
"""The `models` collection, dropped and reseeded with gapped ids and a unique index on `id`.
The index is the assertion's teeth: without it "the id continues the collection" and "a name
already present is left alone" would both pass against a collection that quietly took a
duplicate.
"""
collection = import_mongo_client[MONGO_DATABASE][MODELS_COLLECTION]
collection.drop()
collection.insert_many(
[
{
'id': model_id,
'name': f'seeded_model_{model_id}',
'active': True,
'model_config': {
'transform_flavor': 'sklearn',
'predict_flavor': 'joblib',
'retention_minutes': 60,
'target': 'SE',
},
}
for model_id in SEEDED_MODEL_IDS
]
)
collection.create_index('id', unique=True)
return collection
# --- MinIO --------------------------------------------------------------------------------------
@pytest.fixture
def import_bucket(minio_container) -> Iterator[Any]:
"""The import bucket in the shared MinIO container, with its prefix emptied for this test.
The bucket is kept rather than recreated — a bucket carries the retention lifecycle rule the
import sets, and dropping it between tests would make that rule untestable.
"""
client = minio_container.get_client()
if not client.bucket_exists(IMPORT_BUCKET):
client.make_bucket(IMPORT_BUCKET)
_empty_prefix(client, IMPORT_BUCKET, IMPORT_PREFIX)
yield client
_empty_prefix(client, IMPORT_BUCKET, IMPORT_PREFIX)
def _empty_prefix(client: Any, bucket: str, prefix: str) -> None:
"""Remove every object under `prefix`, so a leftover cannot satisfy the next test."""
for obj in client.list_objects(bucket, prefix=prefix, recursive=True):
client.remove_object(bucket, obj.object_name)
UploadBundle = Callable[..., tuple[str, str]]
@pytest.fixture
def import_upload_bundle(import_bucket: Any) -> UploadBundle:
"""Upload bundle bytes under the import prefix; return `(object_key, hex_digest)`.
The digest is computed over the bytes handed in, which is what the frontend promises and what
gate 1 holds it to. A test that wants the digest to *disagree* passes its own.
"""
import io
def upload(payload: bytes, name: str | None = None) -> tuple[str, str]:
object_name = name or f'e2e_{uuid.uuid4().hex}.sientia'
object_key = f'{IMPORT_PREFIX}{object_name}'
import_bucket.put_object(
IMPORT_BUCKET,
object_key,
io.BytesIO(payload),
length=len(payload),
content_type='application/octet-stream',
)
return object_key, hashlib.sha256(payload).hexdigest()
return upload
# --- building bundles ----------------------------------------------------------------------------
@pytest.fixture
def import_envelope() -> Callable[[str], str]:
"""Wrap a bundle password the way the frontend will, under the key the worker is configured with.
The key is `bundle_factory.DEFAULT_ENVELOPE_KEY`, which `import_activities` puts in
`IMPORT_PASSWORD_KEY` — so an envelope built here is one the worker can actually open, and an
envelope built under any other key is a decryption-step failure, which is a test of its own.
"""
def build(password: str = DEFAULT_PASSWORD) -> str:
return build_envelope(password, DEFAULT_ENVELOPE_KEY)
return build
@pytest.fixture
def import_bundle_factory(import_names: dict[str, str]) -> Callable[..., bytes]:
"""Build a `.sientia` bundle carrying *this test's* unique model and experiment names.
Every knob the gate tests need is passed straight through to the shared factory — there is no
second implementation of the producer's header, KDF or framing here, which is the whole point of
reusing `tests/helpers/bundle_factory.py`.
The names default to `import_names` so two tests provisioning models in the same session never
observe each other's registry state (MLflow is deliberately not reset — design E13). A test that
*wants* the collision, like the second-import case, passes the same names twice.
"""
def build(
*,
password: str = DEFAULT_PASSWORD,
metadata: dict[str, Any] | None = None,
parameters: dict[str, Any] | None = None,
drop_metadata_field: str | None = None,
drop_top_level_key: str | None = None,
drop_parameter: str | None = None,
files: dict[str, bytes] | None = None,
include_metadata: bool = True,
symlinks: dict[str, str] | None = None,
absolute_entry: str | None = None,
traversal_entry: str | None = None,
bomb_bytes: int = 0,
extra_files: dict[str, bytes] | None = None,
**header_kwargs: Any,
) -> bytes:
document = build_metadata_document(
metadata={
'model_name': import_names['model_name'],
'experiment_name': import_names['experiment_name'],
**(metadata or {}),
},
parameters={'model_name': import_names['model_name'], **(parameters or {})},
drop_metadata_field=drop_metadata_field,
drop_top_level_key=drop_top_level_key,
drop_parameter=drop_parameter,
)
spec = ZipSpec(
files=default_tree() if files is None else files,
document=document,
include_metadata=include_metadata,
symlinks=symlinks or {},
absolute_entry=absolute_entry,
traversal_entry=traversal_entry,
bomb_bytes=bomb_bytes,
extra_files=extra_files or {},
)
return make_bundle(spec, password, **header_kwargs)
return build
# --- the MLflow tracking server -----------------------------------------------------------------
def _free_port() -> int:
"""A loopback port nothing is listening on, taken from an ephemeral bind."""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.bind(('127.0.0.1', 0))
return int(sock.getsockname()[1])
@pytest.fixture(scope='session')
def import_mlflow_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[dict[str, Any]]:
"""`mlflow server` from *this* venv, on a free loopback port.
A subprocess rather than a container because the venv pin **is** the production pin (2.10.1,
the same version the producer pins), so this tests the exact client/server pair that will run
in production — no image tag to drift out of step with `requirements.txt`.
sqlite for the backend store because the model registry is not available on a file-backed
tracking store, and `create_model_version` is half of what the provisioning activities do.
`--serve-artifacts` sends `log_artifacts` over HTTP through the server, as it goes in
production, while leaving the uploaded tree readable straight off the local filesystem.
Yields the host, the port and the artifacts directory. A server that never answers `/health`
fails *here*, with its own stderr attached, rather than making the first test fail obscurely.
"""
root = tmp_path_factory.mktemp('import_mlflow')
artifacts = root / 'mlartifacts'
artifacts.mkdir()
log_path = root / 'mlflow-server.log'
port = _free_port()
host = '127.0.0.1'
tracking_uri = f'http://{host}:{port}'
server: dict[str, Any] = {
'host': f'http://{host}',
'port': str(port),
'tracking_uri': tracking_uri,
'artifacts_dir': artifacts,
'root': root,
'log_path': log_path,
'process': _launch_mlflow(root, host, port, artifacts, log_path),
}
try:
_wait_for_mlflow(server['process'], tracking_uri, log_path)
yield server
finally:
_terminate_group(server['process'])
def _launch_mlflow(
root: Path, host: str, port: int, artifacts: Path, log_path: Path
) -> subprocess.Popen:
"""Start the server process. Separate from the fixture so one test can restart it.
The sqlite file and the artifacts directory are reused across a restart on purpose: a test that
stops the server to fail one activity must not also erase what earlier tests provisioned.
"""
with log_path.open('ab') as log_handle:
return subprocess.Popen( # noqa: S603 - fixed argv, no shell, all values built here
[
sys.executable,
'-m',
'mlflow',
'server',
'--host',
host,
'--port',
str(port),
'--backend-store-uri',
f'sqlite:///{root / "mlflow.db"}',
'--artifacts-destination',
str(artifacts),
'--serve-artifacts',
],
stdout=log_handle,
stderr=subprocess.STDOUT,
cwd=str(root),
# Its own process group: `mlflow server` is gunicorn, so it has children, and killing
# only the parent would leave workers holding the port for the rest of the session.
start_new_session=True,
)
@pytest.fixture
def import_mlflow_stopped(import_mlflow_server: dict[str, Any]) -> Any:
"""A context manager that stops the tracking server and **always** brings it back.
This is design E7's second preference in its purest form: no patch, no mock — the dependency is
genuinely gone, which is the thing that actually happens in production when a pod is replaced.
The restart is in a `finally` because the server is session-scoped: a test that left it down
would fail every test after it, and the failure would point at the wrong place.
"""
@contextlib.contextmanager
def stopped() -> Iterator[None]:
_terminate_group(import_mlflow_server['process'])
try:
yield
finally:
import_mlflow_server['process'] = _launch_mlflow(
import_mlflow_server['root'],
'127.0.0.1',
int(import_mlflow_server['port']),
import_mlflow_server['artifacts_dir'],
import_mlflow_server['log_path'],
)
_wait_for_mlflow(
import_mlflow_server['process'],
import_mlflow_server['tracking_uri'],
import_mlflow_server['log_path'],
)
return stopped
def _wait_for_mlflow(process: subprocess.Popen, tracking_uri: str, log_path: Path) -> None:
"""Poll `/health` until it answers, or fail with what the server actually said."""
deadline = time.monotonic() + MLFLOW_HEALTH_TIMEOUT_SECONDS
while time.monotonic() < deadline:
if process.poll() is not None:
_terminate_group(process)
raise RuntimeError(
f'mlflow server exited with {process.returncode} before answering /health:\n'
f'{log_path.read_text(encoding="utf-8", errors="replace")}'
)
try:
if requests.get(f'{tracking_uri}/health', timeout=2).status_code == 200:
return
except requests.RequestException:
pass
time.sleep(MLFLOW_HEALTH_POLL_SECONDS)
_terminate_group(process)
raise RuntimeError(
f'mlflow server did not answer /health within {MLFLOW_HEALTH_TIMEOUT_SECONDS}s:\n'
f'{log_path.read_text(encoding="utf-8", errors="replace")}'
)
def _terminate_group(process: subprocess.Popen) -> None:
"""Stop the server and every gunicorn worker it forked."""
if process.poll() is not None:
return
with contextlib.suppress(ProcessLookupError, PermissionError):
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
try:
process.wait(timeout=15)
except subprocess.TimeoutExpired:
with contextlib.suppress(ProcessLookupError, PermissionError):
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
process.wait(timeout=15)
# --- per-test naming ----------------------------------------------------------------------------
@pytest.fixture
def import_names(request: pytest.FixtureRequest) -> dict[str, str]:
"""Experiment and model names unique to this test.
MLflow state is deliberately not reset between tests (design E13): deleting a registered model
leaves a tombstone `get_registered_model` still finds, which would make "second import of the
same model" read as a leak rather than as the thing it is testing. Unique names per test buy
the same isolation without the tombstones.
"""
token = uuid.uuid4().hex[:8]
stem = request.node.name.replace('[', '_').replace(']', '')[:24]
return {
'token': token,
'model_name': f'm_{stem}_{token}'[:40],
'experiment_name': f'e_{stem}_{token}'[:40],
}
# --- the import Activities ----------------------------------------------------------------------
@pytest.fixture
def import_limits_override() -> dict[str, str]:
"""Extra import environment variables for this test, applied last so they win.
Empty by default. A test that needs a ceiling it can actually cross — gate 1's object size is
1 GiB, and uploading a gigabyte to prove a limit is not a test anybody runs twice — overrides
this fixture inside its own class, so the narrowed ceiling is scoped to the cases that need it
and every other test keeps the production defaults.
"""
return {}
@pytest_asyncio.fixture
async def import_activities(
monkeypatch: pytest.MonkeyPatch,
import_limits_override: dict[str, str],
tmp_path: Path,
postgres_container,
import_bff_container: PostgresContainer,
import_mongo_container: MongoDbContainer,
import_mlflow_server: dict[str, Any],
import_bucket: Any,
minio_container,
mock_logger,
notification_handler,
) -> Any:
"""An `Activities` whose import path talks to nothing fake.
Note what is *not* in the fixture's parameter list: `patch_mlflow`, `patch_minio_repository`
and `patch_create_engine`. Composing any of them would replace exactly the thing under test,
and because `mlflow.set_tracking_uri` is process-global the substitution would be silent — so
the live tracking URI is asserted below rather than assumed.
The import configuration is built by the production `build_import_config` /
`build_import_status_config` from environment variables set here, so the variable *names* are
covered too and not just the dictionaries they produce.
"""
from laborious.utils.connectors_config import build_import_config, build_import_status_config
minio_port = minio_container.get_exposed_port(9000)
monkeypatch.setenv('IMPORT_PASSWORD_KEY', envelope_key_b64(DEFAULT_ENVELOPE_KEY))
monkeypatch.setenv('IMPORT_BUNDLE_BUCKET', IMPORT_BUCKET)
monkeypatch.setenv('IMPORT_BUNDLE_PREFIX', IMPORT_PREFIX)
monkeypatch.setenv('IMPORT_MODELS_COLLECTION', MODELS_COLLECTION)
monkeypatch.setenv('IMPORT_WORK_DIR', str(tmp_path / 'import-work'))
monkeypatch.setenv('IMPORT_STATUS_DB_HOST', import_bff_container.get_container_host_ip())
monkeypatch.setenv('IMPORT_STATUS_DB_PORT', str(import_bff_container.get_exposed_port(5432)))
monkeypatch.setenv('IMPORT_STATUS_DB_NAME', BFF_DATABASE)
monkeypatch.setenv('IMPORT_STATUS_DB_USER', BFF_USER)
monkeypatch.setenv('IMPORT_STATUS_DB_PASSWORD', BFF_PASSWORD)
for name, value in import_limits_override.items():
monkeypatch.setenv(name, value)
activities = Activities(
# The predictions database, which the import must never touch. It is pointed at a real
# server on purpose: "no statement reached it" is only worth asserting when there is a log
# that would have shown one.
postgres_config={
'host': 'localhost',
'port': postgres_container.get_exposed_port(5432),
'user': 'test',
'password': 'test',
'dbname': 'test',
'min_connections': 1,
'max_connections': 5,
},
mlflow_config={
'host': import_mlflow_server['host'],
'port': import_mlflow_server['port'],
'username': 'e2e',
'password': 'e2e',
},
minio_config={
'endpoint_url': f'localhost:{minio_port}',
'access_key': 'minioadmin',
'secret_key': 'minioadmin',
'default_bucket': IMPORT_BUCKET,
'retention_hours': 24,
'secure': False,
},
opc_config={},
pi_web_api_config={
'base_url': 'http://localhost:8080',
'auth_type': 'bearer',
'auth_token': 'e2e',
},
logger=mock_logger,
notification_handler=notification_handler,
mongo_config={
'connection_string': import_mongo_container.get_connection_url(),
'database_name': MONGO_DATABASE,
},
import_config=build_import_config(),
import_status_config=build_import_status_config(),
)
# If a mocked-MLflow fixture ever leaks into an import test, this is where it stops.
assert mlflow.get_tracking_uri() == import_mlflow_server['tracking_uri'], (
f'MLflow tracking URI is {mlflow.get_tracking_uri()!r}, not the test server '
f'{import_mlflow_server["tracking_uri"]!r} — a mocked MLflow fixture has leaked in'
)
try:
yield activities
finally:
await activities.shutdown()
@pytest_asyncio.fixture
async def import_worker(temporal_test_env, import_activities: Any) -> Any:
"""An `ImportModel` worker with the thirteen import activities, on the import queue."""
async with Worker(
temporal_test_env.client,
task_queue=IMPORT_TASK_QUEUE,
workflows=[ImportModel],
activities=[getattr(import_activities, name) for name in IMPORT_ACTIVITY_NAMES],
) as worker:
yield worker
# --- running an import --------------------------------------------------------------------------
@pytest.fixture
def import_build_input() -> Callable[..., dict[str, Any]]:
"""The workflow input, as the frontend assembles it."""
def build(
*,
import_run_id: int,
object_key: str,
expected_digest: str | None,
password_envelope: str,
bucket: str | None = IMPORT_BUCKET,
**extra: Any,
) -> dict[str, Any]:
payload: dict[str, Any] = {
'import_run_id': import_run_id,
'object_key': object_key,
'password_envelope': password_envelope,
}
if expected_digest is not None:
payload['expected_digest'] = expected_digest
if bucket is not None:
payload['bucket'] = bucket
payload.update(extra)
return payload
return build
@pytest.fixture
def import_execute(temporal_test_env, import_worker: Any) -> Callable[..., Any]:
"""Run one import to completion and hand back the handle, whatever the outcome.
The handle rather than the result, because half this suite asserts on *how* a run failed —
the error type Temporal recorded, the event history — and `execute_workflow` would have thrown
that away.
"""
async def execute(payload: dict[str, Any], workflow_id: str | None = None) -> WorkflowHandle:
handle = await temporal_test_env.client.start_workflow(
ImportModel.run,
payload,
id=workflow_id or f'import-e2e-{uuid.uuid4().hex}',
task_queue=IMPORT_TASK_QUEUE,
)
with contextlib.suppress(Exception):
# The exception is the assertion's subject in the negative cases; the caller reads it
# back off the handle so a failure here is never mistaken for a harness error.
await handle.result()
return handle
return execute
@pytest.fixture
def import_history_events() -> Callable[[WorkflowHandle], Any]:
"""Every event of a run's history, flattened to the text the secrecy assertions scan.
Returns `(events, blob)`: the event list for structural assertions, and one string holding
every payload in the history for "this value appears nowhere" assertions.
"""
async def fetch(handle: WorkflowHandle) -> tuple[list[Any], str]:
events = [event async for event in handle.fetch_history_events()]
return events, '\n'.join(str(event) for event in events)
return fetch
# --- reading a failure ---------------------------------------------------------------------------
#
# Temporal wraps what actually went wrong: `WorkflowFailureError` -> `ActivityError` ->
# `ApplicationError`, and only the innermost one carries the sentence and the declared error type.
# `str(exception)` on the outermost is "Workflow execution failed", which is true and useless, so
# every negative assertion in this suite goes through these two functions instead.
def failure_chain(error: BaseException) -> list[BaseException]:
"""The exception and everything it was caused by, outermost first."""
chain: list[BaseException] = []
current: BaseException | None = error
while current is not None and not any(current is seen for seen in chain):
chain.append(current)
current = current.__cause__
return chain
def failure_text(error: BaseException) -> str:
"""Every link of the chain as `Type: message`, joined — what a substring assertion reads."""
return '\n'.join(f'{type(part).__name__}: {part}' for part in failure_chain(error))
def failure_types(error: BaseException) -> set[str]:
"""Every Python class name in the chain, plus every `ApplicationError.type` it declares.
The declared `type` is the one that survives serialisation into the event history, so it is
what "the run failed with the not-claimable error type" actually means.
"""
names: set[str] = set()
for part in failure_chain(error):
names.add(type(part).__name__)
declared = getattr(part, 'type', None)
if isinstance(declared, str):
names.add(declared)
return names
# --- the absence assertion ----------------------------------------------------------------------
@pytest.fixture
def import_assert_nothing_provisioned(
import_mlflow_server: dict[str, Any],
import_models_collection: Any,
import_bucket: Any,
tmp_path: Path,
) -> Callable[..., None]:
"""Assert a rejected import left nothing behind, in one call over all four places.
One helper rather than four assertions per test, because "nothing was provisioned" is a single
claim and a test that checks three of its four halves reads as if it checked all four.
"""
def assert_nothing(
*,
experiment_name: str,
model_name: str,
object_key: str | None = None,
seeded_ids: tuple[str, ...] = SEEDED_MODEL_IDS,
) -> None:
client = mlflow.tracking.MlflowClient(tracking_uri=import_mlflow_server['tracking_uri'])
experiment = client.get_experiment_by_name(experiment_name)
assert experiment is None, f'experiment {experiment_name!r} was created'
versions = client.search_model_versions(f"name='{model_name}'")
assert not list(versions), f'registry holds versions of {model_name!r}'
documents = list(import_models_collection.find({}, {'_id': False}))
assert {str(document['id']) for document in documents} == set(seeded_ids), (
f'the models collection changed: {documents}'
)
assert not [d for d in documents if d.get('name') == model_name], (
f'a model document for {model_name!r} was written'
)
# The uploaded object itself stays — the frontend put it there and retention removes it.
# What must not be there is anything the *import* wrote under the prefix.
stored = {
obj.object_name
for obj in import_bucket.list_objects(
IMPORT_BUCKET, prefix=IMPORT_PREFIX, recursive=True
)
}
assert stored <= ({object_key} if object_key else set()), (
f'the import wrote objects under {IMPORT_PREFIX}: {stored}'
)
work_dir = tmp_path / 'import-work'
leftovers = list(work_dir.rglob('*')) if work_dir.exists() else []
assert not leftovers, f'the worker filesystem still holds {leftovers}'
return assert_nothing
# --- reading the golden fixture -----------------------------------------------------------------
@pytest.fixture(scope='session')
def import_golden_bundle() -> dict[str, Any]:
"""The producer-emitted bundle: its bytes, its digest and the password it was written with.
This is the only place in the suite that pays the producer's real Argon2id cost (256 MiB,
3 passes). Everything else runs the factory's cheap cost, which is the right split — the
reader honours whatever the header declares, up to its ceiling.
"""
payload = GOLDEN_BUNDLE.read_bytes()
return {
'bytes': payload,
'digest': hashlib.sha256(payload).hexdigest(),
'password': GOLDEN_BUNDLE_PASSWORD,
'path': GOLDEN_BUNDLE,
}

196
e2e/test_import_harness_smoke.py Normal file
View File

@@ -0,0 +1,196 @@
"""Does the harness itself come up? Diagnosable separately from the suite that runs on it.
Four containers, an `mlflow server` subprocess and a Temporal worker is a lot of moving parts, and
when one of them fails to start the first *real* test fails somewhere deep inside an activity with
a message about something else. These tests exist so that a broken harness says so in its own
words: each one checks one piece is alive and does nothing with the import path.
The last test is the exception — it runs a real workflow, but with an input designed to be rejected
at the very first step, so what it proves is only that the wiring reaches the first activity.
"""
from __future__ import annotations
import json
from typing import Any
import mlflow
import pytest
from sqlalchemy import text
from temporalio.client import WorkflowFailureError
from e2e.import_harness import (
BFF_DATABASE,
IMPORT_ACTIVITY_NAMES,
IMPORT_BUCKET,
IMPORT_PREFIX,
IMPORT_TASK_QUEUE,
SEEDED_MODEL_IDS,
failure_text,
failure_types,
)
pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
class TestContainers:
"""Each dependency is real, reachable, and the one the harness says it is."""
async def test_bff_container_holds_the_status_table(self, import_status_table: Any) -> None:
with import_status_table.begin() as connection:
database = connection.execute(text('SELECT current_database()')).scalar_one()
columns = {
row[0]
for row in connection.execute(
text(
'SELECT column_name FROM information_schema.columns '
"WHERE table_schema = 'public' AND table_name = 'experiment_run'"
)
)
}
assert database == BFF_DATABASE
assert {'id', 'status', 'run_type', 'request_data', 'orchestrator_response_data'} <= columns
async def test_bff_container_logs_every_statement(self, import_bff_container: Any) -> None:
"""`log_statement=all` — without it the absence assertions have no vantage point."""
result = import_bff_container.exec(
['psql', '-U', 'bff', '-d', BFF_DATABASE, '-tAc', 'SHOW log_statement']
)
assert b'all' in result.output
async def test_the_v12_delta_is_what_separates_the_two_tables(
self, import_status_table_pre_v12: Any
) -> None:
"""The pre-`V12` table has `run_type` and refuses the import terminal statuses."""
with import_status_table_pre_v12.begin() as connection:
constraints = {
row[0]
for row in connection.execute(
text(
'SELECT conname FROM pg_constraint '
"WHERE conrelid = 'public.experiment_run'::regclass"
)
)
}
columns = {
row[0]
for row in connection.execute(
text(
'SELECT column_name FROM information_schema.columns '
"WHERE table_schema = 'public' AND table_name = 'experiment_run'"
)
)
}
assert 'run_type' in columns
assert 'chk_experiment_run_terminal_status_run_type' not in constraints
async def test_the_seeded_collection_has_gapped_ids_and_a_unique_index(
self, import_models_collection: Any
) -> None:
ids = {
str(document['id']) for document in import_models_collection.find({}, {'_id': False})
}
assert ids == set(SEEDED_MODEL_IDS)
unique_keys = {
tuple(index['key'].keys())
for index in import_models_collection.list_indexes()
if index.get('unique')
}
assert ('id',) in unique_keys
async def test_the_import_bucket_exists_and_its_prefix_is_empty(
self, import_bucket: Any
) -> None:
assert import_bucket.bucket_exists(IMPORT_BUCKET)
assert not list(
import_bucket.list_objects(IMPORT_BUCKET, prefix=IMPORT_PREFIX, recursive=True)
)
async def test_uploading_a_bundle_returns_a_key_and_its_digest(
self, import_upload_bundle: Any, import_bucket: Any
) -> None:
object_key, digest = import_upload_bundle(b'not a real bundle, just bytes')
assert object_key.startswith(IMPORT_PREFIX)
assert len(digest) == 64
assert import_bucket.stat_object(IMPORT_BUCKET, object_key).size > 0
async def test_the_mlflow_server_answers_and_serves_a_registry(
self, import_mlflow_server: dict[str, Any]
) -> None:
client = mlflow.tracking.MlflowClient(tracking_uri=import_mlflow_server['tracking_uri'])
experiment_id = client.create_experiment('harness_smoke_experiment')
run = client.create_run(experiment_id, run_name='harness-smoke-run')
client.create_registered_model('harness_smoke_model')
version = client.create_model_version(
'harness_smoke_model', f'runs:/{run.info.run_id}/prediction_model', run.info.run_id
)
assert version.version == '1'
assert version.current_stage in (None, 'None')
class TestActivitiesAndWorker:
"""The object under test is built from the real dependencies, and the worker registers it."""
async def test_activities_point_at_the_real_dependencies(
self, import_activities: Any, import_mlflow_server: dict[str, Any]
) -> None:
assert mlflow.get_tracking_uri() == import_mlflow_server['tracking_uri']
assert import_activities.import_config['bucket'] == IMPORT_BUCKET
assert import_activities.import_config['prefix'] == IMPORT_PREFIX
assert import_activities.status_config['dbname'] == BFF_DATABASE
assert import_activities.mongo_repository is not None
# Not a mock: a MagicMock would satisfy any attribute access, so the type is the assertion.
assert type(import_activities.import_minio_repository).__name__ == 'MinioRepository'
async def test_every_import_activity_is_registered(self, import_activities: Any) -> None:
for name in IMPORT_ACTIVITY_NAMES:
assert callable(getattr(import_activities, name)), f'{name} is missing'
async def test_the_worker_polls_the_import_queue(self, import_worker: Any) -> None:
assert import_worker.task_queue == IMPORT_TASK_QUEUE
async def test_a_workflow_input_reaches_the_first_activity(
self,
import_status_table: Any,
import_execute: Any,
import_build_input: Any,
import_read_status_row: Any,
) -> None:
"""The claim runs, and the row it could not claim proves the wiring is live.
An id no row carries is the cheapest input that gets past workflow validation and into
`claim_import_status`, so a run that ends failed *there* means the client, the worker, the
workflow, the activity and the BFF connection are all in place. Nothing else is exercised.
"""
handle = await import_execute(
import_build_input(
import_run_id=987654321,
object_key=f'{IMPORT_PREFIX}nothing.sientia',
expected_digest='0' * 64,
password_envelope='not-a-real-envelope',
)
)
with pytest.raises(WorkflowFailureError) as failure:
await handle.result()
text_of_failure = failure_text(failure.value)
assert 'claim' in text_of_failure.lower()
assert '987654321' in text_of_failure
assert 'ImportLogRowNotClaimableError' in failure_types(failure.value)
assert import_read_status_row(987654321) is None
async def test_the_insert_helper_writes_a_claimable_row(
self,
import_status_table: Any,
import_insert_status_row: Any,
import_read_status_row: Any,
) -> None:
object_key = f'{IMPORT_PREFIX}claimable.sientia'
row_id = import_insert_status_row(object_key=object_key, expected_digest='a' * 64)
row = import_read_status_row(row_id)
assert row is not None
assert row['status'] == 'PENDING'
assert row['run_type'] == 'IMPORT'
assert row['file_name'] == object_key
request_data = row['request_data']
payload = json.loads(request_data) if isinstance(request_data, str) else request_data
assert payload['expected_digest'] == 'a' * 64

214
e2e/test_import_model_digest.py Normal file
View File

@@ -0,0 +1,214 @@
"""The digest is a promise about the bytes in the bucket, and it is kept against the bucket.
Three ways the promise can be broken, and where each is caught:
- the uploader computed the digest over *other* bytes — caught at `download`, after the object is
fetched and before its header is parsed, so a wrong file never costs an Argon2id pass;
- the object was replaced between the upload and the run — the same failure, which is the point:
the check is against what is stored now, not against what was stored when the row was written;
- no digest at all — caught at `received`, before anything is downloaded, because a request with no
digest is not a request this workflow can honour.
The first two matter because MinIO is real here. A mocked object store would return whatever the
test told it to, and "the digest is enforced against the bytes that were uploaded" would be a
statement about the mock.
"""
from __future__ import annotations
import hashlib
import io
import json
from typing import Any
import pytest
from temporalio.client import WorkflowFailureError
from e2e.import_harness import IMPORT_BUCKET, failure_text
from laborious.activities.model_import_errors import IMPORT_FAILURE_CATALOG
from laborious.utils.bundle.steps import ImportStep
pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
VALIDATION_ERROR_STATUS = 'ORCHESTRATOR_VALIDATION_ERROR'
FACTORY_PASSWORD = 'Factory1Model!Test'
def _as_dict(value: Any) -> dict[str, Any]:
return json.loads(value) if isinstance(value, str) else (value or {})
def _assert_step(row: dict[str, Any], step: ImportStep, workflow_id: str) -> None:
code, reason = IMPORT_FAILURE_CATALOG[step]
detail = _as_dict(row['orchestrator_response_data'])['import']
assert row['status'] == VALIDATION_ERROR_STATUS, row['status']
assert detail['step'] == step.value, detail
assert detail['code'] == code.value, detail
assert detail['workflow_id'] == workflow_id
assert row['error_message'] == reason
class TestWrongDigest:
"""A digest computed over bytes that are not the ones in the bucket."""
async def test_a_digest_from_another_file_fails_at_download(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
) -> None:
"""The bundle is perfectly valid; only the digest is another file's.
The row carries the same wrong digest as the input, so the request and the record agree and
the `received` guard passes — otherwise this would be a `received` failure about a
contradictory request instead of the download-step digest check it is meant to exercise.
"""
payload = import_bundle_factory()
other_digest = hashlib.sha256(
import_bundle_factory(metadata={'run_id': 'other'})
).hexdigest()
assert other_digest != hashlib.sha256(payload).hexdigest()
object_key, _ = import_upload_bundle(payload)
import_run_id = import_insert_status_row(
object_key=object_key, expected_digest=other_digest
)
handle = await import_execute(
import_build_input(
import_run_id=import_run_id,
object_key=object_key,
expected_digest=other_digest,
password_envelope=import_envelope(FACTORY_PASSWORD),
)
)
with pytest.raises(WorkflowFailureError) as failure:
await handle.result()
row = import_read_status_row(import_run_id)
assert row is not None
_assert_step(row, ImportStep.DOWNLOAD, handle.id)
# No header was parsed: the failure names the digest, not the format. If the digest check
# ran *after* the header the message here would be about magic bytes or a version.
text_of_failure = failure_text(failure.value).lower()
assert 'digest' in text_of_failure, text_of_failure
assert 'marker' not in text_of_failure and 'version' not in text_of_failure
# The names are still unwritten, which is the other half of "no header was parsed": the
# bundle was never opened, so nothing knew what to call it.
assert row['run_name'] is None
assert row['experiment_name'] == 'import-e2e-placeholder'
import_assert_nothing_provisioned(
experiment_name=import_names['experiment_name'],
model_name=import_names['model_name'],
object_key=object_key,
)
class TestOverwrittenObject:
"""The bytes changed after the row was written. Only a real object store can show this."""
async def test_replacing_the_object_after_upload_fails_at_download(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_bucket: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
) -> None:
original = import_bundle_factory()
object_key, digest = import_upload_bundle(original)
import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
# Same key, different bytes — also a valid bundle, so nothing but the digest can tell.
replacement = import_bundle_factory(metadata={'run_id': 'ff00ff00ff00ff00ff00ff00ff00ff00'})
assert hashlib.sha256(replacement).hexdigest() != digest
import_bucket.put_object(
IMPORT_BUCKET,
object_key,
io.BytesIO(replacement),
length=len(replacement),
content_type='application/octet-stream',
)
handle = await import_execute(
import_build_input(
import_run_id=import_run_id,
object_key=object_key,
expected_digest=digest,
password_envelope=import_envelope(FACTORY_PASSWORD),
)
)
with pytest.raises(WorkflowFailureError):
await handle.result()
row = import_read_status_row(import_run_id)
assert row is not None
_assert_step(row, ImportStep.DOWNLOAD, handle.id)
import_assert_nothing_provisioned(
experiment_name=import_names['experiment_name'],
model_name=import_names['model_name'],
object_key=object_key,
)
class TestMissingDigest:
"""No digest is refused at the door, before a single byte is fetched."""
async def test_an_absent_digest_fails_at_received_with_nothing_downloaded(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
tmp_path: Any,
) -> None:
object_key, digest = import_upload_bundle(import_bundle_factory())
import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
handle = await import_execute(
import_build_input(
import_run_id=import_run_id,
object_key=object_key,
expected_digest=None,
password_envelope=import_envelope(FACTORY_PASSWORD),
)
)
with pytest.raises(WorkflowFailureError) as failure:
await handle.result()
row = import_read_status_row(import_run_id)
assert row is not None
_assert_step(row, ImportStep.RECEIVED, handle.id)
assert 'expected_digest' in failure_text(failure.value)
# Nothing downloaded: the work directory was never even created.
work_dir = tmp_path / 'import-work'
assert not work_dir.exists() or not list(work_dir.rglob('*'))
import_assert_nothing_provisioned(
experiment_name=import_names['experiment_name'],
model_name=import_names['model_name'],
object_key=object_key,
)

675
e2e/test_import_model_gates.py Normal file
View File

@@ -0,0 +1,675 @@
"""One rejection per gate, against a real object, a real archive and a real filesystem.
The gates are unit-tested already. What only a real run can show is that a rejection is *recorded*
correctly and that it provisions nothing: the row ends `ORCHESTRATOR_VALIDATION_ERROR` naming that
gate's step, its catalog code and its catalog sentence, and no experiment, run, artifact, registry
version, model document or extracted file survives.
Every case here is hostile **input** — design E7's first preference. Nothing is patched, no
dependency is broken; the bundles are built by the shared factory and the gates do the rest. The
one exception to "input only" is the object-size ceiling, where the ceiling itself is narrowed
rather than a gigabyte being uploaded; that is a configuration change, not a patched code path, and
it is scoped to the one class that needs it.
`_assert_rejected` is where the shared claim of task 5.8 lives — terminal status, recorded step,
recorded code, catalog sentence, and `assert_nothing_provisioned` — so no case can quietly assert
four fifths of it.
"""
from __future__ import annotations
import json
import tempfile
from typing import Any
import pytest
from temporalio.client import WorkflowFailureError
from e2e.import_harness import (
IMPORT_BUCKET,
IMPORT_PREFIX,
failure_text,
)
from laborious.activities.model_import_errors import IMPORT_FAILURE_CATALOG
from laborious.utils.bundle.steps import ImportStep
pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
VALIDATION_ERROR_STATUS = 'ORCHESTRATOR_VALIDATION_ERROR'
FACTORY_PASSWORD = 'Factory1Model!Test'
def _as_dict(value: Any) -> dict[str, Any]:
return json.loads(value) if isinstance(value, str) else (value or {})
async def _run_rejected_import(
*,
payload: bytes,
password: str,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
envelope_password: str | None = None,
) -> tuple[Any, int, str]:
"""Upload a hostile bundle, insert the row, run the import and expect it to fail."""
object_key, digest = import_upload_bundle(payload)
import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
handle = await import_execute(
import_build_input(
import_run_id=import_run_id,
object_key=object_key,
expected_digest=digest,
password_envelope=import_envelope(
envelope_password if envelope_password is not None else password
),
)
)
return handle, import_run_id, object_key
async def _assert_rejected(
*,
handle: Any,
import_run_id: int,
object_key: str,
step: ImportStep,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
import_names: dict[str, str],
experiment_name: str | None = None,
model_name: str | None = None,
) -> dict[str, Any]:
"""The whole of task 5.8, in one call. Returns the row so a case can add to it."""
with pytest.raises(WorkflowFailureError):
await handle.result()
row = import_read_status_row(import_run_id)
assert row is not None, 'the rejection lost the row it was recorded on'
assert row['status'] == VALIDATION_ERROR_STATUS, (
f'terminal status is {row["status"]!r}, expected {VALIDATION_ERROR_STATUS!r}'
)
expected_code, expected_reason = IMPORT_FAILURE_CATALOG[step]
detail = _as_dict(row['orchestrator_response_data'])['import']
assert detail['step'] == step.value, (
f'recorded step is {detail["step"]!r}, expected {step.value!r}'
)
assert detail['code'] == expected_code.value, (
f'recorded code is {detail["code"]!r}, expected {expected_code.value!r}'
)
assert detail['workflow_id'] == handle.id
assert row['error_message'] == expected_reason, (
'error_message is not the catalog sentence for this step'
)
import_assert_nothing_provisioned(
experiment_name=experiment_name or import_names['experiment_name'],
model_name=model_name or import_names['model_name'],
object_key=object_key,
)
return row
class TestGate1ObjectLimits:
"""Gate 1 rejects before the object's body is fetched at all."""
@pytest.fixture
def import_limits_override(self) -> dict[str, str]:
"""A 1 KiB ceiling, so a factory bundle is over it.
The production ceiling is 1 GiB. Narrowing it is the only way to cross gate 1 without
uploading a gigabyte, and it changes a configured limit rather than a code path — the same
comparison runs, against a different number.
"""
return {'IMPORT_MAX_OBJECT_BYTES': '1024'}
async def test_an_oversized_object_is_rejected_at_download(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
) -> None:
payload = import_bundle_factory()
assert len(payload) > 1024, 'the fixture bundle must exceed the narrowed ceiling'
handle, import_run_id, object_key = await _run_rejected_import(
payload=payload,
password=FACTORY_PASSWORD,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
await _assert_rejected(
handle=handle,
import_run_id=import_run_id,
object_key=object_key,
step=ImportStep.DOWNLOAD,
import_read_status_row=import_read_status_row,
import_assert_nothing_provisioned=import_assert_nothing_provisioned,
import_names=import_names,
)
class TestGate2Header:
"""Four header rejections, all landing on `download` — the step that says "not these bytes"."""
@pytest.mark.parametrize(
('case', 'header_kwargs'),
[
('wrong magic', {'magic': b'NOTSIENT'}),
('unsupported version', {'format_version': 9}),
('kdf memory above the ceiling', {'declared_memlimit': 512 * 1024 * 1024}),
('kdf passes above the ceiling', {'declared_opslimit': 9}),
('digest not of the chunk stream', {'digest_override': bytes(32)}),
],
)
async def test_a_malformed_header_is_rejected_at_download(
self,
case: str,
header_kwargs: dict[str, Any],
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
) -> None:
handle, import_run_id, object_key = await _run_rejected_import(
payload=import_bundle_factory(**header_kwargs),
password=FACTORY_PASSWORD,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
row = await _assert_rejected(
handle=handle,
import_run_id=import_run_id,
object_key=object_key,
step=ImportStep.DOWNLOAD,
import_read_status_row=import_read_status_row,
import_assert_nothing_provisioned=import_assert_nothing_provisioned,
import_names=import_names,
)
# None of these may borrow the decryption sentence: the password is not implicated.
assert 'password' not in (row['error_message'] or '').lower(), case
class TestGate3Decryption:
"""The password, the tag and the framing — the three ways a stream refuses to open."""
async def test_a_wrong_password_is_rejected_at_decryption(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
) -> None:
"""The envelope opens — it is the *bundle* password inside it that is wrong.
Worth separating: an envelope the worker cannot open is also a decryption-step failure, but
for a different reason, and only this one proves the KDF actually ran and produced a key
the stream rejected.
"""
handle, import_run_id, object_key = await _run_rejected_import(
payload=import_bundle_factory(password=FACTORY_PASSWORD),
password=FACTORY_PASSWORD,
envelope_password='Wrong1Password!Entirely',
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
await _assert_rejected(
handle=handle,
import_run_id=import_run_id,
object_key=object_key,
step=ImportStep.DECRYPTION,
import_read_status_row=import_read_status_row,
import_assert_nothing_provisioned=import_assert_nothing_provisioned,
import_names=import_names,
)
async def test_a_tampered_chunk_is_rejected_at_decryption(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
) -> None:
"""One flipped byte inside the stream, with the header digest repaired to match.
Repairing the digest is what makes this gate 3 rather than gate 2: the file is
self-consistent, so only the AEAD can tell it was altered.
"""
from tests.helpers.bundle_factory import tamper_ciphertext
handle, import_run_id, object_key = await _run_rejected_import(
payload=tamper_ciphertext(import_bundle_factory()),
password=FACTORY_PASSWORD,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
await _assert_rejected(
handle=handle,
import_run_id=import_run_id,
object_key=object_key,
step=ImportStep.DECRYPTION,
import_read_status_row=import_read_status_row,
import_assert_nothing_provisioned=import_assert_nothing_provisioned,
import_names=import_names,
)
async def test_a_stream_without_its_final_tag_is_rejected_at_decryption(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
) -> None:
"""Every chunk framed as MESSAGE, so the stream claims more is coming and never ends.
The digest cannot see this — the bytes on disk are exactly the ones it covers. Only the
`secretstream` framing knows the file is incomplete.
"""
handle, import_run_id, object_key = await _run_rejected_import(
payload=import_bundle_factory(drop_final_tag=True),
password=FACTORY_PASSWORD,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
await _assert_rejected(
handle=handle,
import_run_id=import_run_id,
object_key=object_key,
step=ImportStep.DECRYPTION,
import_read_status_row=import_read_status_row,
import_assert_nothing_provisioned=import_assert_nothing_provisioned,
import_names=import_names,
)
class TestGate4ArchiveInspection:
"""The central directory alone decides — nothing has been written to disk yet."""
@pytest.fixture
def import_limits_override(self) -> dict[str, str]:
"""A 1 MiB declared-expansion ceiling, for the oversized-expansion case in this class."""
return {'IMPORT_MAX_UNCOMPRESSED_BYTES': str(1024 * 1024)}
@pytest.mark.parametrize(
('case', 'kwargs'),
[
('entry escaping the tree', {'traversal_entry': 'artifacts/../../escaped.csv'}),
('absolute entry path', {'absolute_entry': '/opt/absolute.csv'}),
('symbolic link', {'symlinks': {'artifacts/link.csv': 'artifacts/train_data.csv'}}),
('declared expansion over the ceiling', {'bomb_bytes': 4 * 1024 * 1024}),
],
)
async def test_a_hostile_archive_is_rejected_before_anything_is_extracted(
self,
case: str,
kwargs: dict[str, Any],
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
tmp_path: Any,
) -> None:
handle, import_run_id, object_key = await _run_rejected_import(
payload=import_bundle_factory(**kwargs),
password=FACTORY_PASSWORD,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
await _assert_rejected(
handle=handle,
import_run_id=import_run_id,
object_key=object_key,
step=ImportStep.ARCHIVE_INSPECTION,
import_read_status_row=import_read_status_row,
import_assert_nothing_provisioned=import_assert_nothing_provisioned,
import_names=import_names,
)
# "Nothing extracted" is stronger than "nothing provisioned": not one member reached disk.
work_dir = tmp_path / 'import-work'
assert not list(work_dir.rglob('*')) if work_dir.exists() else True, case
class TestGate5Extraction:
"""A member that gate 4 passed and extraction still cannot place.
Gate 5 is about *placement*, not size: containment on the resolved path, and never overwriting
a file already written. A zip carrying the same name twice passes gate 4 — which checks names
and modes, not duplicates — and is refused here on the second member.
"""
async def test_a_duplicate_member_is_rejected_at_extraction(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
) -> None:
from tests.helpers.bundle_factory import default_tree
# `bomb_bytes` writes `artifacts/bomb.csv`, and `files` already carries that name, so the
# archive declares it twice. Small enough that no size ceiling is involved.
files = dict(default_tree())
files['artifacts/bomb.csv'] = b'first\n'
handle, import_run_id, object_key = await _run_rejected_import(
payload=import_bundle_factory(files=files, bomb_bytes=64),
password=FACTORY_PASSWORD,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
await _assert_rejected(
handle=handle,
import_run_id=import_run_id,
object_key=object_key,
step=ImportStep.EXTRACTION,
import_read_status_row=import_read_status_row,
import_assert_nothing_provisioned=import_assert_nothing_provisioned,
import_names=import_names,
)
class TestGate6Structure:
"""The layout and the metadata schema, checked on the extracted tree."""
@pytest.mark.parametrize(
('case', 'kwargs'),
[
('no metadata.json', {'include_metadata': False}),
('metadata missing run_id', {'drop_metadata_field': 'run_id'}),
('parameters missing target_variable', {'drop_parameter': 'target_variable'}),
],
)
async def test_an_incomplete_bundle_is_rejected_at_structure_validation(
self,
case: str,
kwargs: dict[str, Any],
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
) -> None:
handle, import_run_id, object_key = await _run_rejected_import(
payload=import_bundle_factory(**kwargs),
password=FACTORY_PASSWORD,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
await _assert_rejected(
handle=handle,
import_run_id=import_run_id,
object_key=object_key,
step=ImportStep.STRUCTURE_VALIDATION,
import_read_status_row=import_read_status_row,
import_assert_nothing_provisioned=import_assert_nothing_provisioned,
import_names=import_names,
)
async def test_a_bundle_without_prediction_model_is_rejected(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
) -> None:
"""Without `prediction_model` there is nothing to register a version against."""
from tests.helpers.bundle_factory import default_tree
files = {
name: content
for name, content in default_tree().items()
if not name.startswith('artifacts/prediction_model/')
}
handle, import_run_id, object_key = await _run_rejected_import(
payload=import_bundle_factory(files=files),
password=FACTORY_PASSWORD,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
await _assert_rejected(
handle=handle,
import_run_id=import_run_id,
object_key=object_key,
step=ImportStep.STRUCTURE_VALIDATION,
import_read_status_row=import_read_status_row,
import_assert_nothing_provisioned=import_assert_nothing_provisioned,
import_names=import_names,
)
async def test_an_over_long_experiment_name_never_reaches_the_varchar_column(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
import_bff_engine: Any,
) -> None:
"""A 60-character experiment name is refused at gate 6, not by Postgres three steps later.
`experiment_name` is `VARCHAR(50)` and the import writes the bundle's own name onto the row
*before* provisioning. If the gate did not enforce the column's limit, the names write would
be a `DataError` — non-retryable, recorded as a status failure, on a row whose import was
otherwise fine. Asserting the row's `experiment_name` is untouched is what proves the write
never happened.
"""
from sqlalchemy import text
long_name = 'x' * 60
handle, import_run_id, object_key = await _run_rejected_import(
payload=import_bundle_factory(metadata={'experiment_name': long_name}),
password=FACTORY_PASSWORD,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
row = await _assert_rejected(
handle=handle,
import_run_id=import_run_id,
object_key=object_key,
step=ImportStep.STRUCTURE_VALIDATION,
import_read_status_row=import_read_status_row,
import_assert_nothing_provisioned=import_assert_nothing_provisioned,
import_names=import_names,
experiment_name=long_name,
)
assert row['experiment_name'] == 'import-e2e-placeholder', (
'the over-long name reached the row, so the gate did not hold the column'
)
with import_bff_engine.begin() as connection:
width = connection.execute(
text(
'SELECT character_maximum_length FROM information_schema.columns '
"WHERE table_name = 'experiment_run' AND column_name = 'experiment_name'"
)
).scalar_one()
assert width == 50, 'the column this gate exists to protect is no longer VARCHAR(50)'
class TestGate7ContentPolicy:
"""A file kind an MLflow artifact tree has no reason to carry."""
async def test_a_forbidden_file_type_is_rejected_at_content_policy(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_assert_nothing_provisioned: Any,
) -> None:
handle, import_run_id, object_key = await _run_rejected_import(
payload=import_bundle_factory(extra_files={'artifacts/payload.exe': b'MZ\x00\x00'}),
password=FACTORY_PASSWORD,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
await _assert_rejected(
handle=handle,
import_run_id=import_run_id,
object_key=object_key,
step=ImportStep.CONTENT_POLICY,
import_read_status_row=import_read_status_row,
import_assert_nothing_provisioned=import_assert_nothing_provisioned,
import_names=import_names,
)
class TestTheRejectionSaysNothingTechnical:
"""`error_message` is a sentence for a person: no path, no object key, no traceback.
One case is enough to make the point structurally — the catalog is keyed by step and
`classify_import_failure` discards the exception's text entirely — and the per-gate cases above
each assert the sentence is *exactly* the catalog's, which is the same guarantee stated
positively.
"""
async def test_no_technical_detail_reaches_the_recorded_reason(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
) -> None:
handle, import_run_id, object_key = await _run_rejected_import(
payload=import_bundle_factory(include_metadata=False),
password=FACTORY_PASSWORD,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
with pytest.raises(WorkflowFailureError) as failure:
await handle.result()
row = import_read_status_row(import_run_id)
assert row is not None
message = row['error_message'] or ''
for forbidden in (
object_key,
IMPORT_BUCKET,
IMPORT_PREFIX,
'experiment_run',
'Traceback',
'BundleGateError',
tempfile.gettempdir(),
'gate',
):
assert forbidden not in message, f'{forbidden!r} leaked into error_message: {message!r}'
# The technical truth is not lost — it is on the Temporal error, where support can read it.
assert 'metadata' in failure_text(failure.value).lower()

314
e2e/test_import_model_happy_path.py Normal file
View File

@@ -0,0 +1,314 @@
"""An import that works, twice over: once from the producer's own bytes and once from the factory.
Two origins, because they prove different things. The **golden** bundle is bytes the producer's code
emitted — it decrypts at the producer's real Argon2id cost (256 MiB, 3 passes) and carries the tree
the producer really writes, so it is the only case that would notice the producer's layout moving.
The **factory** bundle is synthesised here, cheaply, with names unique to the test; it is what the
rest of the suite runs on, so a happy path over it is what makes the negative cases' "and nothing
was provisioned" meaningful.
What "it worked" means, in full: the experiment exists, a run exists inside it carrying the derived
run name, the artifact tree is under the run with `prediction_model` and `data_model` at its root,
every parameter the bundle declared is logged plus the origin's identity under `source.*`, a
registry version exists, the MongoDB document exists, and the status row ends `IMPORT_SUCCESS`.
Two things a successful import must **not** have done are asserted here too, because they are the
difference between "imported" and "live": no version is in `Production`, and the document is
`active: false`. Promotion and activation are manual operator steps (QTZPOC-21).
"""
from __future__ import annotations
import json
from typing import Any
import mlflow
import pytest
from e2e.import_harness import (
IMPORT_BUCKET,
IMPORT_PREFIX,
SEEDED_MODEL_IDS,
)
pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
GOLDEN_MODEL_NAME = 'fixture_model'
GOLDEN_EXPERIMENT_NAME = 'fixture_experiment'
GOLDEN_MODEL_VERSION = '3'
GOLDEN_TARGET = 'SE'
REQUIRED_ARTIFACT_ROOTS = {'prediction_model', 'data_model'}
def _mlflow_client(server: dict[str, Any]) -> Any:
return mlflow.tracking.MlflowClient(tracking_uri=server['tracking_uri'])
def _as_dict(value: Any) -> dict[str, Any]:
"""`request_data` / `orchestrator_response_data` come back as a dict or as JSON text."""
return json.loads(value) if isinstance(value, str) else (value or {})
async def _run_import(
*,
payload: bytes,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
password: str,
import_envelope: Any,
) -> tuple[Any, int, str]:
"""Upload, insert the row the BFF would have inserted, run the workflow. Return the outcome."""
object_key, digest = import_upload_bundle(payload)
import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
handle = await import_execute(
import_build_input(
import_run_id=import_run_id,
object_key=object_key,
expected_digest=digest,
password_envelope=import_envelope(password),
)
)
return handle, import_run_id, object_key
def _assert_provisioned(
*,
client: Any,
experiment_name: str,
model_name: str,
run_name: str,
expected_parameters: dict[str, str],
expected_source_fields: dict[str, str],
) -> tuple[str, Any]:
"""The whole positive claim about MLflow, in one place. Returns `(run_id, version)`."""
experiment = client.get_experiment_by_name(experiment_name)
assert experiment is not None, f'no experiment named {experiment_name!r}'
runs = [
run
for run in client.search_runs([experiment.experiment_id])
if run.info.run_name == run_name
]
assert len(runs) == 1, f'expected exactly one run named {run_name!r}, found {len(runs)}'
run = runs[0]
assert run.info.status == 'FINISHED'
artifact_roots = {artifact.path for artifact in client.list_artifacts(run.info.run_id)}
assert REQUIRED_ARTIFACT_ROOTS <= artifact_roots, (
f'the run holds {sorted(artifact_roots)}, missing {sorted(REQUIRED_ARTIFACT_ROOTS - artifact_roots)}'
)
# `prediction_model` is a directory the legacy loader walks, not an opaque blob.
prediction_files = {
artifact.path.rsplit('/', 1)[-1]
for artifact in client.list_artifacts(run.info.run_id, 'prediction_model')
}
assert {'MLmodel', 'model.pkl'} <= prediction_files, prediction_files
for name, value in expected_parameters.items():
assert run.data.params.get(name) == value, (
f'parameter {name!r} is {run.data.params.get(name)!r}, expected {value!r}'
)
for name, value in expected_source_fields.items():
assert run.data.params.get(f'source.{name}') == value, (
f'source.{name} is {run.data.params.get(f"source.{name}")!r}, expected {value!r}'
)
versions = list(client.search_model_versions(f"name='{model_name}'"))
assert len(versions) == 1, f'expected one registered version, found {len(versions)}'
version = versions[0]
assert version.run_id == run.info.run_id
return run.info.run_id, version
class TestGoldenBundle:
"""The producer's own bytes, at the producer's own KDF cost."""
async def test_the_producer_artefact_imports_end_to_end(
self,
import_status_table: Any,
import_models_collection: Any,
import_golden_bundle: dict[str, Any],
import_mlflow_server: dict[str, Any],
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
) -> None:
handle, import_run_id, object_key = await _run_import(
payload=import_golden_bundle['bytes'],
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
password=import_golden_bundle['password'],
import_envelope=import_envelope,
)
result = await handle.result()
assert result['model_name'] == GOLDEN_MODEL_NAME
assert result['import_run_id'] == import_run_id
assert result['version'] == '1', 'the destination numbering starts at 1, not at the bundle'
run_id, version = _assert_provisioned(
client=_mlflow_client(import_mlflow_server),
experiment_name=GOLDEN_EXPERIMENT_NAME,
model_name=GOLDEN_MODEL_NAME,
run_name=f'import-{GOLDEN_MODEL_NAME}-v{GOLDEN_MODEL_VERSION}',
expected_parameters={'target_variable': GOLDEN_TARGET, 'nan_treatment': 'drop'},
expected_source_fields={
'model_name': GOLDEN_MODEL_NAME,
'experiment_name': GOLDEN_EXPERIMENT_NAME,
'model_version': GOLDEN_MODEL_VERSION,
'model_project': 'fixture_project',
},
)
assert result['run_id'] == run_id
document = import_models_collection.find_one({'name': GOLDEN_MODEL_NAME}, {'_id': False})
assert document is not None, 'no model document was written'
assert document['id'] == str(max(int(i) for i in SEEDED_MODEL_IDS) + 1)
assert document['model_config']['target'] == GOLDEN_TARGET
row = import_read_status_row(import_run_id)
assert row is not None
assert row['status'] == 'IMPORT_SUCCESS'
assert row['error_message'] is None
assert row['experiment_name'] == GOLDEN_EXPERIMENT_NAME
assert row['run_name'] == f'import-{GOLDEN_MODEL_NAME}-v{GOLDEN_MODEL_VERSION}'
assert row['file_name'] == object_key
assert row['bucket_name'] == IMPORT_BUCKET
assert object_key.startswith(IMPORT_PREFIX)
# Imported, not live.
assert version.current_stage in (None, 'None'), version.current_stage
assert document['active'] is False
class TestFactoryBundle:
"""A synthesised tree with names unique to this test — the shape the rest of the suite runs on."""
async def test_a_factory_bundle_imports_end_to_end(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_mlflow_server: dict[str, Any],
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
) -> None:
payload = import_bundle_factory(parameters={'target_variable': 'SE'})
handle, import_run_id, _ = await _run_import(
payload=payload,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
password='Factory1Model!Test',
import_envelope=import_envelope,
)
result = await handle.result()
assert result['model_name'] == import_names['model_name']
run_id, version = _assert_provisioned(
client=_mlflow_client(import_mlflow_server),
experiment_name=import_names['experiment_name'],
model_name=import_names['model_name'],
# DEFAULT_METADATA declares model_version '2'.
run_name=f'import-{import_names["model_name"]}-v2',
expected_parameters={'target_variable': 'SE', 'retrain': 'False'},
expected_source_fields={
'model_name': import_names['model_name'],
'experiment_name': import_names['experiment_name'],
'model_version': '2',
},
)
assert result['run_id'] == run_id
assert result['version'] == '1'
document = import_models_collection.find_one(
{'name': import_names['model_name']}, {'_id': False}
)
assert document is not None
assert document['model_config'] == {
'transform_flavor': 'sklearn',
'predict_flavor': 'joblib',
'retention_minutes': 60,
'target': 'SE',
}
row = import_read_status_row(import_run_id)
assert row is not None
assert row['status'] == 'IMPORT_SUCCESS'
detail = _as_dict(row['orchestrator_response_data'])
assert detail['import']['workflow_id'] == handle.id
assert version.current_stage in (None, 'None')
assert document['active'] is False
class TestLegacyLoaderResolution:
"""The registry source an import writes has to be one the *existing* loader can parse.
This is the assertion that could only ever be made against a real registry: the importer writes
the run's resolved artifact URI with `prediction_model` appended, because `get_model_run_id`
extracts the run id as `source.split('/')[2]` — an index that is the run id only in the resolved
form. A literal `runs:/<run_id>/prediction_model` would put `prediction_model` at index 2, and a
`models:/m-*` source would have none of the parts at all.
It is called with the stage the version is actually in rather than `Production`, because an
import deliberately promotes nothing — passing `Production` here would test the promotion step
this feature does not have.
"""
async def test_the_imported_source_resolves_to_the_import_run(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_activities: Any,
import_mlflow_server: dict[str, Any],
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
) -> None:
payload = import_bundle_factory()
handle, _, _ = await _run_import(
payload=payload,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
password='Factory1Model!Test',
import_envelope=import_envelope,
)
result = await handle.result()
model_name = import_names['model_name']
client = _mlflow_client(import_mlflow_server)
version = next(iter(client.search_model_versions(f"name='{model_name}'")))
assert version.source.endswith('/prediction_model'), version.source
# The production method, on the production repository object, against the real registry.
resolved = import_activities.import_mlflow_repository.get_model_run_id(
model_name, stage=version.current_stage
)
assert resolved == result['run_id'], (
f'get_model_run_id resolved {resolved!r} from source {version.source!r}, '
f'but the import created run {result["run_id"]!r}'
)
artifact_roots = {artifact.path for artifact in client.list_artifacts(resolved)}
assert REQUIRED_ARTIFACT_ROOTS <= artifact_roots, sorted(artifact_roots)

364
e2e/test_import_model_provisioning_failures.py Normal file
View File

@@ -0,0 +1,364 @@
"""Each provisioning step fails as itself: its own code, its own sentence, nothing half-built.
The four steps after the gates — experiment creation, artifact upload, registration, the model
document — each get one failure, and every one lands `IMPORT_ERROR` rather than
`ORCHESTRATOR_VALIDATION_ERROR`. That split is not cosmetic: it is the table's existing distinction
between "the person who submitted this can fix it" and "only support can", and a provisioning
failure is always the second kind.
**How each failure is caused**, in design E7's order of preference:
| Step | Cause | Preference |
|---|---|---|
| `experiment_creation` | the MLflow server is genuinely stopped | broken dependency |
| `artifact_upload` | targeted patch on `log_artifacts` | patch, justified in place |
| `registration` | targeted patch on `create_model_version` | patch, justified in place |
| `model_document` | a document already holding the id the activity will compute, against the real unique index | broken dependency |
Two patches rather than the one design E7 anticipated. The reason is the same for both and is
recorded in each test: with one server serving both tracking and registry, and MLflow validating
nothing about a registered model name except that it is non-empty (which gate 6 already requires),
there is no way to make the world hostile to *just* one of these two steps. Stopping the server
fails experiment creation first; a malformed name is not malformed to MLflow. The patch is on the
client call the step makes, not on the activity or the workflow, so what is being tested is still
"the workflow attributes this step's failure to this step".
"""
from __future__ import annotations
import json
import tempfile
from typing import Any
import pytest
from temporalio.client import WorkflowFailureError
from e2e.import_harness import IMPORT_BUCKET, IMPORT_PREFIX, SEEDED_MODEL_IDS
from laborious.activities.model_import_errors import IMPORT_FAILURE_CATALOG
from laborious.utils.bundle.steps import ImportStep
pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
PLATFORM_ERROR_STATUS = 'IMPORT_ERROR'
FACTORY_PASSWORD = 'Factory1Model!Test'
def _as_dict(value: Any) -> dict[str, Any]:
return json.loads(value) if isinstance(value, str) else (value or {})
def _assert_platform_failure(
*,
row: dict[str, Any] | None,
step: ImportStep,
workflow_id: str,
object_key: str,
) -> None:
"""The shared claim of task 7.5, asserted by every case in this module.
`error_message` is the catalog sentence *exactly* — not "contains" — which is the strongest
possible form of "carries no path, table name, object key or traceback": there is nothing in it
that did not come from the catalog.
"""
assert row is not None, 'the failure lost the row it was recorded on'
assert row['status'] == PLATFORM_ERROR_STATUS, (
f"status is {row['status']!r}; a provisioning failure is not the user's to fix"
)
code, reason = IMPORT_FAILURE_CATALOG[step]
detail = _as_dict(row['orchestrator_response_data'])['import']
assert detail['step'] == step.value, detail
assert detail['code'] == code.value, detail
assert detail['workflow_id'] == workflow_id, detail
assert row['error_message'] == reason
message = row['error_message'] or ''
for forbidden in (
object_key,
IMPORT_BUCKET,
IMPORT_PREFIX,
'experiment_run',
'Traceback',
'Error',
'mlflow',
'Mongo',
tempfile.gettempdir(),
):
assert forbidden not in message, f'{forbidden!r} leaked into error_message: {message!r}'
async def _upload_and_start(
*,
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
**factory_kwargs: Any,
) -> tuple[Any, int, str]:
object_key, digest = import_upload_bundle(import_bundle_factory(**factory_kwargs))
import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
handle = await import_execute(
import_build_input(
import_run_id=import_run_id,
object_key=object_key,
expected_digest=digest,
password_envelope=import_envelope(FACTORY_PASSWORD),
)
)
return handle, import_run_id, object_key
class TestExperimentCreation:
"""The tracking server is not there. No patch, no mock — the process is stopped."""
async def test_a_stopped_tracking_server_fails_at_experiment_creation(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
import_mlflow_stopped: Any,
import_assert_nothing_provisioned: Any,
) -> None:
with import_mlflow_stopped():
handle, import_run_id, object_key = await _upload_and_start(
import_bundle_factory=import_bundle_factory,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
with pytest.raises(WorkflowFailureError):
await handle.result()
row = import_read_status_row(import_run_id)
_assert_platform_failure(
row=row,
step=ImportStep.EXPERIMENT_CREATION,
workflow_id=handle.id,
object_key=object_key,
)
# The bundle *was* opened before MLflow was reached, so the names are on the row — which is
# what task 9.3 asks for, observed from a failure that stopped exactly there.
assert row is not None
assert row['experiment_name'] == import_names['experiment_name']
assert row['run_name'] == f'import-{import_names["model_name"]}-v2'
# Nothing in Mongo: the document is the last step and never ran.
documents = list(import_models_collection.find({}, {'_id': False}))
assert {str(d['id']) for d in documents} == set(SEEDED_MODEL_IDS)
import_assert_nothing_provisioned(
experiment_name=import_names['experiment_name'],
model_name=import_names['model_name'],
object_key=object_key,
)
class TestArtifactUpload:
"""The step design E7 names as the one a patch is needed for."""
async def test_a_failing_artifact_upload_is_recorded_as_its_own_step(
self,
monkeypatch: pytest.MonkeyPatch,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_activities: Any,
import_mlflow_server: dict[str, Any],
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
) -> None:
"""Why a patch: with the server up and the bundle valid, nothing else fails only this step.
Hostile input cannot reach here — a bundle that gets past gate 7 has a readable tree, and
one that does not never reaches artifact upload. A broken dependency cannot isolate it
either: the only dependency this step has is the tracking server, and stopping that fails
experiment creation one step earlier. So the patch is on the single client call the step
makes, `log_artifacts`; everything around it — the run creation, the parameters, the
workflow's attribution of the failure — is the real code.
"""
client = import_activities.import_mlflow_repository.client
def refuse(*_args: Any, **_kwargs: Any) -> None:
raise OSError('artifact store is unreachable')
monkeypatch.setattr(client, 'log_artifacts', refuse)
handle, import_run_id, object_key = await _upload_and_start(
import_bundle_factory=import_bundle_factory,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
with pytest.raises(WorkflowFailureError):
await handle.result()
_assert_platform_failure(
row=import_read_status_row(import_run_id),
step=ImportStep.ARTIFACT_UPLOAD,
workflow_id=handle.id,
object_key=object_key,
)
# The experiment exists — it was the previous step and it succeeded. Nothing after does.
mlflow_client = _fresh_client(import_mlflow_server)
assert mlflow_client.get_experiment_by_name(import_names['experiment_name']) is not None
assert not list(mlflow_client.search_model_versions(f"name='{import_names['model_name']}'"))
assert import_models_collection.find_one({'name': import_names['model_name']}) is None
class TestRegistration:
"""The artifacts are stored and the model still cannot be registered."""
async def test_a_failing_registration_is_recorded_as_its_own_step(
self,
monkeypatch: pytest.MonkeyPatch,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_activities: Any,
import_mlflow_server: dict[str, Any],
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
) -> None:
"""Why a patch here too, contrary to design E7's expectation that this step needs none.
Hostile input was the first thing tried: a registered model name MLflow refuses. It does not
exist — `_validate_model_name` rejects only the empty string, which gate 6 already refuses,
and the sqlite backend store does not enforce the `VARCHAR(256)` the schema declares, so
even a 300-character name is accepted. A broken dependency was the second: one process
serves both tracking and the registry, so stopping it fails experiment creation three steps
earlier. The patch is on `create_model_version`, the one call this step is.
"""
client = import_activities.import_mlflow_repository.client
def refuse(*_args: Any, **_kwargs: Any) -> None:
raise RuntimeError('registry write rejected')
monkeypatch.setattr(client, 'create_model_version', refuse)
handle, import_run_id, object_key = await _upload_and_start(
import_bundle_factory=import_bundle_factory,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
with pytest.raises(WorkflowFailureError):
await handle.result()
_assert_platform_failure(
row=import_read_status_row(import_run_id),
step=ImportStep.REGISTRATION,
workflow_id=handle.id,
object_key=object_key,
)
# The run and its artifacts survive — "the model's files were stored, but the model itself
# could not be registered" is exactly what the catalog sentence for this step says.
mlflow_client = _fresh_client(import_mlflow_server)
experiment = mlflow_client.get_experiment_by_name(import_names['experiment_name'])
assert experiment is not None
runs = mlflow_client.search_runs([experiment.experiment_id])
assert len(runs) == 1
assert {a.path for a in mlflow_client.list_artifacts(runs[0].info.run_id)} >= {
'prediction_model',
'data_model',
}
assert import_models_collection.find_one({'name': import_names['model_name']}) is None
class TestModelDocument:
"""The last step, failed by a real unique index on a real collection."""
async def test_a_duplicate_id_fails_at_model_document(
self,
import_status_table: Any,
import_models_collection: Any,
import_names: dict[str, str],
import_bundle_factory: Any,
import_mlflow_server: dict[str, Any],
import_upload_bundle: Any,
import_insert_status_row: Any,
import_build_input: Any,
import_execute: Any,
import_envelope: Any,
import_read_status_row: Any,
) -> None:
"""A document already holding the id `max(int(id)) + 1` will compute, under another name.
Under another name on purpose: a document with the *same* name would make the activity skip
the insert entirely, which is a different behaviour (task 8.6) and would not fail. This one
collides only on `id`, so the insert is attempted and the index refuses it.
No patch and no mock: the failure is `pymongo`'s duplicate-key error from a real index,
which is the thing that would happen in production if two imports raced.
"""
collision_id = str(max(int(seeded) for seeded in SEEDED_MODEL_IDS) + 1)
import_models_collection.insert_one(
{'id': collision_id, 'name': 'a_different_model', 'active': True, 'model_config': {}}
)
handle, import_run_id, object_key = await _upload_and_start(
import_bundle_factory=import_bundle_factory,
import_upload_bundle=import_upload_bundle,
import_insert_status_row=import_insert_status_row,
import_build_input=import_build_input,
import_execute=import_execute,
import_envelope=import_envelope,
)
with pytest.raises(WorkflowFailureError):
await handle.result()
_assert_platform_failure(
row=import_read_status_row(import_run_id),
step=ImportStep.MODEL_DOCUMENT,
workflow_id=handle.id,
object_key=object_key,
)
# No document for this model, and the colliding one is untouched.
assert import_models_collection.find_one({'name': import_names['model_name']}) is None
blocker = import_models_collection.find_one({'id': collision_id}, {'_id': False})
assert blocker == {
'id': collision_id,
'name': 'a_different_model',
'active': True,
'model_config': {},
}
# Everything before it stands: "the model was registered, but the settings ... could not be
# saved" is the catalog sentence, and this is what makes it true rather than merely stored.
mlflow_client = _fresh_client(import_mlflow_server)
versions = list(mlflow_client.search_model_versions(f"name='{import_names['model_name']}'"))
assert len(versions) == 1
def _fresh_client(server: dict[str, Any]) -> Any:
"""An MlflowClient bound to the test server, independent of the one the activities hold."""
import mlflow
return mlflow.tracking.MlflowClient(tracking_uri=server['tracking_uri'])