Files
vitor-aignosi dac4f1718f QTZPOC-20
QTZPOC-20: Implement model import workflow for .sientia bundles

Introduces the import_model workflow and its associated activities to enable importing .sientia bundles into MLflow and MongoDB. This includes:

*   New core workflow and activity logic for handling bundle download, decryption, validation, MLflow provisioning (experiments, runs, registered versions), and MongoDB model document creation.
*   Comprehensive E2E and unit test suites, including a dedicated test harness and a manual run script for development.
*   Updated .env.example and README.md with new configurations and usage instructions.
*   Ensures correct MLflow model source resolution for legacy loader compatibility and robust MongoDB document management.
*   Refactors existing utility scripts into a scripts/ directory.
2026-08-25 17:07:42 -03:00

175 lines
6.6 KiB
Python

"""A recording stand-in for the import log's SQLAlchemy engine.
The import's status writes are the part of the feature with the strictest contract: which table they
name, which predicate they carry, which columns they touch, how many transactions they open and
which of them may fail without failing the import. Asserting that against a mock whose `execute`
returns a `MagicMock` proves nothing — the assertions need the statement text and the parameters.
So this double records every statement it is handed, answers reads from a queue the test fills, and
lets a test make a write fail on a chosen attempt. It deliberately understands nothing about SQL
beyond the first word of the statement.
"""
from __future__ import annotations
from typing import Any
class RecordedStatement:
"""One statement as the engine saw it: its normalised text and its bound parameters."""
def __init__(self, statement: Any, parameters: dict[str, Any] | None) -> None:
self.sql = ' '.join(str(statement).split())
self.parameters: dict[str, Any] = dict(parameters or {})
@property
def verb(self) -> str:
return self.sql.split(' ', 1)[0].upper()
@property
def is_write(self) -> bool:
return self.verb in ('UPDATE', 'INSERT', 'DELETE')
@property
def table(self) -> str:
"""The relation the statement names, for the "one table only" assertions."""
words = self.sql.split()
if self.verb == 'UPDATE':
return words[1]
for index, word in enumerate(words):
if word.upper() in ('FROM', 'INTO') and index + 1 < len(words):
return words[index + 1]
return ''
@property
def assignments(self) -> set[str]:
"""The column names a `SET` clause assigns, so a test can assert what is never written."""
upper = self.sql.upper()
if ' SET ' not in upper:
return set()
body = self.sql[upper.index(' SET ') + 5 :]
where = body.upper().find(' WHERE ')
if where != -1:
body = body[:where]
return {part.split('=')[0].strip() for part in body.split(',') if '=' in part}
def __repr__(self) -> str: # pragma: no cover - debugging aid
return f'RecordedStatement({self.sql!r}, {self.parameters!r})'
class FakeResult:
"""What `Connection.execute` returns: a rowcount for writes, mappings for reads."""
def __init__(self, rowcount: int = 1, rows: list[dict[str, Any]] | None = None) -> None:
self.rowcount = rowcount
self._rows = rows or []
def mappings(self) -> FakeResult:
return self
def first(self) -> dict[str, Any] | None:
return self._rows[0] if self._rows else None
def all(self) -> list[dict[str, Any]]:
return list(self._rows)
class FakeConnection:
"""A connection that records and then answers from the engine's queues."""
def __init__(self, engine: FakeStatusEngine) -> None:
self._engine = engine
def execute(self, statement: Any, parameters: dict[str, Any] | None = None) -> FakeResult:
recorded = RecordedStatement(statement, parameters)
self._engine.statements.append(recorded)
if recorded.is_write:
return self._engine._answer_write()
return self._engine._answer_read()
class _Scope:
"""The context manager `engine.begin()` and `engine.connect()` both hand out."""
def __init__(self, engine: FakeStatusEngine, transactional: bool) -> None:
self._engine = engine
self._transactional = transactional
def __enter__(self) -> FakeConnection:
self._engine.open_scopes += 1
if self._transactional:
self._engine.transactions += 1
else:
self._engine.connections += 1
return FakeConnection(self._engine)
def __exit__(self, *_exc: object) -> bool:
self._engine.open_scopes -= 1
return False
class FakeStatusEngine:
"""An engine double that records statements and can be told to fail a write.
Attributes:
statements: every statement executed, in order.
rowcounts: rowcounts for writes, consumed in order; the last one repeats.
rows: rows for reads, consumed in order; the last one repeats. `None` means "no such row".
write_errors: exceptions to raise instead of executing a write, consumed in order. A `None`
entry means "this attempt succeeds", which is how a retry is modelled.
transactions: how many `begin()` scopes were opened — one per write, or the writes are
sharing a transaction.
open_scopes: scopes currently open, so a test can assert none is left open.
"""
def __init__(
self,
rowcounts: list[int] | None = None,
rows: list[dict[str, Any] | None] | None = None,
write_errors: list[BaseException | None] | None = None,
) -> None:
self.statements: list[RecordedStatement] = []
self.rowcounts: list[int] = list(rowcounts or [])
self.rows: list[dict[str, Any] | None] = list(rows or [])
self.write_errors: list[BaseException | None] = list(write_errors or [])
self.transactions = 0
self.connections = 0
self.open_scopes = 0
self.disposed = False
# ------------------------------------------------------------------ engine API
def begin(self) -> _Scope:
return _Scope(self, transactional=True)
def connect(self) -> _Scope:
return _Scope(self, transactional=False)
def dispose(self) -> None:
self.disposed = True
# ------------------------------------------------------------------- answering
def _answer_write(self) -> FakeResult:
if self.write_errors:
error = self.write_errors.pop(0)
if error is not None:
raise error
rowcount = self.rowcounts.pop(0) if len(self.rowcounts) > 1 else (self.rowcounts or [1])[0]
return FakeResult(rowcount=rowcount)
def _answer_read(self) -> FakeResult:
row = self.rows.pop(0) if len(self.rows) > 1 else (self.rows or [None])[0]
return FakeResult(rows=[row] if row is not None else [])
# ---------------------------------------------------------------- convenience
@property
def writes(self) -> list[RecordedStatement]:
return [statement for statement in self.statements if statement.is_write]
@property
def reads(self) -> list[RecordedStatement]:
return [statement for statement in self.statements if not statement.is_write]
@property
def tables(self) -> set[str]:
return {statement.table for statement in self.statements if statement.table}