QTZPOC-20

QTZPOC-20: Refactor model import workflow, introduce atomic model ID generation, and clarify MinIO expiry.

Fuses bundle download, decryption, MLflow experiment creation, and artifact upload into a single activity to prevent cross-worker issues. Implements an atomic counter for MongoDB model IDs, replacing the max(id) + 1 scan to avoid collisions during concurrent imports. Adds comprehensive documentation for MinIO object expiry mechanisms and refines lifecycle rule application. Improves error handling with clearer responsibility classification and centralizes MLflow repository injection.
This commit is contained in:
2026-09-08 17:06:25 -03:00
parent e4fc12e529
commit 192834bde3
34 changed files with 1485 additions and 531 deletions

View File

@@ -17,7 +17,9 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.temporal.activities.postgres import Postgres
from sientia_do.temporal.constants import now
from laborious.utils.messages import MISSING_MINIO_REPOSITORY
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
from laborious.utils.time import to_naive
_LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'})
@@ -83,7 +85,7 @@ class Storage(Postgres, MinioManager):
dict[str, Any]: Flat ``MinioDataFramePayload`` dict or ``success: False`` on failure.
"""
if self.minio_repository is None:
raise ValueError('Minio repository not initialized')
raise ValueError(MISSING_MINIO_REPOSITORY)
metadata: dict = input_data.get('metadata', {})
model_name = input_data['model_name']
@@ -129,6 +131,11 @@ class Storage(Postgres, MinioManager):
"""
Delete objects under the given prefixes that are older than the retention window.
See also: `ModelImport.ensure_import_bundle_retention`
(laborious/activities/model_import.py) — a newer, unrelated lifecycle-policy expiry
mechanism for the import bundle prefix. The two are independent by design for now; see
docs/minio-object-expiry.md.
Args (input_data):
metadata (dict): Workflow metadata for logging and metrics.
prefixes (list[str]): Key prefixes to scan (one level or subtree per prefix).
@@ -137,15 +144,13 @@ class Storage(Postgres, MinioManager):
dict[str, Any]: ``success``, ``deleted_count``, and optional ``message``.
"""
if self.minio_repository is None:
raise ValueError('Minio repository not initialized')
raise ValueError(MISSING_MINIO_REPOSITORY)
metadata = input_data.get('metadata', {})
payload = MinioDataFramePayload.from_dict(input_data['data'])
prefix = payload.cleanup_prefix()
base = now()
cutoff = (base.replace(tzinfo=None) if base.tzinfo else base) - timedelta(
hours=self.retention_hours
)
cutoff = to_naive(base) - timedelta(hours=self.retention_hours)
report: dict[str, Any] = {
'failed': {},