gitea_admin/sientia-dataops-laborious-legacy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

QTZPOC-20

Browse Source 
QTZPOC-20: Refactor model import workflow, introduce atomic model ID generation, and clarify MinIO expiry.

Fuses bundle download, decryption, MLflow experiment creation, and artifact upload into a single activity to prevent cross-worker issues. Implements an atomic counter for MongoDB model IDs, replacing the max(id) + 1 scan to avoid collisions during concurrent imports. Adds comprehensive documentation for MinIO object expiry mechanisms and refines lifecycle rule application. Improves error handling with clearer responsibility classification and centralizes MLflow repository injection.
This commit is contained in:
Vitor Pimentel dos Santos
2026-09-08 17:06:25 -03:00
parent e4fc12e529
commit 192834bde3
34 changed files with 1485 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
laborious/workflows/import_model.py
View File

@@ -16,11 +16,13 @@ Shape of this workflow, and the reasons it has that shape:
boundary, and the password reaches this module only as an envelope it cannot open.
"""
import asyncio
from datetime import timedelta
from temporalio import workflow
with workflow.unsafe.imports_passed_through():
from collections.abc import Awaitable, Callable
from typing import Any
from sientia_do.temporal.policies import retry_policy
@@ -29,7 +31,6 @@ with workflow.unsafe.imports_passed_through():
from laborious.activities.model_import import (
IMPORT_STATUS_HINT_RETRY_POLICY,
IMPORT_STATUS_RETRY_POLICY,
derive_run_name,
)
from laborious.utils.bundle.steps import ImportStep
from laborious.utils.models.import_errors import (
@@ -44,10 +45,15 @@ with workflow.unsafe.imports_passed_through():
# which is the one configuration mistake that would silently undo design D20.
STATUS_WRITE_TIMEOUT = timedelta(seconds=30)
STATUS_WRITE_ENVELOPE = timedelta(seconds=900)
DOWNLOAD_TIMEOUT = timedelta(seconds=1800)
OPEN_TIMEOUT = timedelta(seconds=1800)
EXPERIMENT_TIMEOUT = timedelta(seconds=300)
ARTIFACT_UPLOAD_TIMEOUT = timedelta(seconds=1800)
# Download, decryption/extraction, experiment creation and artifact upload now run inside one
# activity attempt (bug 8): the budget is the sum of what the four separate activities used to get,
# since a retry of the fused activity redoes all four phases.
PREPARE_AND_UPLOAD_TIMEOUT = (
timedelta(seconds=1800)
+ timedelta(seconds=1800)
+ timedelta(seconds=300)
+ timedelta(seconds=1800)
)
REGISTRATION_TIMEOUT = timedelta(seconds=600)
DOCUMENT_TIMEOUT = timedelta(seconds=300)
CLEANUP_TIMEOUT = timedelta(seconds=600)
@@ -124,8 +130,7 @@ class ImportModel:
schedule_to_close_timeout=STATUS_WRITE_ENVELOPE,
)
current_step = ImportStep.RECEIVED
failed_step: str | None = None
step: str = ImportStep.RECEIVED.value
gate: int | None = None
succeeded = False
source: dict[str, Any] = {}
@@ -134,79 +139,74 @@ class ImportModel:
try:
self._validate_input(input_data, claim)
current_step = ImportStep.DOWNLOAD
await self._hint(base, current_step)
await self._ensure_retention(base, input_data)
download = await workflow.execute_activity_method(
Activities.download_import_bundle,
{
**base,
'bucket': input_data.get('bucket'),
'object_key': input_data['object_key'],
'expected_digest': input_data['expected_digest'],
},
retry_policy=retry_policy,
start_to_close_timeout=DOWNLOAD_TIMEOUT,
# Download, decryption/extraction, experiment creation and artifact upload run inside
# one activity attempt (bug 8), so no later phase can depend on a file a different,
# separately-scheduled activity wrote to a different worker's disk. One hint covers the
# whole fused step; an exception it raises with no location of its own falls back to
# this step, since that is what the workflow was attempting.
step = ImportStep.DOWNLOAD.value
await self._record_hint(
base, Activities.record_import_status, {'step': step, 'source': None}
)
current_step = ImportStep.DECRYPTION
await self._hint(base, current_step)
opened = await workflow.execute_activity_method(
Activities.open_import_bundle,
{
**base,
'path': download['path'],
'password_envelope': input_data['password_envelope'],
},
retry_policy=retry_policy,
start_to_close_timeout=OPEN_TIMEOUT,
# Achado #9: the retention check carries no data this step consumes and already
# swallows its own failure (`_ensure_retention`), so it runs alongside the fused
# download+decrypt+provision activity instead of blocking it. Achado #8 fused
# `download_import_bundle` into `prepare_and_upload_import_bundle` before this change
# landed, so this gathers against that fused activity, not the original isolated one.
_, prepared = await asyncio.gather(
self._ensure_retention(base, input_data),
workflow.execute_activity_method(
Activities.prepare_and_upload_import_bundle,
{
**base,
'bucket': input_data.get('bucket'),
'object_key': input_data['object_key'],
'expected_digest': input_data['expected_digest'],
'password_envelope': input_data['password_envelope'],
},
retry_policy=retry_policy,
start_to_close_timeout=PREPARE_AND_UPLOAD_TIMEOUT,
),
)
source = opened['metadata']
parameters = opened['parameters']
source = prepared['metadata']
parameters = prepared['parameters']
model_name = source['model_name']
metadata['model_name'] = model_name
run_name = derive_run_name(model_name, source['model_version'])
run_name = prepared['run_name']
# The names go on the record before anything exists in MLflow, so a person watching the
# import list sees *which* model is being imported while it is still running.
await self._record_names(base, source, run_name)
current_step = ImportStep.EXPERIMENT_CREATION
await self._hint(base, current_step, source)
experiment = await workflow.execute_activity_method(
Activities.create_import_experiment,
{**base, 'experiment_name': source['experiment_name']},
retry_policy=retry_policy,
start_to_close_timeout=EXPERIMENT_TIMEOUT,
)
current_step = ImportStep.ARTIFACT_UPLOAD
await self._hint(base, current_step, source)
run = await workflow.execute_activity_method(
Activities.upload_import_artifacts,
# The names are known only once the fused activity returns; they are still recorded
# before the two provisioning steps that follow (the registered version and the model
# document), which is the guarantee `model-import-status-record` now makes.
await self._record_hint(
base,
Activities.record_import_names,
{
**base,
'experiment_id': experiment['experiment_id'],
'experiment_name': source['experiment_name'],
'run_name': run_name,
'extracted_dir': opened['extracted_dir'],
'parameters': parameters,
'source': source,
'step': ImportStep.EXPERIMENT_CREATION.value,
},
retry_policy=retry_policy,
start_to_close_timeout=ARTIFACT_UPLOAD_TIMEOUT,
)
current_step = ImportStep.REGISTRATION
await self._hint(base, current_step, source)
step = ImportStep.REGISTRATION.value
await self._record_hint(
base,
Activities.record_import_status,
{'step': step, 'source': source or None},
)
version = await workflow.execute_activity_method(
Activities.register_import_model_version,
{**base, 'model_name': model_name, 'run_id': run['run_id']},
{**base, 'model_name': model_name, 'run_id': prepared['run_id']},
retry_policy=retry_policy,
start_to_close_timeout=REGISTRATION_TIMEOUT,
)
current_step = ImportStep.MODEL_DOCUMENT
await self._hint(base, current_step, source)
step = ImportStep.MODEL_DOCUMENT.value
await self._record_hint(
base,
Activities.record_import_status,
{'step': step, 'source': source or None},
)
document = await workflow.execute_activity_method(
Activities.write_import_model_document,
{
@@ -221,23 +221,23 @@ class ImportModel:
succeeded = True
result = {
'import_run_id': import_run_id,
'run_id': run['run_id'],
'run_id': prepared['run_id'],
'model_name': model_name,
'version': version['version'],
'document_id': document.get('id'),
}
except BaseException as error: # NOSONAR - classified once below, then re-raised untouched
located_step, located_gate = failure_location(error)
failed_step = located_step or current_step.value
step = located_step or step
gate = located_gate
await self._report_failure(base, error, step=failed_step, gate=gate)
await self._report_failure(base, error, step=step, gate=gate)
raise
finally:
cleanup = await self._cleanup(base)
await self._record_terminal(
base,
succeeded=succeeded,
step=failed_step or current_step.value,
step=step,
gate=gate,
source=source,
cleanup_failed=not cleanup.get('cleaned', True),
@@ -294,43 +294,23 @@ class ImportModel:
# --------------------------------------------------------------- status writes
async def _hint(
async def _record_hint(
self,
base: dict[str, Any],
step: ImportStep,
source: dict[str, Any] | None = None,
activity: Callable[..., Awaitable[Any]],
extra: dict[str, Any],
) -> None:
"""Record the step about to be attempted. A hint: its failure never fails the import."""
"""Record a status hint. A hint: its failure never fails the import."""
try:
await workflow.execute_activity_method(
Activities.record_import_status,
{**base, 'step': step.value, 'source': source or None},
activity,
{**base, **extra},
retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY,
start_to_close_timeout=STATUS_WRITE_TIMEOUT,
)
except BaseException as error: # NOSONAR - a lost hint must not fail a working import
await self._report_status_write_failure(base, error)
async def _record_names(
self, base: dict[str, Any], source: dict[str, Any], run_name: str
) -> None:
"""Write the experiment and run names. Also a hint: the run is created either way."""
try:
await workflow.execute_activity_method(
Activities.record_import_names,
{
**base,
'experiment_name': source['experiment_name'],
'run_name': run_name,
'source': source,
'step': ImportStep.EXPERIMENT_CREATION.value,
},
retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY,
start_to_close_timeout=STATUS_WRITE_TIMEOUT,
)
except BaseException as error: # NOSONAR - the names are for a human reading the list
await self._report_status_write_failure(base, error)
async def _record_terminal(
self,
base: dict[str, Any],