gitea_admin/sientia-dataops-laborious-legacy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

QTZPOC-20

Browse Source 
QTZPOC-20: Implement structured logging with metadata for repository, worker, and OPC activities.
This commit is contained in:
Vitor Pimentel dos Santos
2026-09-10 14:15:42 -03:00
parent 4cccc45c34
commit 6c0d178f99
5 changed files with 60 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
laborious/utils/repository/model_repository.py
View File

@@ -52,7 +52,7 @@ MODEL_PICKLE_NAME = 'model.pkl'
INVALID_FLAVOR_MESSAGE = "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'joblib'."
def force_memory_release(logger: Logger):
def force_memory_release(logger: Logger, metadata: dict[str, Any] | None = None):
"""Attempt to release memory from the Python process.
Executes a garbage collection cycle and calls `malloc_trim(0)` on glibc
@@ -61,14 +61,17 @@ def force_memory_release(logger: Logger):
Args:
logger (Logger): Logger for observability.
metadata (dict[str, Any] | None): Metadata used for structured logging.
"""
gc.collect()
metadata = metadata or {}
try:
ctypes.CDLL('libc.so.6').malloc_trim(0)
logger.info('Memory released')
logger.custom_info('Memory released', metadata)
except Exception as e:
logger.info(f'Memory release failed: {e}')
logger.custom_info(f'Memory release failed: {e}', metadata)
class MLFlowRepository(SientiaMonitoring):
@@ -320,7 +323,7 @@ class MLFlowRepository(SientiaMonitoring):
rmtree(full_path)
makedirs(output_dir, exist_ok=True)
self.info(f'Downloading artifacts from {run_id} to {output_dir}')
self.info(f'Downloading artifacts from {run_id} to {output_dir}', metadata)
core_labels = self.get_core_labels(metadata, operation_type='download_artifacts')
@@ -676,7 +679,7 @@ class MLFlowRepository(SientiaMonitoring):
model_uri = None if flavor == 'joblib' else f'models:/{model_name}/production'
if model_uri is not None:
self.info(f'Loading prediction model {model_name} from {model_uri}')
self.info(f'Loading prediction model {model_name} from {model_uri}', metadata)
core_labels = self.get_core_labels(metadata, operation_type='load_predict_model')
start_time = time.time()
@@ -756,7 +759,7 @@ class MLFlowRepository(SientiaMonitoring):
)
if model_uri is not None:
self.info(f'Loading data model {model_name} from {model_uri}')
self.info(f'Loading data model {model_name} from {model_uri}', metadata)
core_labels = self.get_core_labels(metadata, operation_type='load_transform_model')
start_time = time.time()
@@ -822,7 +825,8 @@ class MLFlowRepository(SientiaMonitoring):
"""
self.info(
f'Downloading {model_type} model {model_name} with flavor {flavor} and load_wrapper {load_wrapper}'
f'Downloading {model_type} model {model_name} with flavor {flavor} and load_wrapper {load_wrapper}',
metadata,
)
if model_type not in ['predict', 'transform']:
@@ -838,7 +842,10 @@ class MLFlowRepository(SientiaMonitoring):
if load_wrapper:
core_labels = self.get_core_labels(metadata, operation_type='download_model_wrapper')
self.info(f'Loading wrapper for {model_type} model {model_name} with flavor {flavor}')
self.info(
f'Loading wrapper for {model_type} model {model_name} with flavor {flavor}',
metadata,
)
try:
artifact_path = await self.dowload_artifacts(
@@ -846,7 +853,8 @@ class MLFlowRepository(SientiaMonitoring):
)
self.info(
f'Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}'
f'Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}',
metadata,
)
raw_model = mlflow.pyfunc.load_model(artifact_path)
@@ -959,33 +967,37 @@ class MLFlowRepository(SientiaMonitoring):
return False
return True
def handle_valid_model(self, model_name: str, cache: dict) -> dict:
def handle_valid_model(self, model_name: str, cache: dict, metadata: dict[str, Any]) -> dict:
"""
Return the cached model configuration when retention is valid.
Args:
model_name (str): Name of the model (for logging/consistency).
cache (dict): Cached model data structure.
metadata (dict[str, Any]): Metadata used for structured logging.
Returns:
dict: Model configuration.
"""
self.debug(f'Model {model_name} is still valid, using cached version')
self.debug(f'Model {model_name} is still valid, using cached version', metadata)
return cache['target']
def handle_outdated_model(self, model_name: str, model_key: str) -> None:
def handle_outdated_model(
self, model_name: str, model_key: str, metadata: dict[str, Any]
) -> None:
"""
Clean up outdated cached model and its artifacts.
Args:
model_name (str): Name of the model.
model_key (str): Cache key for the model.
metadata (dict[str, Any]): Metadata used for structured logging.
Returns:
None
"""
self.debug(f'Model {model_name} is outdated, downloading a new one')
self.debug(f'Model {model_name} is outdated, downloading a new one', metadata)
del self.model_cache[model_key]['target']
del self.model_cache[model_key]
@@ -1031,13 +1043,18 @@ class MLFlowRepository(SientiaMonitoring):
# Check if config has changed or is outdated
if self.check_cache_retention(cache, retention):
return self.handle_valid_model(model_name=model_name, cache=cache)
return self.handle_valid_model(
model_name=model_name, cache=cache, metadata=metadata
)
else:
# Model is outdated, delete old model files
self.handle_outdated_model(model_name=model_name, model_key=model_key)
self.handle_outdated_model(
model_name=model_name, model_key=model_key, metadata=metadata
)
else:
self.debug(
f'Model {model_name} is not in {model_type} cache, downloading a new one'
f'Model {model_name} is not in {model_type} cache, downloading a new one',
metadata,
)
# Donwload new model (without lock to avoid blocking other threads)
@@ -1113,10 +1130,10 @@ class MLFlowRepository(SientiaMonitoring):
prediction = model.predict(data)
if retention == 0:
self.info(f'Deleting model {model_name}:{operation} from memory')
self.info(f'Deleting model {model_name}:{operation} from memory', metadata)
del model
force_memory_release(self.logger)
force_memory_release(self.logger, metadata)
return prediction
@@ -1550,7 +1567,7 @@ class MLFlowRepository(SientiaMonitoring):
del data_model['model']
del data_model
force_memory_release(self.logger)
force_memory_release(self.logger, metadata)
return {
'run_id': run_id,