SIENTIAPDE-1712

Update MinioDataFramePayload to reflect changes from training to prediction datasets

- Renamed TRAINING_DATASETS_PREFIX to PREDICTION_DATASETS_PREFIX for clarity.
- Updated object key naming convention to use prediction datasets directory.
This commit is contained in:
vitor-aignosi
2026-03-23 10:24:58 -03:00
parent e17824eb85
commit ccbed58234

View File

@@ -34,9 +34,9 @@ OFFLOAD_THRESHOLD_BYTES = int(
float(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES', '1.5')) * 1024 * 1024 float(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES', '1.5')) * 1024 * 1024
) )
# Relative prefix used for storing offloaded training datasets in MinIO. # Relative prefix used for storing offloaded prediction datasets in MinIO.
# It is also the root directory for retention cleanup listing. # It is also the root directory for retention cleanup listing.
TRAINING_DATASETS_PREFIX = 'training_datasets' PREDICTION_DATASETS_PREFIX = 'prediction_datasets'
OperationKind = Literal['initial', 'transform', 'predict'] OperationKind = Literal['initial', 'transform', 'predict']
@@ -56,11 +56,11 @@ def _build_object_key(
tuple[str, str | None]: Full object key and normalized prefix (or None if at bucket root). tuple[str, str | None]: Full object key and normalized prefix (or None if at bucket root).
""" """
# Naming convention: # Naming convention:
# - Directory is always `training_datasets/<model_name>` # - Directory is always `prediction_datasets/<model_name>`
# - Filename follows the retention-parsing pattern # - Filename follows the retention-parsing pattern
basename = f'{model_name}-{operation}-{timestamp}.parquet' basename = f'{model_name}-{operation}-{timestamp}.parquet'
model_dir = model_name.strip().strip('/') model_dir = model_name.strip().strip('/')
prefix = f'{TRAINING_DATASETS_PREFIX}/{model_dir}' prefix = f'{PREDICTION_DATASETS_PREFIX}/{model_dir}'
return f'{prefix}/{basename}', prefix return f'{prefix}/{basename}', prefix