SIENTIAPDE-1712
Update MinioDataFramePayload to reflect changes from training to prediction datasets - Renamed TRAINING_DATASETS_PREFIX to PREDICTION_DATASETS_PREFIX for clarity. - Updated object key naming convention to use prediction datasets directory.
This commit is contained in:
@@ -34,9 +34,9 @@ OFFLOAD_THRESHOLD_BYTES = int(
|
|||||||
float(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES', '1.5')) * 1024 * 1024
|
float(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES', '1.5')) * 1024 * 1024
|
||||||
)
|
)
|
||||||
|
|
||||||
# Relative prefix used for storing offloaded training datasets in MinIO.
|
# Relative prefix used for storing offloaded prediction datasets in MinIO.
|
||||||
# It is also the root directory for retention cleanup listing.
|
# It is also the root directory for retention cleanup listing.
|
||||||
TRAINING_DATASETS_PREFIX = 'training_datasets'
|
PREDICTION_DATASETS_PREFIX = 'prediction_datasets'
|
||||||
|
|
||||||
OperationKind = Literal['initial', 'transform', 'predict']
|
OperationKind = Literal['initial', 'transform', 'predict']
|
||||||
|
|
||||||
@@ -56,11 +56,11 @@ def _build_object_key(
|
|||||||
tuple[str, str | None]: Full object key and normalized prefix (or None if at bucket root).
|
tuple[str, str | None]: Full object key and normalized prefix (or None if at bucket root).
|
||||||
"""
|
"""
|
||||||
# Naming convention:
|
# Naming convention:
|
||||||
# - Directory is always `training_datasets/<model_name>`
|
# - Directory is always `prediction_datasets/<model_name>`
|
||||||
# - Filename follows the retention-parsing pattern
|
# - Filename follows the retention-parsing pattern
|
||||||
basename = f'{model_name}-{operation}-{timestamp}.parquet'
|
basename = f'{model_name}-{operation}-{timestamp}.parquet'
|
||||||
model_dir = model_name.strip().strip('/')
|
model_dir = model_name.strip().strip('/')
|
||||||
prefix = f'{TRAINING_DATASETS_PREFIX}/{model_dir}'
|
prefix = f'{PREDICTION_DATASETS_PREFIX}/{model_dir}'
|
||||||
return f'{prefix}/{basename}', prefix
|
return f'{prefix}/{basename}', prefix
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user