SIENTIAPDE-1712
Update MinioDataFramePayload to reflect changes from training to prediction datasets - Renamed TRAINING_DATASETS_PREFIX to PREDICTION_DATASETS_PREFIX for clarity. - Updated object key naming convention to use prediction datasets directory.
This commit is contained in:
@@ -34,9 +34,9 @@ OFFLOAD_THRESHOLD_BYTES = int(
|
||||
float(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES', '1.5')) * 1024 * 1024
|
||||
)
|
||||
|
||||
# Relative prefix used for storing offloaded training datasets in MinIO.
|
||||
# Relative prefix used for storing offloaded prediction datasets in MinIO.
|
||||
# It is also the root directory for retention cleanup listing.
|
||||
TRAINING_DATASETS_PREFIX = 'training_datasets'
|
||||
PREDICTION_DATASETS_PREFIX = 'prediction_datasets'
|
||||
|
||||
OperationKind = Literal['initial', 'transform', 'predict']
|
||||
|
||||
@@ -56,11 +56,11 @@ def _build_object_key(
|
||||
tuple[str, str | None]: Full object key and normalized prefix (or None if at bucket root).
|
||||
"""
|
||||
# Naming convention:
|
||||
# - Directory is always `training_datasets/<model_name>`
|
||||
# - Directory is always `prediction_datasets/<model_name>`
|
||||
# - Filename follows the retention-parsing pattern
|
||||
basename = f'{model_name}-{operation}-{timestamp}.parquet'
|
||||
model_dir = model_name.strip().strip('/')
|
||||
prefix = f'{TRAINING_DATASETS_PREFIX}/{model_dir}'
|
||||
prefix = f'{PREDICTION_DATASETS_PREFIX}/{model_dir}'
|
||||
return f'{prefix}/{basename}', prefix
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user