SIENTIAPDE-1712
Refactor MinioDataFramePayload usage across activities - Updated instances of MinioDataFramePayload initialization in Gates, MLFlow, and Storage classes to use the new from_dict method for better data reconstruction from dictionaries. - Enhanced the PredictionProcess workflow to utilize the updated payload handling. - Added passthrough fixtures in tests to accommodate the new from_dict method for consistent testing behavior.
This commit is contained in:
@@ -214,3 +214,55 @@ async def test_from_dataframe_offloaded(mock_now):
|
||||
assert result.bucket == 'test-bucket'
|
||||
assert result.uri == 's3://test-bucket/full/key.parquet'
|
||||
minio.upload_file.assert_awaited_once()
|
||||
|
||||
|
||||
def test_from_dict_inline():
|
||||
raw = {
|
||||
'last_timestamp': '2024-01-01T00:00:00+00:00',
|
||||
'status': None,
|
||||
'data': {'col1': {0: 'val1'}},
|
||||
'bucket': None,
|
||||
'object_key': None,
|
||||
'object_prefix': None,
|
||||
'uri': None,
|
||||
}
|
||||
payload = MinioDataFramePayload.from_dict(raw)
|
||||
assert isinstance(payload, MinioDataFramePayload)
|
||||
assert payload.last_timestamp == '2024-01-01T00:00:00+00:00'
|
||||
assert payload.data == {'col1': {0: 'val1'}}
|
||||
assert payload.object_key is None
|
||||
|
||||
|
||||
def test_from_dict_offloaded():
|
||||
raw = {
|
||||
'last_timestamp': '2024-06-15T10:30:45+00:00',
|
||||
'status': {'success': True},
|
||||
'data': None,
|
||||
'bucket': 'my-bucket',
|
||||
'object_key': 'training_datasets/model/model-initial-2024-06-15_10-30-45.parquet',
|
||||
'object_prefix': 'training_datasets/model',
|
||||
'uri': 's3://my-bucket/training_datasets/model/model-initial-2024-06-15_10-30-45.parquet',
|
||||
}
|
||||
payload = MinioDataFramePayload.from_dict(raw)
|
||||
assert isinstance(payload, MinioDataFramePayload)
|
||||
assert payload.data is None
|
||||
assert payload.bucket == 'my-bucket'
|
||||
assert payload.object_key == raw['object_key']
|
||||
assert payload.object_prefix == 'training_datasets/model'
|
||||
assert payload.uri == raw['uri']
|
||||
assert payload.status == {'success': True}
|
||||
|
||||
|
||||
def test_from_dict_minimal_keys():
|
||||
raw = {'last_timestamp': '2024-01-01'}
|
||||
payload = MinioDataFramePayload.from_dict(raw)
|
||||
assert payload.last_timestamp == '2024-01-01'
|
||||
assert payload.data is None
|
||||
assert payload.bucket is None
|
||||
assert payload.object_key is None
|
||||
|
||||
|
||||
def test_from_dict_passthrough_existing_instance():
|
||||
original = MinioDataFramePayload(last_timestamp='2024-01-01', data={'a': 1}, bucket='b')
|
||||
result = MinioDataFramePayload.from_dict(original)
|
||||
assert result is original
|
||||
|
||||
Reference in New Issue
Block a user