SIENTIAPDE-1712

Refactor MinioDataFramePayload usage across activities

- Updated instances of MinioDataFramePayload initialization in Gates, MLFlow, and Storage classes to use the new from_dict method for better data reconstruction from dictionaries.
- Enhanced the PredictionProcess workflow to utilize the updated payload handling.
- Added passthrough fixtures in tests to accommodate the new from_dict method for consistent testing behavior.
This commit is contained in:
vitor-aignosi
2026-03-20 15:52:04 -03:00
parent 8789e6693f
commit 67942c45e0
12 changed files with 155 additions and 31 deletions

View File

@@ -214,3 +214,55 @@ async def test_from_dataframe_offloaded(mock_now):
assert result.bucket == 'test-bucket'
assert result.uri == 's3://test-bucket/full/key.parquet'
minio.upload_file.assert_awaited_once()
def test_from_dict_inline():
raw = {
'last_timestamp': '2024-01-01T00:00:00+00:00',
'status': None,
'data': {'col1': {0: 'val1'}},
'bucket': None,
'object_key': None,
'object_prefix': None,
'uri': None,
}
payload = MinioDataFramePayload.from_dict(raw)
assert isinstance(payload, MinioDataFramePayload)
assert payload.last_timestamp == '2024-01-01T00:00:00+00:00'
assert payload.data == {'col1': {0: 'val1'}}
assert payload.object_key is None
def test_from_dict_offloaded():
raw = {
'last_timestamp': '2024-06-15T10:30:45+00:00',
'status': {'success': True},
'data': None,
'bucket': 'my-bucket',
'object_key': 'training_datasets/model/model-initial-2024-06-15_10-30-45.parquet',
'object_prefix': 'training_datasets/model',
'uri': 's3://my-bucket/training_datasets/model/model-initial-2024-06-15_10-30-45.parquet',
}
payload = MinioDataFramePayload.from_dict(raw)
assert isinstance(payload, MinioDataFramePayload)
assert payload.data is None
assert payload.bucket == 'my-bucket'
assert payload.object_key == raw['object_key']
assert payload.object_prefix == 'training_datasets/model'
assert payload.uri == raw['uri']
assert payload.status == {'success': True}
def test_from_dict_minimal_keys():
raw = {'last_timestamp': '2024-01-01'}
payload = MinioDataFramePayload.from_dict(raw)
assert payload.last_timestamp == '2024-01-01'
assert payload.data is None
assert payload.bucket is None
assert payload.object_key is None
def test_from_dict_passthrough_existing_instance():
original = MinioDataFramePayload(last_timestamp='2024-01-01', data={'a': 1}, bucket='b')
result = MinioDataFramePayload.from_dict(original)
assert result is original