e6018af23f
Update README, requirements, and E2E tests for improved configuration and functionality - Enhanced the README with updated model configuration examples, including the addition of an alias for production. - Removed the `requirements-light.txt` file and updated `requirements-local.txt` and `requirements.txt` to replace `asyncua` with `opcua`. - Refactored E2E test scenarios to utilize scenario input files for better maintainability and clarity. - Improved test coverage for MinIO offload functionality and added new helper functions for loading scenario inputs. - Updated `values.yaml` to reflect new global configurations and environment variables for the laborious worker.
109 lines
4.2 KiB
Python
109 lines
4.2 KiB
Python
"""
|
|
E2E-style tests for MinIO offload using a real MinIO testcontainer.
|
|
"""
|
|
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
from sqlalchemy import text
|
|
from temporalio.testing import WorkflowEnvironment
|
|
from temporalio.worker import Worker
|
|
|
|
from e2e.helpers import (
|
|
insert_sample_data,
|
|
load_scenario_input,
|
|
make_workflow_id,
|
|
start_and_await_workflow,
|
|
)
|
|
from laborious.activities.activities import Activities
|
|
from laborious.utils.models import minio_dataframe_payload as mdp
|
|
from laborious.workflows.predictions_batch import PredictionsBatch
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_load_query_with_minio_offload_writes_object_to_bucket(
|
|
postgres_engine,
|
|
minio_container,
|
|
test_activities_real_minio: Activities,
|
|
):
|
|
"""
|
|
With a tiny offload threshold, query results are uploaded as Parquet to MinIO.
|
|
|
|
Uses real MinioRepository against testcontainers MinIO (no MinIO mock).
|
|
"""
|
|
model_id = 501
|
|
with postgres_engine.begin() as conn:
|
|
conn.execute(text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}'))
|
|
insert_sample_data(postgres_engine, model_id, [1.0, 2.0])
|
|
|
|
scenario_input = load_scenario_input('minio_offload_load_query.json', model_id=model_id)
|
|
metadata = {'metadata': scenario_input['metadata']}
|
|
with patch.object(mdp, 'OFFLOAD_THRESHOLD_BYTES', 1):
|
|
payload = test_activities_real_minio.load_query_with_minio_offload(scenario_input)
|
|
assert payload.object_key, 'offloaded payload must reference a MinIO object'
|
|
assert payload.data is None or payload.data == {}, 'large payloads should not inline tabular dict'
|
|
|
|
df = payload.retrieve(test_activities_real_minio.minio_repository, metadata['metadata'])
|
|
assert len(df) >= 1
|
|
|
|
client = minio_container.get_client()
|
|
listed = list(client.list_objects('test-bucket', recursive=True))
|
|
names = [getattr(o, 'object_name', None) or getattr(o, '_object_name', '') for o in listed]
|
|
assert any(n and 'prediction_datasets' in n for n in names), f'unexpected object listing: {names!r}'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_predictions_batch_with_minio_offload_path(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_real_minio: Worker,
|
|
postgres_engine,
|
|
test_activities_real_minio: Activities,
|
|
):
|
|
"""
|
|
Full PredictionsBatch run with offload: load step stores Parquet in MinIO; pipeline completes.
|
|
"""
|
|
model_id = 502
|
|
with postgres_engine.begin() as conn:
|
|
conn.execute(text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}'))
|
|
conn.execute(text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}'))
|
|
conn.execute(text(f'DELETE FROM predictions_schema.transformed_data WHERE model_id = {model_id}'))
|
|
insert_sample_data(postgres_engine, model_id, [10.0, 20.0, 30.0])
|
|
|
|
input_data = load_scenario_input('minio_offload_workflow.json', model_id=model_id)
|
|
|
|
with patch.object(mdp, 'OFFLOAD_THRESHOLD_BYTES', 1):
|
|
await start_and_await_workflow(
|
|
temporal_test_env.client,
|
|
PredictionsBatch.run,
|
|
input_data,
|
|
make_workflow_id('test-batch-minio-offload'),
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
count = conn.execute(
|
|
text(f'SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = {model_id}')
|
|
).scalar()
|
|
assert count == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_load_query_with_inline_payload_when_below_threshold(
|
|
postgres_engine,
|
|
test_activities_real_minio: Activities,
|
|
):
|
|
"""Scenario 4.2.1: payload stays inline when threshold is high enough."""
|
|
model_id = 503
|
|
with postgres_engine.begin() as conn:
|
|
conn.execute(text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}'))
|
|
insert_sample_data(postgres_engine, model_id, [1.0, 2.0])
|
|
|
|
scenario_input = load_scenario_input('minio_offload_load_query.json', model_id=model_id)
|
|
with patch.object(mdp, 'OFFLOAD_THRESHOLD_BYTES', 10**9):
|
|
payload = test_activities_real_minio.load_query_with_minio_offload(scenario_input)
|
|
|
|
assert payload.object_key is None
|
|
assert payload.data is not None
|