e6018af23f
Update README, requirements, and E2E tests for improved configuration and functionality - Enhanced the README with updated model configuration examples, including the addition of an alias for production. - Removed the `requirements-light.txt` file and updated `requirements-local.txt` and `requirements.txt` to replace `asyncua` with `opcua`. - Refactored E2E test scenarios to utilize scenario input files for better maintainability and clarity. - Improved test coverage for MinIO offload functionality and added new helper functions for loading scenario inputs. - Updated `values.yaml` to reflect new global configurations and environment variables for the laborious worker.
323 lines
11 KiB
Python
323 lines
11 KiB
Python
"""
|
|
End-to-end tests for the SimpleMetrics workflow.
|
|
|
|
Coverage focus:
|
|
|
|
- Happy path computes rmse/mse/mae/r2 from predictions joined against ``laborious_data``
|
|
and persists rows to ``predictions_schema.simple_metrics_data`` with all required columns.
|
|
- Subset metric selection (only rmse) writes exactly the requested rows.
|
|
- Zero-variance target produces ``r2=0`` per division-by-zero guard.
|
|
- Empty join (no overlapping data) short-circuits without persisting anything.
|
|
"""
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
from decimal import Decimal
|
|
|
|
import math
|
|
import pytest
|
|
from sqlalchemy import text
|
|
from temporalio.testing import WorkflowEnvironment
|
|
from temporalio.worker import Worker
|
|
|
|
from e2e.helpers import load_scenario_input, make_workflow_id, start_and_await_workflow
|
|
from laborious.activities.activities import Activities
|
|
from laborious.workflows.simple_metrics import SimpleMetrics
|
|
|
|
EXPECTED_SIMPLE_METRICS_COLUMNS = [
|
|
'id',
|
|
'model_id',
|
|
'metric',
|
|
'value',
|
|
'timestamp',
|
|
'data_size',
|
|
'interval_minutes',
|
|
'created_at',
|
|
]
|
|
|
|
|
|
def _simple_metrics_input(model_id: int, **overrides) -> dict:
|
|
"""Load and override the simple-metrics base scenario."""
|
|
payload = load_scenario_input('simple_metrics_base.json', model_id=model_id)
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
def _seed_predictions_and_targets(
|
|
postgres_engine,
|
|
model_id: int,
|
|
pairs: list[tuple[float, float]],
|
|
target_name: str = 'sensor_target',
|
|
offset_minutes: int = 6,
|
|
) -> list[str]:
|
|
"""
|
|
Insert matching prediction/target rows used by the SimpleMetrics SQL JOIN.
|
|
|
|
For each ``(prediction, target)`` pair we write a row in ``predictions`` and
|
|
a matching row in ``laborious_data`` with ``variable=target_name`` so the
|
|
inner join in the workflow query yields one row per pair.
|
|
|
|
Args:
|
|
- postgres_engine: SQLAlchemy engine bound to the test container.
|
|
- model_id: Model id stamped on every row.
|
|
- pairs: ``(prediction, target)`` pairs, one per minute.
|
|
- target_name: Variable name in ``laborious_data`` representing the target.
|
|
- offset_minutes: Earliest row sits this many minutes ago so timestamps fall
|
|
inside the workflow's recent-data window.
|
|
|
|
Return:
|
|
List of timestamp strings written for the inserted rows.
|
|
"""
|
|
base = datetime.now(timezone.utc).replace(second=0, microsecond=0) - timedelta(
|
|
minutes=offset_minutes
|
|
)
|
|
timestamps = [
|
|
(base + timedelta(minutes=i)).strftime('%Y-%m-%d %H:%M:%S%z')
|
|
for i in range(len(pairs))
|
|
]
|
|
|
|
prediction_rows = []
|
|
target_rows = []
|
|
for index, (prediction, target_value) in enumerate(pairs):
|
|
ts = timestamps[index]
|
|
prediction_rows.append(
|
|
f"({model_id}, {prediction}, 0, 0, 'Good', '{ts}', '{ts}')"
|
|
)
|
|
target_rows.append(
|
|
f"({model_id}, '{target_name}', {target_value}, '{ts}', '{ts}')"
|
|
)
|
|
|
|
with postgres_engine.begin() as conn:
|
|
conn.execute(text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}'))
|
|
conn.execute(text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}'))
|
|
# The SimpleMetrics SQL JOIN only filters ``predictions.model_id``; it does
|
|
# NOT filter ``laborious_data.model_id`` (see ``e2e/CODE_ISSUES.md`` issue
|
|
# SM-1). Without this cross-model cleanup, a previous test's target rows
|
|
# under the same variable name would join into this test's predictions
|
|
# whenever timestamps happened to overlap.
|
|
conn.execute(
|
|
text(
|
|
"DELETE FROM predictions_schema.laborious_data "
|
|
"WHERE variable IN (:sensor_default, :target_name) "
|
|
"AND timestamp >= NOW() - INTERVAL '120 minutes'"
|
|
),
|
|
{'sensor_default': 'sensor_target', 'target_name': target_name},
|
|
)
|
|
if prediction_rows:
|
|
conn.execute(
|
|
text(
|
|
'INSERT INTO predictions_schema.predictions '
|
|
'(model_id, prediction, prediction_confidence, response_time, '
|
|
'prediction_status, "timestamp", created_at) VALUES '
|
|
+ ', '.join(prediction_rows)
|
|
)
|
|
)
|
|
conn.execute(
|
|
text(
|
|
'INSERT INTO predictions_schema.laborious_data '
|
|
'(model_id, variable, value, "timestamp", created_at) VALUES '
|
|
+ ', '.join(target_rows)
|
|
)
|
|
)
|
|
return timestamps
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_simple_metrics_happy_path_persists_all_metrics_and_columns(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_simple_metrics: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
):
|
|
"""
|
|
Scenario S.1.1: rmse/mse/mae/r2 are calculated from a deterministic
|
|
prediction/target pair set and written one row per metric. Every column
|
|
expected by ``predictions_schema.simple_metrics_data`` must be populated and
|
|
the numerical values must match closed-form expectations.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 511
|
|
|
|
pairs = [
|
|
(1.0, 2.0),
|
|
(2.0, 4.0),
|
|
(3.0, 5.0),
|
|
(4.0, 9.0),
|
|
(5.0, 12.0),
|
|
]
|
|
diffs = [target - prediction for prediction, target in pairs]
|
|
n = len(diffs)
|
|
expected_rmse = math.sqrt(sum(d * d for d in diffs) / n)
|
|
expected_mse = sum(d * d for d in diffs) / n
|
|
expected_mae = sum(abs(d) for d in diffs) / n
|
|
target_mean = sum(t for _, t in pairs) / n
|
|
ss_res = sum((target - prediction) ** 2 for prediction, target in pairs)
|
|
ss_tot = sum((t - target_mean) ** 2 for _, t in pairs)
|
|
expected_r2 = 1.0 - (ss_res / ss_tot)
|
|
|
|
_seed_predictions_and_targets(postgres_engine, model_id=model_id, pairs=pairs)
|
|
|
|
input_data = _simple_metrics_input(model_id)
|
|
await start_and_await_workflow(
|
|
client,
|
|
SimpleMetrics.run,
|
|
input_data,
|
|
make_workflow_id('test-simple-metrics-happy'),
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
rows = (
|
|
conn.execute(
|
|
text(
|
|
'SELECT * FROM predictions_schema.simple_metrics_data '
|
|
'WHERE model_id = :m ORDER BY metric'
|
|
),
|
|
{'m': model_id},
|
|
)
|
|
.mappings()
|
|
.all()
|
|
)
|
|
|
|
assert len(rows) == 4, f'Expected 4 metric rows, got {len(rows)}'
|
|
for column in EXPECTED_SIMPLE_METRICS_COLUMNS:
|
|
assert column in rows[0], f'Missing simple_metrics column: {column}'
|
|
for row in rows:
|
|
for column in EXPECTED_SIMPLE_METRICS_COLUMNS:
|
|
assert row[column] is not None, f"Column '{column}' is NULL in {dict(row)}"
|
|
|
|
by_metric = {row['metric']: row for row in rows}
|
|
assert set(by_metric) == {'rmse', 'mse', 'mae', 'r2'}
|
|
|
|
def _decimal_close(actual, expected, places: int = 6) -> bool:
|
|
return abs(float(actual) - expected) < 10 ** (-places)
|
|
|
|
assert _decimal_close(by_metric['rmse']['value'], expected_rmse)
|
|
assert _decimal_close(by_metric['mse']['value'], expected_mse)
|
|
assert _decimal_close(by_metric['mae']['value'], expected_mae)
|
|
assert _decimal_close(by_metric['r2']['value'], expected_r2)
|
|
|
|
assert all(row['data_size'] == n for row in rows), 'data_size must equal target row count'
|
|
assert all(row['interval_minutes'] == 60 for row in rows)
|
|
assert all(row['model_id'] == model_id for row in rows)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_simple_metrics_subset_metrics_writes_only_requested_rows(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_simple_metrics: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
):
|
|
"""
|
|
Scenario S.1.2: Requesting ``metrics=['rmse']`` must persist exactly one row
|
|
with metric ``rmse`` and skip mse/mae/r2.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 512
|
|
|
|
pairs = [(1.0, 2.0), (2.0, 4.0), (3.0, 6.0)]
|
|
_seed_predictions_and_targets(postgres_engine, model_id=model_id, pairs=pairs)
|
|
|
|
input_data = _simple_metrics_input(model_id, metrics=['rmse'])
|
|
await start_and_await_workflow(
|
|
client,
|
|
SimpleMetrics.run,
|
|
input_data,
|
|
make_workflow_id('test-simple-metrics-subset'),
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
metrics = [
|
|
r[0]
|
|
for r in conn.execute(
|
|
text(
|
|
'SELECT metric FROM predictions_schema.simple_metrics_data '
|
|
'WHERE model_id = :m'
|
|
),
|
|
{'m': model_id},
|
|
).all()
|
|
]
|
|
assert metrics == ['rmse']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_simple_metrics_zero_variance_target_returns_zero_r2(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_simple_metrics: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
):
|
|
"""
|
|
Scenario S.2.1: When the target column has zero variance the activity must
|
|
return ``r2 = 0`` (division-by-zero guard) and still persist all four metrics.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 521
|
|
|
|
pairs = [(0.0, 5.0), (1.0, 5.0), (2.0, 5.0), (3.0, 5.0)]
|
|
_seed_predictions_and_targets(postgres_engine, model_id=model_id, pairs=pairs)
|
|
|
|
input_data = _simple_metrics_input(model_id)
|
|
await start_and_await_workflow(
|
|
client,
|
|
SimpleMetrics.run,
|
|
input_data,
|
|
make_workflow_id('test-simple-metrics-zero-variance'),
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
r2_value = conn.execute(
|
|
text(
|
|
"SELECT value FROM predictions_schema.simple_metrics_data "
|
|
"WHERE model_id = :m AND metric = 'r2'"
|
|
),
|
|
{'m': model_id},
|
|
).scalar()
|
|
assert r2_value is not None
|
|
assert Decimal(str(r2_value)) == Decimal('0'), f'expected r2=0, got {r2_value!r}'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_simple_metrics_no_overlapping_data_short_circuits(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_simple_metrics: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
):
|
|
"""
|
|
Scenario S.3.1: When the join produces no rows (no matching laborious_data
|
|
row for the configured ``target``), the workflow returns early without
|
|
invoking ``calculate_simple_metrics`` and writes nothing.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 531
|
|
|
|
# Insert predictions but no matching target rows for the configured variable.
|
|
_seed_predictions_and_targets(
|
|
postgres_engine,
|
|
model_id=model_id,
|
|
pairs=[(1.0, 1.0)],
|
|
target_name='wrong_variable_name',
|
|
)
|
|
|
|
input_data = _simple_metrics_input(model_id)
|
|
await start_and_await_workflow(
|
|
client,
|
|
SimpleMetrics.run,
|
|
input_data,
|
|
make_workflow_id('test-simple-metrics-empty-join'),
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
count = conn.execute(
|
|
text(
|
|
'SELECT COUNT(*) FROM predictions_schema.simple_metrics_data '
|
|
'WHERE model_id = :m'
|
|
),
|
|
{'m': model_id},
|
|
).scalar()
|
|
assert count == 0, 'Empty target data must short-circuit and skip persistence'
|