SIENTIAPDE-1646
Update README, requirements, and E2E tests for improved configuration and functionality - Enhanced the README with updated model configuration examples, including the addition of an alias for production. - Removed the `requirements-light.txt` file and updated `requirements-local.txt` and `requirements.txt` to replace `asyncua` with `opcua`. - Refactored E2E test scenarios to utilize scenario input files for better maintainability and clarity. - Improved test coverage for MinIO offload functionality and added new helper functions for loading scenario inputs. - Updated `values.yaml` to reflect new global configurations and environment variables for the laborious worker.
This commit is contained in:
97
README.md
97
README.md
@@ -463,7 +463,7 @@ flowchart LR
|
||||
"source_table_name": "laborious_data",
|
||||
"target_table_name": "drift_metrics",
|
||||
"interval": 60,
|
||||
"model_config": { "target": "temperature" },
|
||||
"model_config": { "target": "temperature", "alias": "production" },
|
||||
"drift_metrics": ["kolmogorov_smirnov", "jensen_shannon", "wasserstein"],
|
||||
"chunk_period": "min"
|
||||
}
|
||||
@@ -498,7 +498,7 @@ flowchart LR
|
||||
"data_table_name": "laborious_data",
|
||||
"target_table_name": "simple_metrics",
|
||||
"interval_minutes": 60,
|
||||
"model_config": { "target": "temperature" },
|
||||
"model_config": { "target": "temperature", "alias": "production" },
|
||||
"metrics": ["rmse", "mse", "mae", "r2"]
|
||||
}
|
||||
```
|
||||
@@ -953,7 +953,7 @@ For single OPC server, use individual environment variables:
|
||||
|
||||
### PI Web API Configuration
|
||||
|
||||
PI Web API configuration is built from environment variables using the `build_api_config` function from `sientia_do.connectors_config`. The configuration includes:
|
||||
PI Web API configuration is built from environment variables using the `build_api_config` function from `sientia_do.utils.connectors_config`. The configuration includes:
|
||||
|
||||
- `PI_WEB_API_BASE_URL`: Base URL of the PI Web API server
|
||||
- `PI_WEB_API_AUTH_TYPE`: Authentication type ('basic' or 'bearer')
|
||||
@@ -986,9 +986,30 @@ Where:
|
||||
|
||||
MongoDB pipeline configuration:
|
||||
|
||||
#### Predictions Batch Workflow configuration sample
|
||||
#### MongoDB input samples (updated)
|
||||
|
||||
This is the configuration for the Predictions Batch Workflow, to be inserted into the MongoDB pipeline collection.
|
||||
Updated examples are available in `input_sample.json` at the repository root.
|
||||
The sample already reflects the runtime-aware and alias-based flow:
|
||||
|
||||
- `model_config` uses `target`, `retention_minutes`, and `alias`.
|
||||
- `transform_flavor` / `predict_flavor` are not used anymore.
|
||||
|
||||
Example model document:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "4",
|
||||
"name": "vcm-nox",
|
||||
"active": false,
|
||||
"model_config": {
|
||||
"alias": "production",
|
||||
"retention_minutes": 60,
|
||||
"target": "CI-W3W01A3"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Example predictions_batch schedule document:
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -998,41 +1019,67 @@ This is the configuration for the Predictions Batch Workflow, to be inserted int
|
||||
"frequency": "30s",
|
||||
"max_retry_policy": 1,
|
||||
"query": "select * from sientia_data.laborious_data where model_id = 1 and \"timestamp\" > NOW() - INTERVAL '5 minutes' order by \"timestamp\" desc limit 30;",
|
||||
"retention_time": 60,
|
||||
"write_tags": [
|
||||
{
|
||||
"server_id": "server1",
|
||||
"server_id": "1",
|
||||
"type": "prediction",
|
||||
"addr": "ns=2;i=5",
|
||||
"data_type": "double"
|
||||
},
|
||||
{
|
||||
"server_id": "server1",
|
||||
"server_id": "1",
|
||||
"type": "confidence",
|
||||
"addr": "ns=2;i=6",
|
||||
"addr": "ns=2;i=5",
|
||||
"data_type": "double"
|
||||
}
|
||||
],
|
||||
"input_filters": {
|
||||
"EMPTY_DATA": {"POLICY": "STOP"},
|
||||
"SPECIFIC_VARIABLES_NULL_VALUES": {
|
||||
"POLICY": "CONTINUE",
|
||||
"config": {"variables": ["Counter"]}
|
||||
"input_filters": [
|
||||
{
|
||||
"filter_name": "EMPTY_DATA",
|
||||
"policy": "STOP"
|
||||
},
|
||||
{
|
||||
"filter_name": "SPECIFIC_VARIABLES_NULL_VALUES",
|
||||
"policy": "CONTINUE",
|
||||
"config": {
|
||||
"variables": ["Counter"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"mlflow_transform_filters": {
|
||||
"API_ERROR": {"POLICY": "REPEAT"},
|
||||
"NAN_VALUES": {"POLICY": "STOP"}
|
||||
},
|
||||
"mlflow_predict_filters": {
|
||||
"API_ERROR": {"POLICY": "CONTINUE"}
|
||||
},
|
||||
],
|
||||
"mlflow_transform_filters": [
|
||||
{
|
||||
"filter_name": "API_ERROR",
|
||||
"policy": "REPEAT"
|
||||
},
|
||||
{
|
||||
"filter_name": "NAN_VALUES",
|
||||
"policy": "STOP"
|
||||
}
|
||||
],
|
||||
"mlflow_predict_filters": [
|
||||
{
|
||||
"filter_name": "API_ERROR",
|
||||
"policy": "CONTINUE"
|
||||
}
|
||||
],
|
||||
"path_priority": ["STOP", "CONTINUE", "REPEAT"],
|
||||
"active": true,
|
||||
"updated_at": {
|
||||
"$date": "2025-09-16T10:00:00.000Z"
|
||||
"$date": "2026-01-27T17:35:01.600Z"
|
||||
},
|
||||
"datetime_columns": ["timestamp", "created_at"],
|
||||
"predictions_storage_policy": "lts:1"
|
||||
"save_transform": false,
|
||||
"pi_web_api_output_config": {
|
||||
"endpoint": "/streamsets/value",
|
||||
"prediction_tags": {},
|
||||
"confidence_tags": {}
|
||||
},
|
||||
"model_config": {
|
||||
"alias": "production",
|
||||
"retention_minutes": 60,
|
||||
"target": "CI-W3W01A3"
|
||||
},
|
||||
"datetime_columns": ["timestamp", "created_at"]
|
||||
}
|
||||
```
|
||||
|
||||
@@ -1125,7 +1172,7 @@ laborious/
|
||||
2. **MLFlow Connection Issues**
|
||||
- Verify MLFlow server is running and accessible
|
||||
- Check authentication credentials and permissions
|
||||
- Ensure model names and versions exist
|
||||
- Ensure model names exist and the expected alias (for example `production`) is registered
|
||||
|
||||
3. **Database Connection Issues**
|
||||
- Verify PostgreSQL service is running
|
||||
|
||||
Reference in New Issue
Block a user