SIENTIAPDE-1646
SIENTIAPDE-1646 Merge origin/main into release/SIENTIAPDE-1646 Bring main up to date while keeping this branch's sync execution, PluginStore runtime and sientia_model (SientiaMLflowRepository/SientiaModel) structure. Resolution decisions: - OPC: keep the sync asyncua.sync implementation; port main's server_uri validation, e2e race fix (lock held until released) and the new unit tests converted to sync. - Model import workflow (.sientia bundles), joblib/transform flavors and the local MLFlowRepository (model_repository.py) are dropped, with their tests, docs, metrics, env vars and connectors config. - Worker: drift/simple_metrics stay pinned to the 'core' runtime; port main's HOSTNAME pod id, logger-based prometheus startup and sys.exit after finally. - values.yaml removed, following main (SIENTIAPDE-1945). - Deps: keep sientia_model; take main's fsspec/s3fs/botocore/boto3 pins and sientia_do>=1.12.2; drop import-only deps (pynacl, cryptography, pymongo, python-dotenv, pycaret, joblib, gcsfs). - e2e: keep sientia_data schema and scenario input files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -48,4 +48,4 @@ MINIO_ENDPOINT_URL="http://localhost:9000"
|
||||
MINIO_ACCESS_KEY="sientia"
|
||||
MINIO_SECRET_KEY="sientia"
|
||||
MINIO_REGION_NAME="sa-east-1"
|
||||
MINIO_DEFAULT_BUCKET="sientia"
|
||||
MINIO_DEFAULT_BUCKET="sientia"
|
||||
|
||||
35
.github/workflows/deploy.yml
vendored
Normal file
35
.github/workflows/deploy.yml
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
name: Deploy DataOps Worker
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [main, 'release/**', 'feature/**']
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
name: Deploy
|
||||
if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true
|
||||
uses: Aignosi/github_workflow_templates/.github/workflows/reusable-deploy-dataops.yml@main
|
||||
with:
|
||||
branch_name: ${{ github.event.pull_request.head.ref }}
|
||||
project_name: 'laborious'
|
||||
chart_name: 'laborious-worker'
|
||||
# Only the "legacy" runtime, for now (card 13 correction) — laborious-worker has two
|
||||
# Deployments (basic, legacy). basic's GITHUB_BRANCH lives in the chart base (.global.env)
|
||||
# and is deliberately left untouched here. legacy's own override exists ONLY at the profile
|
||||
# level — profiles/gcp.yaml's laborious-worker.runtimes.legacy.env, not the chart base at
|
||||
# all — hence profile/profile_env_list_path below instead of env_list_path.
|
||||
profile: 'gcp'
|
||||
profile_env_list_path: '[".runtimes.legacy.env"]'
|
||||
update_version_in: '["pyproject"]'
|
||||
create_tarball: ${{ github.event_name == 'pull_request' }}
|
||||
mirror: true
|
||||
bump_chart: true
|
||||
deploy: true
|
||||
secrets: inherit
|
||||
18
.github/workflows/quality-gate.yml
vendored
18
.github/workflows/quality-gate.yml
vendored
@@ -1,15 +1,29 @@
|
||||
name: Quality gate
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- 'release/**'
|
||||
- 'feature/**'
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- 'release/**'
|
||||
- 'feature/**'
|
||||
types: [ opened, synchronize, reopened ]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
quality-gate:
|
||||
uses: Aignosi/github_workflow_templates/.github/workflows/dataops-module-quality-gate.yml@main
|
||||
permissions: write-all
|
||||
uses: Aignosi/github_workflow_templates/.github/workflows/python-quality-gate.yml@main
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
with:
|
||||
project_name: 'laborious'
|
||||
repositories: 'sientia-dataops-library, sientia-mlops-library'
|
||||
|
||||
25
.github/workflows/release.yml
vendored
25
.github/workflows/release.yml
vendored
@@ -1,25 +0,0 @@
|
||||
name: Create Release on Merge to Main
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: 'Version to release'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
release:
|
||||
if: |
|
||||
(github.event_name == 'pull_request' && github.event.pull_request.merged == true) ||
|
||||
github.event_name == 'workflow_dispatch'
|
||||
uses: Aignosi/github_workflow_templates/.github/workflows/dataops-module-release.yml@main
|
||||
permissions: write-all
|
||||
with:
|
||||
project_name: 'laborious'
|
||||
release_version: ${{ github.event.inputs.version || '' }}
|
||||
secrets: inherit
|
||||
16
.github/workflows/require-chart-ref-label.yml
vendored
Normal file
16
.github/workflows/require-chart-ref-label.yml
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
name: Require chart-ref label
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened, labeled, unlabeled]
|
||||
branches:
|
||||
- main
|
||||
- 'release/**'
|
||||
- 'feature/**'
|
||||
|
||||
permissions:
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
require-chart-ref-label:
|
||||
uses: Aignosi/github_workflow_templates/.github/workflows/require-chart-ref-label.yml@main
|
||||
6
.gitignore
vendored
6
.gitignore
vendored
@@ -54,6 +54,6 @@ catboost_info/
|
||||
mlruns/
|
||||
|
||||
relatorio*
|
||||
openspec/*
|
||||
.cursor/*
|
||||
openspec/
|
||||
openspec/
|
||||
.cursor/
|
||||
.claude/
|
||||
|
||||
7
Makefile
7
Makefile
@@ -1,7 +0,0 @@
|
||||
VERSION = 1.0.8
|
||||
name = sientia-laborious
|
||||
# ENVIRONMENT = production
|
||||
|
||||
docker-hub:
|
||||
@docker build --no-cache -t aignosi.azurecr.io/$(name):$(VERSION) .
|
||||
@docker push aignosi.azurecr.io/$(name):$(VERSION)
|
||||
79
README.md
79
README.md
@@ -43,6 +43,7 @@ A comprehensive, Temporal-based ML orchestration system for industrial data proc
|
||||
- [Testing](#testing)
|
||||
- [Test Structure](#test-structure)
|
||||
- [Test Execution](#test-execution)
|
||||
- [End-to-End Tests](#end-to-end-tests-e2e)
|
||||
- [Monitoring and Metrics](#monitoring-and-metrics)
|
||||
- [Application Health Metrics](#application-health-metrics)
|
||||
- [Prediction Operation Metrics](#prediction-operation-metrics)
|
||||
@@ -126,10 +127,20 @@ Laborious uses a Temporal-based architecture with strong separation of concerns
|
||||
### Key Components
|
||||
|
||||
#### **Worker (`laborious/worker/worker.py`)**
|
||||
- Temporal client setup, worker lifecycle, task queues
|
||||
- Temporal client setup, four workers via `sientia_do.temporal.worker.prepare_worker`
|
||||
- `RUNTIME` is required: installs the PluginStore runtime and suffixes the model-loading queues
|
||||
- Metrics server initialization, notification handler setup
|
||||
- Graceful shutdown and autoscaling-friendly behavior
|
||||
|
||||
Task queues, one per worker:
|
||||
|
||||
| Queue | Worker |
|
||||
|---|---|
|
||||
| `predictions_batch-{RUNTIME}-queue` | PredictionsBatch (+ its two sub-workflows) |
|
||||
| `minimal_retrain-{RUNTIME}-queue` | MinimalRetrain |
|
||||
| `drift-core-queue` | Drift (pinned to `core`: no model code loaded) |
|
||||
| `simple_metrics-core-queue` | SimpleMetrics (pinned to `core`: no model code loaded) |
|
||||
|
||||
#### **Workflows (`laborious/workflows/`)**
|
||||
- `predictions_batch.py`: Batch prediction entry point
|
||||
- `sub_workflows/prediction_process.py`: Core prediction pipeline
|
||||
@@ -584,10 +595,10 @@ Use the provided script to run the application locally:
|
||||
|
||||
```bash
|
||||
# Make script executable (first time only)
|
||||
chmod +x run_local.sh
|
||||
chmod +x scripts/run_local.sh
|
||||
|
||||
# Run the application
|
||||
./run_local.sh
|
||||
# Run the application (from the repository root)
|
||||
./scripts/run_local.sh
|
||||
```
|
||||
|
||||
The script will:
|
||||
@@ -601,10 +612,10 @@ Use the provided script to run tests with coverage:
|
||||
|
||||
```bash
|
||||
# Make script executable (first time only)
|
||||
chmod +x run_coverage.sh
|
||||
chmod +x scripts/run_coverage.sh
|
||||
|
||||
# Run tests with coverage
|
||||
./run_coverage.sh
|
||||
# Run tests with coverage (from the repository root)
|
||||
./scripts/run_coverage.sh
|
||||
```
|
||||
|
||||
The script will:
|
||||
@@ -755,6 +766,53 @@ pytest tests/laborious/activities/test_gates.py
|
||||
pytest tests/laborious/workflows/test_predictions_batch.py
|
||||
```
|
||||
|
||||
### End-to-End Tests (`e2e/`)
|
||||
|
||||
The `e2e/` suite runs against real dependencies in Docker containers via
|
||||
[testcontainers](https://testcontainers.com/): PostgreSQL and MinIO. It lives outside `testpaths`,
|
||||
so it is never collected by the unit gate.
|
||||
|
||||
**One-time setup, required before running anything under `e2e/`:**
|
||||
|
||||
```bash
|
||||
echo "ryuk.disabled=true" >> ~/.testcontainers.properties
|
||||
```
|
||||
|
||||
Without it, **every** e2e test fails at setup with a Docker mount error and never reaches an
|
||||
assertion:
|
||||
|
||||
```
|
||||
docker.errors.APIError: 500 Server Error ... /containers/<id>/start:
|
||||
Mounts denied: The path /socket_mnt/home/<user>/.docker/desktop/docker.sock
|
||||
is not shared from the host and is not known to Docker.
|
||||
```
|
||||
|
||||
The cause is not the tests. Before starting any test container, testcontainers starts a reaper
|
||||
container ("Ryuk") that bind-mounts the Docker socket so it can clean up if the test process dies
|
||||
without teardown. Under Docker Desktop on Linux that socket is `~/.docker/desktop/docker.sock`, a path
|
||||
Docker Desktop's File Sharing does not expose, so the reaper's own container cannot start. Note the
|
||||
symptom is `ERROR`, not `FAILED` — the failure is in fixture setup.
|
||||
|
||||
Disabling it costs nothing in a normal run: every fixture stops its own containers in teardown. Ryuk
|
||||
only matters when the process is killed outright (`kill -9`, a crash, a hard stop from the IDE). After
|
||||
one of those, clean up with:
|
||||
|
||||
```bash
|
||||
docker ps -a --filter "label=org.testcontainers=true" -q | xargs -r docker rm -f
|
||||
```
|
||||
|
||||
**Running it:**
|
||||
|
||||
```bash
|
||||
source ./venv/bin/activate
|
||||
|
||||
# the predictions and OPC suites
|
||||
pytest e2e/ --override-ini testpaths=e2e -m integration
|
||||
```
|
||||
|
||||
Full scenario catalogue:
|
||||
[`e2e/scenarios.md`](e2e/scenarios.md).
|
||||
|
||||
## 📊 Monitoring and Metrics
|
||||
|
||||
The Laborious system exposes comprehensive Prometheus metrics for operational visibility and performance monitoring:
|
||||
@@ -801,6 +859,7 @@ See [OPC UA Communication](#opc-ua-communication) for semantics, concurrency, an
|
||||
|----------|-------------|---------|----------|
|
||||
| `TEMPORAL_HOST` | Temporal server address | `localhost:7233` | Yes |
|
||||
| `TEMPORAL_NAMESPACE` | Temporal namespace | `laborious` | No |
|
||||
| `RUNTIME` | Task queue suffix for all workflows (`{workflow}-{RUNTIME}-queue`) | _(none)_ | Yes |
|
||||
| `POSTGRES_HOST` | PostgreSQL hostname | `localhost` | Yes |
|
||||
| `POSTGRES_PORT` | PostgreSQL port | `5432` | Yes |
|
||||
| `POSTGRES_USER` | PostgreSQL username | `sientia` | Yes |
|
||||
@@ -845,7 +904,7 @@ See [OPC UA Communication](#opc-ua-communication) for semantics, concurrency, an
|
||||
| `PROJECT_NAME` | Project name for metrics | `laborious` | No |
|
||||
| `HTTP_METRICS_PORT` | Prometheus metrics port | `9090` | No |
|
||||
| `HTTP_SDK_METRICS_PORT` | Temporal SDK metrics port | `9091` | No |
|
||||
| `POD_ID` | Kubernetes pod identifier | `None` | No |
|
||||
| `HOSTNAME` | Pod identifier (set by Kubernetes) used in metrics labels | `None` | No |
|
||||
| `MINIO_ENDPOINT_URL` | MinIO endpoint URL | `http://localhost:9000` | Yes |
|
||||
| `MINIO_ACCESS_KEY` | MinIO access key | `minioadmin` | Yes |
|
||||
| `MINIO_SECRET_KEY` | MinIO secret key | `minioadmin` | Yes |
|
||||
@@ -1233,7 +1292,9 @@ export LOG_LEVEL=DEBUG
|
||||
### Scaling Considerations
|
||||
|
||||
- **Horizontal Scaling**: Deploy multiple worker instances
|
||||
- **Task Queue Distribution**: Use multiple task queues for different workflow types
|
||||
- **Task Queue Distribution**: One worker pod per `RUNTIME`; queues are
|
||||
`predictions_batch-{RUNTIME}-queue`, `minimal_retrain-{RUNTIME}-queue`,
|
||||
`drift-core-queue`, `simple_metrics-core-queue`
|
||||
- **Database Performance**: Optimize indexes and connection pooling
|
||||
- **MLFlow Performance**: Configure appropriate model serving resources
|
||||
|
||||
|
||||
@@ -147,8 +147,11 @@ class OpcE2ETestServer:
|
||||
value = await self._confidence_node.read_value()
|
||||
return float(value)
|
||||
|
||||
async def _pre_write_callback(self, _event, _service) -> None:
|
||||
if self._session_bad_on_write:
|
||||
async def _pre_write_callback(self, event, _service) -> None:
|
||||
# Only client writes are faulted. asyncua's own clock task writes CurrentTime
|
||||
# through this same PreWrite dispatch, and raising there kills its time_task;
|
||||
# the stored exception then resurfaces as a teardown failure in server.stop().
|
||||
if self._session_bad_on_write and getattr(event, 'is_external', False):
|
||||
raise ServiceError(ua.StatusCodes.BadSessionIdInvalid)
|
||||
|
||||
|
||||
|
||||
@@ -7,10 +7,51 @@ It is a functional reference of scenario behavior, inputs, and expected outcomes
|
||||
|
||||
## Execution Context
|
||||
|
||||
### One-time setup: disable the testcontainers reaper
|
||||
|
||||
Do this once per machine, before running anything under `e2e/`:
|
||||
|
||||
```bash
|
||||
echo "ryuk.disabled=true" >> ~/.testcontainers.properties
|
||||
```
|
||||
|
||||
**Why.** Before starting any test container, testcontainers starts a reaper container ("Ryuk") whose
|
||||
job is to clean up containers if the test process dies without running its teardown. Ryuk bind-mounts
|
||||
the Docker socket. Under **Docker Desktop on Linux** the socket lives at
|
||||
`~/.docker/desktop/docker.sock`, which Docker Desktop's File Sharing does not expose, so the mount is
|
||||
denied and the reaper cannot start:
|
||||
|
||||
```
|
||||
docker.errors.APIError: 500 Server Error ... /containers/<id>/start:
|
||||
Mounts denied: The path /socket_mnt/home/<user>/.docker/desktop/docker.sock
|
||||
is not shared from the host and is not known to Docker.
|
||||
```
|
||||
|
||||
Every test then errors **at setup** — `ERROR`, not `FAILED`, and never inside an assertion. If you see
|
||||
that, this is why.
|
||||
|
||||
**What you give up.** Nothing in a normal run: every fixture in `conftest.py` stops its own containers
|
||||
in teardown. Ryuk only covers the case where the process is killed without teardown (`kill -9`, a
|
||||
crash, a hard stop from the IDE). After one of those, clean up by hand:
|
||||
|
||||
```bash
|
||||
docker ps -a --filter "label=org.testcontainers=true" -q | xargs -r docker rm -f
|
||||
```
|
||||
|
||||
*Alternative, if you want to keep the reaper:* point Docker at the classic socket with
|
||||
`export DOCKER_HOST=unix:///var/run/docker.sock`. That only works if a native Docker Engine is
|
||||
running alongside Docker Desktop; if the Desktop daemon is the only one, disabling Ryuk is the way.
|
||||
|
||||
|
||||
- Tests run under `e2e/` and are marked with `@pytest.mark.integration`.
|
||||
- PostgreSQL and MinIO are provisioned with testcontainers.
|
||||
- `test_minio_offload.py` uses real MinIO I/O; other scenario suites may use stubs/mocks for optional outputs.
|
||||
- Real OPC UA scenarios use `@pytest.mark.opc` and an in-process asyncua server (`e2e/test_opc_real_server.py`).
|
||||
- **Nothing under `e2e/` is covered by `validate`**: that script lints `laborious/` and `tests/`
|
||||
only, and its test step is `pytest tests/` (`testpaths = ["tests"]`). Lint these files by hand
|
||||
(`ruff format --check e2e/ && ruff check e2e/`), and remember that `addopts` carries
|
||||
`--strict-markers`: a test without `@pytest.mark.integration` is not collected, and a run that
|
||||
collects nothing still exits 0. Check the collected count, not the exit code.
|
||||
|
||||
### Local validation
|
||||
|
||||
|
||||
@@ -7,13 +7,17 @@ Mock-based OPC tests remain in test_predictions_batch_format_export.py.
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from temporalio.testing import WorkflowEnvironment
|
||||
from temporalio.worker import Worker
|
||||
|
||||
from e2e.helpers import assert_prediction, insert_sample_data, make_workflow_id, start_and_await_workflow
|
||||
from e2e.helpers import (
|
||||
assert_prediction,
|
||||
insert_sample_data,
|
||||
make_workflow_id,
|
||||
start_and_await_workflow,
|
||||
)
|
||||
from e2e.opc_test_server import UNKNOWN_NODE_ID, OpcE2ETestServer, build_opc_output_config
|
||||
from e2e.test_predictions_batch_format_export import get_base_input_data
|
||||
from laborious.activities.activities import Activities
|
||||
@@ -22,16 +26,24 @@ from laborious.utils.repository.opc_repository import OpcRepository
|
||||
from laborious.workflows.predictions_batch import PredictionsBatch
|
||||
|
||||
|
||||
def _slow_reconnect_under_lock(repo: OpcRepository, hold_seconds: float = 0.75) -> None:
|
||||
def _reconnect_under_held_lock(
|
||||
repo: OpcRepository, acquired: threading.Event, release: threading.Event
|
||||
) -> None:
|
||||
"""
|
||||
Hold the connection lock briefly so concurrent writes see reconnect_in_progress.
|
||||
Hold the connection lock until released, then reconnect.
|
||||
|
||||
The lock must stay held for the whole workflow, and how long the workflow takes to
|
||||
reach the OPC write is not bounded — a fixed sleep here lets `_reconnect_locked`
|
||||
finish first, which sets `_session_ready` again and the write then succeeds.
|
||||
|
||||
Args:
|
||||
repo (OpcRepository): Connected repository.
|
||||
hold_seconds (float): Time to keep the lock before reconnecting.
|
||||
acquired (threading.Event): Set once the lock is held, before any write can race.
|
||||
release (threading.Event): Waited on while holding the lock; set by the caller.
|
||||
"""
|
||||
with repo._connection_lock:
|
||||
time.sleep(hold_seconds)
|
||||
acquired.set()
|
||||
release.wait()
|
||||
repo._reconnect_locked()
|
||||
|
||||
|
||||
@@ -172,12 +184,15 @@ async def test_scenario_3_2_5_opc_write_blocked_during_reconnect_real_server(
|
||||
|
||||
repo = test_activities_real_opc.opc_repository['1']
|
||||
repo._session_ready.clear()
|
||||
acquired = threading.Event()
|
||||
release = threading.Event()
|
||||
reconnect_thread = threading.Thread(
|
||||
target=_slow_reconnect_under_lock,
|
||||
args=(repo,),
|
||||
target=_reconnect_under_held_lock,
|
||||
args=(repo, acquired, release),
|
||||
daemon=True,
|
||||
)
|
||||
reconnect_thread.start()
|
||||
acquired.wait(timeout=5.0)
|
||||
|
||||
input_data = get_base_input_data(model_id)
|
||||
input_data['opc_output_config'] = build_opc_output_config(opc_e2e_server.node_ids)
|
||||
@@ -191,6 +206,7 @@ async def test_scenario_3_2_5_opc_write_blocked_during_reconnect_real_server(
|
||||
make_workflow_id('test-opc-real-reconnect-block'),
|
||||
)
|
||||
finally:
|
||||
release.set()
|
||||
reconnect_thread.join(timeout=5.0)
|
||||
|
||||
assert_prediction(
|
||||
|
||||
13
encode.sh
13
encode.sh
@@ -1,13 +0,0 @@
|
||||
source ./venv/bin/activate
|
||||
|
||||
pip install pathspec
|
||||
pip install pyyaml
|
||||
|
||||
echo "
|
||||
.git" >> .gitignore
|
||||
|
||||
python encrypt.py ./ code --ignore .gitignore --chunk-size 100000
|
||||
|
||||
sed -i '/.git/d' .gitignore
|
||||
|
||||
xdg-open .
|
||||
113
encrypt.py
113
encrypt.py
@@ -1,113 +0,0 @@
|
||||
import os
|
||||
import argparse
|
||||
from pathspec import PathSpec
|
||||
import yaml # type: ignore
|
||||
from typing import Any
|
||||
|
||||
'''
|
||||
Usage:
|
||||
python .\encrypt.py path_to_dir output_file --ignore ignore_file --chunk-size 100000
|
||||
'''
|
||||
|
||||
|
||||
def load_ignore_patterns(ignore_file, include_library):
|
||||
# Ensure the .gitignore file exists
|
||||
if not os.path.exists(ignore_file):
|
||||
raise FileNotFoundError(f"Ignore file not found at {ignore_file}")
|
||||
|
||||
# Load and parse the .gitignore patterns
|
||||
with open(ignore_file, 'r') as file:
|
||||
patterns = file.readlines()
|
||||
if not include_library:
|
||||
patterns.append('**/deploy/library/')
|
||||
|
||||
spec = PathSpec.from_lines('gitwildmatch', patterns)
|
||||
return spec
|
||||
|
||||
|
||||
def is_ignored(file_path, spec):
|
||||
"""Check if a file should be ignored based on the ignore patterns."""
|
||||
return spec.match_file(file_path) if spec else False
|
||||
|
||||
|
||||
def encode_file_tree_to_yaml(directory, ignore_file, include_library):
|
||||
"""Encode the file tree into a single YAML file."""
|
||||
ignore_patterns = load_ignore_patterns(
|
||||
ignore_file, include_library) if ignore_file else None
|
||||
file_tree: dict[str, Any] = {}
|
||||
|
||||
for root, dirs, files in os.walk(directory):
|
||||
# Skip ignored directories
|
||||
dirs[:] = [d for d in dirs if not is_ignored(
|
||||
os.path.join(root, d), ignore_patterns)]
|
||||
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
|
||||
# Skip ignored files
|
||||
if is_ignored(file_path, ignore_patterns):
|
||||
continue
|
||||
|
||||
# Read file content
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
except Exception as e:
|
||||
print(f"Error reading file {file_path}: {e}")
|
||||
raise
|
||||
|
||||
# Create nested dictionary structure
|
||||
path_parts = os.path.relpath(file_path, directory).split(os.sep)
|
||||
current_level = file_tree
|
||||
|
||||
# all except the last part (the file name)
|
||||
for part in path_parts[:-1]:
|
||||
current_level = current_level.setdefault(part, {})
|
||||
|
||||
# Add the file and its content
|
||||
current_level[path_parts[-1]] = content
|
||||
return yaml.dump(file_tree, default_flow_style=False)
|
||||
|
||||
|
||||
def chunk_and_write_file_tree_to_yaml(yaml_content, output_file, chunk_size=None):
|
||||
"""Chunk the YAML content and write it to the output file."""
|
||||
|
||||
chunks = [yaml_content] if chunk_size is None else [
|
||||
yaml_content[i:i + chunk_size] for i in range(0, len(yaml_content), chunk_size)]
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
chunk_file = f"{output_file}_{i}.yaml"
|
||||
# Write the file tree to the output YAML file
|
||||
with open(chunk_file, 'w', encoding='utf-8') as yaml_file:
|
||||
yaml_file.write(chunk)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Encrypts file tree to yaml file")
|
||||
parser.add_argument("input_directory", help="Directory to encode")
|
||||
parser.add_argument("output_yaml_file", help="Output YAML file")
|
||||
parser.add_argument("--ignore", default=None,
|
||||
help="Path to the ignore file")
|
||||
parser.add_argument("--chunk-size", type=int, default=None,
|
||||
help="Chunk size for the output YAML file")
|
||||
parser.add_argument("--library", type=bool, default=False,
|
||||
help="Incude the library in the output YAML file")
|
||||
|
||||
# Parse arguments
|
||||
args = parser.parse_args()
|
||||
|
||||
# Example usage
|
||||
directory_to_encode = args.input_directory
|
||||
ignore_file_path = args.ignore
|
||||
output_yaml_file = args.output_yaml_file
|
||||
include_library = args.library
|
||||
|
||||
content = encode_file_tree_to_yaml(
|
||||
directory_to_encode, ignore_file_path, include_library)
|
||||
chunk_and_write_file_tree_to_yaml(
|
||||
content, output_yaml_file, args.chunk_size)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -29,6 +29,7 @@ with workflow.unsafe.imports_passed_through():
|
||||
from sientia_model.model_repository.plugin_store import PluginStore
|
||||
|
||||
from laborious.utils.dataframe_debug import build_dataframe_debug_message
|
||||
from laborious.utils.messages import MISSING_MINIO_REPOSITORY
|
||||
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
|
||||
|
||||
|
||||
@@ -445,7 +446,7 @@ class MLFlow(SientiaMonitoring):
|
||||
"""
|
||||
|
||||
if self.minio_repository is None:
|
||||
raise ValueError('Minio repository not initialized')
|
||||
raise ValueError(MISSING_MINIO_REPOSITORY)
|
||||
|
||||
metadata = input_data['metadata']
|
||||
|
||||
|
||||
@@ -16,7 +16,9 @@ with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.temporal.activities.postgres_sync import Postgres
|
||||
from sientia_do.temporal.constants import now
|
||||
|
||||
from laborious.utils.messages import MISSING_MINIO_REPOSITORY
|
||||
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
|
||||
from laborious.utils.time import to_naive
|
||||
|
||||
_LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'})
|
||||
|
||||
@@ -84,7 +86,7 @@ class Storage(Postgres, SientiaMonitoring):
|
||||
dict[str, Any]: Flat ``MinioDataFramePayload`` dict or ``success: False`` on failure.
|
||||
"""
|
||||
if self.minio_repository is None:
|
||||
raise ValueError('Minio repository not initialized')
|
||||
raise ValueError(MISSING_MINIO_REPOSITORY)
|
||||
|
||||
metadata: dict = input_data.get('metadata', {})
|
||||
model_name = input_data['model_name']
|
||||
@@ -152,15 +154,13 @@ class Storage(Postgres, SientiaMonitoring):
|
||||
dict[str, Any]: ``success``, ``deleted_count``, and optional ``message``.
|
||||
"""
|
||||
if self.minio_repository is None:
|
||||
raise ValueError('Minio repository not initialized')
|
||||
raise ValueError(MISSING_MINIO_REPOSITORY)
|
||||
|
||||
metadata = input_data.get('metadata', {})
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
prefix = payload.cleanup_prefix()
|
||||
base = now()
|
||||
cutoff = (base.replace(tzinfo=None) if base.tzinfo else base) - timedelta(
|
||||
hours=self.retention_hours
|
||||
)
|
||||
cutoff = to_naive(base) - timedelta(hours=self.retention_hours)
|
||||
|
||||
report: dict[str, Any] = {
|
||||
'failed': {},
|
||||
@@ -223,7 +223,8 @@ class Storage(Postgres, SientiaMonitoring):
|
||||
sessions, and monitoring hooks), then closes the optional MinIO repository
|
||||
and clears the local reference to avoid accidental reuse after shutdown.
|
||||
"""
|
||||
Postgres.close(self)
|
||||
if hasattr(self, 'engine'):
|
||||
Postgres.close(self)
|
||||
if self.minio_repository is not None:
|
||||
try:
|
||||
self.minio_repository.close()
|
||||
|
||||
1
laborious/utils/messages.py
Normal file
1
laborious/utils/messages.py
Normal file
@@ -0,0 +1 @@
|
||||
MISSING_MINIO_REPOSITORY = 'Minio repository not initialized'
|
||||
@@ -87,7 +87,7 @@ def _opc_authentication_token_str(client: Client | None) -> str:
|
||||
if tok is None:
|
||||
return 'unknown'
|
||||
return str(tok)
|
||||
except Exception:
|
||||
except Exception: # best-effort label for logs/metrics, must not fail loud
|
||||
return 'unknown'
|
||||
|
||||
|
||||
@@ -278,12 +278,14 @@ class OpcRepository(SientiaMonitoring):
|
||||
Configure certificates and timeouts on the sync asyncua client.
|
||||
|
||||
Raises:
|
||||
ValueError: If cert paths or client are missing.
|
||||
ValueError: If cert paths, server URI or client are missing.
|
||||
"""
|
||||
if self.cert_path is None or self.private_key_path is None:
|
||||
raise ValueError(
|
||||
'Certificate and private key paths must be provided for secure connection.'
|
||||
)
|
||||
if self.server_uri is None:
|
||||
raise ValueError('server_uri must be provided for secure connection.')
|
||||
|
||||
cert = Path(self.cert_path)
|
||||
private_key = Path(self.private_key_path)
|
||||
|
||||
6
laborious/utils/time.py
Normal file
6
laborious/utils/time.py
Normal file
@@ -0,0 +1,6 @@
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def to_naive(moment: datetime) -> datetime:
|
||||
"""Strip tzinfo, if any. Postgres `TIMESTAMP WITHOUT TIME ZONE` columns need naive values."""
|
||||
return moment.replace(tzinfo=None) if moment.tzinfo else moment
|
||||
@@ -8,8 +8,8 @@ repository, single PluginStore handle) but polls a different task queue.
|
||||
Task queues (see ``sientia_do.temporal.worker.prepare_worker``):
|
||||
- ``predictions_batch-{runtime}-queue`` + sub-workflows on the same queue (ML-heavy path).
|
||||
- ``minimal_retrain-{runtime}-queue`` (retrain + promote + export).
|
||||
- ``drift-queue`` and ``simple_metrics-queue`` without a runtime suffix so existing schedulers
|
||||
keep stable queue names.
|
||||
- ``drift-core-queue`` and ``simple_metrics-core-queue``: pinned to the ``core`` runtime because
|
||||
drift and metrics do not load model code, so every runtime shares the same queues.
|
||||
|
||||
Bootstrap order:
|
||||
1. Prometheus app metrics and Mongo-backed notification handler.
|
||||
@@ -24,7 +24,7 @@ Environment Variables:
|
||||
- RUNTIME: Required non-empty string passed to ``install_runtime``.
|
||||
- STORE_* / PYPI_*: Plugin store and private index (see ``build_plugin_store_config``).
|
||||
- TEMPORAL_HOST, TEMPORAL_NAMESPACE: Cluster connection.
|
||||
- POD_ID, HTTP_METRICS_PORT, HTTP_SDK_METRICS_PORT: Observability.
|
||||
- HOSTNAME (pod id), HTTP_METRICS_PORT, HTTP_SDK_METRICS_PORT: Observability.
|
||||
- PROJECT_NAME, MONGODB_*: Notifications (via ``build_mongodb_config`` in handler).
|
||||
- POSTGRES_*, MINIO_*, OPC_*, PI_WEB_API_*, MLFLOW_*: Passed through ``Activities`` helpers.
|
||||
"""
|
||||
@@ -75,14 +75,14 @@ async def main():
|
||||
|
||||
Exits the process with code 0 on normal completion of all worker tasks, or 1 after logging
|
||||
if any worker raises. ``finally`` always shuts down notifications and activities and sets
|
||||
``app_up`` to 0 before ``sys.exit``.
|
||||
``app_up`` to 0; ``sys.exit`` runs after it.
|
||||
|
||||
Raises:
|
||||
Exception: Propagated from ``asyncio.gather`` only before ``finally`` handling; typically
|
||||
workers run until cancelled.
|
||||
|
||||
Return:
|
||||
None (process terminates via ``sys.exit`` from the ``finally`` block).
|
||||
None (process terminates via ``sys.exit`` after the ``finally`` block).
|
||||
"""
|
||||
host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
|
||||
logger = get_logger(__name__)
|
||||
@@ -98,7 +98,7 @@ async def main():
|
||||
logger.custom_info(f'Starting Worker with POD_ID: {POD_ID}', metadata)
|
||||
|
||||
logger.custom_info('Starting prometheus client...', metadata)
|
||||
start_prometheus_server()
|
||||
start_prometheus_server(logger, metadata)
|
||||
|
||||
logger.custom_info('Starting Notification Handler...', metadata)
|
||||
|
||||
@@ -157,7 +157,7 @@ async def main():
|
||||
metrics.APP_UP.labels(pod_id=POD_ID).set(0)
|
||||
sys.exit(1)
|
||||
|
||||
logger.custom_info('Starting Activities...', metadata)
|
||||
logger.custom_info('Starting Activities...', metadata_runtime)
|
||||
|
||||
activities = Activities(
|
||||
postgres_config=build_postgres_config(),
|
||||
@@ -170,10 +170,13 @@ async def main():
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
logger.custom_info('Initializing OPC...', metadata)
|
||||
logger.custom_info('Initializing OPC...', metadata_runtime)
|
||||
activities.init_opc()
|
||||
|
||||
logger.custom_info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata)
|
||||
logger.custom_info(
|
||||
f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...',
|
||||
metadata_runtime,
|
||||
)
|
||||
|
||||
new_runtime = Runtime(
|
||||
telemetry=TelemetryConfig(
|
||||
@@ -181,7 +184,7 @@ async def main():
|
||||
)
|
||||
)
|
||||
|
||||
logger.custom_info(f'Starting Temporal Client at {host}...', metadata)
|
||||
logger.custom_info(f'Starting Temporal Client at {host}...', metadata_runtime)
|
||||
|
||||
temporal_client = await client.Client.connect(
|
||||
target_host=host,
|
||||
@@ -189,7 +192,7 @@ async def main():
|
||||
runtime=new_runtime,
|
||||
)
|
||||
|
||||
logger.custom_info('Starting Workers...', metadata)
|
||||
logger.custom_info(f'Starting Workers (runtime={runtime})...', metadata_runtime)
|
||||
|
||||
workers = [
|
||||
prepare_worker(
|
||||
@@ -267,22 +270,29 @@ async def main():
|
||||
for w in workers:
|
||||
handlers.append(w.run())
|
||||
|
||||
logger.custom_info('Workers started successfully', metadata)
|
||||
logger.custom_info('Workers started successfully', metadata_runtime)
|
||||
|
||||
exit_code = 0
|
||||
try:
|
||||
await asyncio.gather(*handlers)
|
||||
except BaseException as e: # NOSONAR
|
||||
logger.custom_error(f'An unhandled exception occurred: {e}', metadata)
|
||||
except Exception as e:
|
||||
logger.custom_error(f'An unhandled exception occurred: {e}', metadata_runtime)
|
||||
exit_code = 1
|
||||
finally:
|
||||
# No `except (SystemExit, KeyboardInterrupt)` on purpose: `finally` runs on every path,
|
||||
# caught or not, so cleanup still happens for a deliberate shutdown (Ctrl+C, a handler's
|
||||
# own sys.exit) without us having to catch and re-raise it ourselves — which would only
|
||||
# let us relabel its exit code before `sys.exit(exit_code)` below discards it anyway.
|
||||
# Letting it propagate on its own keeps the original code (e.g. a handler's sys.exit(N))
|
||||
# intact instead of collapsing it to 0.
|
||||
notification_handler.shutdown()
|
||||
activities.shutdown()
|
||||
metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN
|
||||
sys.exit(exit_code)
|
||||
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
def start_prometheus_server():
|
||||
def start_prometheus_server(logger, metadata: dict):
|
||||
"""
|
||||
Starts the Prometheus metrics server for monitoring and observability.
|
||||
|
||||
@@ -294,7 +304,7 @@ def start_prometheus_server():
|
||||
|
||||
Environment Variables:
|
||||
HTTP_METRICS_PORT: Port for the metrics server (default: 9090)
|
||||
POD_ID: Pod identifier for metrics labeling
|
||||
HOSTNAME: Pod identifier for metrics labeling
|
||||
|
||||
Raises:
|
||||
SystemExit: If the metrics server fails to start
|
||||
@@ -302,10 +312,10 @@ def start_prometheus_server():
|
||||
try:
|
||||
port = int(os.getenv('HTTP_METRICS_PORT', 9090))
|
||||
start_http_server(port)
|
||||
print(f'Prometheus server started on port {port}.')
|
||||
logger.custom_info(f'Prometheus server started on port {port}.', metadata)
|
||||
metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP
|
||||
except Exception as e:
|
||||
print(f'Failed to start Prometheus server: {e}')
|
||||
logger.custom_critical(f'Failed to start Prometheus server: {e}', metadata)
|
||||
os._exit(1)
|
||||
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ with workflow.unsafe.imports_passed_through():
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from temporalio.exceptions import ApplicationError
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
|
||||
@@ -87,7 +88,11 @@ class MinimalRetrain:
|
||||
|
||||
storage_payload = MinioDataFramePayload.from_dict(storage_result)
|
||||
if not storage_payload.has_data():
|
||||
raise ValueError('No data returned from query')
|
||||
raise ApplicationError(
|
||||
'No data returned from query',
|
||||
type='NoDataReturnedError',
|
||||
non_retryable=True,
|
||||
)
|
||||
|
||||
experiment_response = await workflow.execute_activity_method(
|
||||
Activities.retrain_model,
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"id": "e838ff21",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import csv\n",
|
||||
"\n",
|
||||
"def csv_to_tag_lists(csv_path: str) -> dict:\n",
|
||||
" read_tags = []\n",
|
||||
" write_tags = []\n",
|
||||
"\n",
|
||||
" def to_float(val):\n",
|
||||
" try:\n",
|
||||
" return float(str(val).strip())\n",
|
||||
" except Exception:\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
" with open(csv_path, newline=\"\", encoding=\"utf-8\") as f:\n",
|
||||
" reader = csv.DictReader(f)\n",
|
||||
" for row in reader:\n",
|
||||
" # Basic normalization\n",
|
||||
" op = (row.get(\"operation\") or \"\").strip()\n",
|
||||
"\n",
|
||||
" if op == \"READ\":\n",
|
||||
" # Build common tag payload with required mappings\n",
|
||||
" tag = {\n",
|
||||
" \"server_id\": \"1\",\n",
|
||||
" \"tag_address\": row.get(\"opc_tag\"),\n",
|
||||
" \"tag_name\": row.get(\"name\"),\n",
|
||||
" \"data_range\": [to_float(row.get(\"min_value\")), to_float(row.get(\"max_value\"))],\n",
|
||||
" \"aggr_func\": row.get(\"aggregation_func\").lower(),\n",
|
||||
" # keep other fields with their original names\n",
|
||||
" \"frequency\": row.get(\"frequency\"),\n",
|
||||
" \"local\": row.get(\"local\"),\n",
|
||||
" \"area\": row.get(\"area\"),\n",
|
||||
" \"description\": row.get(\"description\"),\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" read_tags.append(tag)\n",
|
||||
"\n",
|
||||
" else:\n",
|
||||
" tag = {\n",
|
||||
" \"server_id\": \"1\",\n",
|
||||
" \"addr\": row.get(\"opc_tag\"),\n",
|
||||
" \"tag_name\": row.get(\"name\"),\n",
|
||||
" \"local\": row.get(\"local\"),\n",
|
||||
" \"area\": row.get(\"area\"),\n",
|
||||
" \"description\": row.get(\"description\"),\n",
|
||||
" }\n",
|
||||
" \n",
|
||||
" if op == \"WRITE_PREDICTION\":\n",
|
||||
" tag[\"type\"] = \"prediction\"\n",
|
||||
" write_tags.append(tag)\n",
|
||||
" elif op == \"WRITE_CONFIDENCE\":\n",
|
||||
" tag[\"type\"] = \"confidence\"\n",
|
||||
" write_tags.append(tag)\n",
|
||||
" # ignore any other operation values silently\n",
|
||||
"\n",
|
||||
" return {\"read_tags\": read_tags, \"write_tags\": write_tags}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4621cd43",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"file_names = [\"Courier - Página1.csv\"]\n",
|
||||
"\n",
|
||||
"for file_name in file_names:\n",
|
||||
" write_file = file_name.replace(\".csv\", \".json\")\n",
|
||||
"\n",
|
||||
" with open(write_file, \"w\", encoding=\"utf-8\") as f:\n",
|
||||
" json.dump(csv_to_tag_lists(file_name), f, indent=2, ensure_ascii=False)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "venv",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.13"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "laborious"
|
||||
version = "0.0.0"
|
||||
version = "0.7.2"
|
||||
description = "Sientia DataOps Laborious - ML Model Orchestration System"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
@@ -60,6 +60,16 @@ ignore = [
|
||||
"S105", # hardcoded passwords ok in tests
|
||||
"S106", # hardcoded passwords ok in tests
|
||||
]
|
||||
"e2e/conftest.py" = [
|
||||
# Imports below the sientia/ModelAnalysis stub are load-order critical: the stub
|
||||
# must be in sys.modules before Activities is imported, so they cannot move up.
|
||||
"E402",
|
||||
]
|
||||
"scripts/**/*.py" = [
|
||||
# Notebook-style scripts split into `# %%` cells: each cell imports what it needs
|
||||
# where it needs it, so imports legitimately appear below other statements.
|
||||
"E402",
|
||||
]
|
||||
|
||||
[tool.ruff.lint.mccabe]
|
||||
max-complexity = 15
|
||||
@@ -116,12 +126,29 @@ python_functions = ["test_*"]
|
||||
addopts = [
|
||||
"-v",
|
||||
"--strict-markers",
|
||||
# pytest>=9.1 has a known bug where its unraisableexception plugin crashes
|
||||
# (tracemalloc partially-initialized AttributeError) when 2+ unraisable
|
||||
# exceptions land close together — e.g. "coroutine was never awaited" from
|
||||
# AsyncMock-mocked sync methods (metrics_controller, minio_repository) being
|
||||
# GC'd. Harmless mock artifacts turned into a hard ERROR by the plugin itself.
|
||||
"-p", "no:unraisableexception",
|
||||
]
|
||||
filterwarnings = [
|
||||
# `kafka-python`, pulled in transitively by `sientia_do`, reads its protocol schemas through the
|
||||
# legacy `importlib.resources` API that Python 3.11 deprecated. It emits ~109 warnings per run,
|
||||
# twice each (once at the call site, once inside `importlib`), which is enough noise to bury a
|
||||
# warning worth reading. Nothing here can fix it — the change belongs in `kafka-python`.
|
||||
#
|
||||
# Matched on the message rather than by module so the filter cannot quietly swallow an unrelated
|
||||
# deprecation: any *other* DeprecationWarning, including one from our own code, still shows up.
|
||||
"ignore:read_text is deprecated:DeprecationWarning",
|
||||
"ignore:open_text is deprecated:DeprecationWarning",
|
||||
]
|
||||
markers = [
|
||||
"asyncio: marks tests as async",
|
||||
"integration: marks tests as integration tests",
|
||||
"unit: marks tests as unit tests",
|
||||
"opc: marks E2E tests that use in-process asyncua + real OpcRepository",
|
||||
"opc: marks tests that use the in-process OPC UA server (OpcRepository E2E)",
|
||||
]
|
||||
|
||||
[tool.coverage.run]
|
||||
@@ -148,6 +175,7 @@ exclude_lines = [
|
||||
"if TYPE_CHECKING:",
|
||||
"class .*\\bProtocol\\):",
|
||||
"@(abc\\.)?abstractmethod",
|
||||
"@overload",
|
||||
]
|
||||
|
||||
[tool.coverage.html]
|
||||
|
||||
@@ -13,7 +13,10 @@ types-requests>=2.31.0 # Type stubs for requests
|
||||
pytest>=7.4.0 # Testing framework
|
||||
pytest-cov>=4.1.0 # Coverage plugin for pytest
|
||||
pytest-asyncio>=0.21.0 # Async test support (already in main requirements)
|
||||
testcontainers[postgres,minio] # PostgreSQL and MinIO containers for E2E tests
|
||||
testcontainers[postgres,minio,mongodb]>=4.15.0 # PostgreSQL, MinIO and MongoDB containers for E2E tests
|
||||
# Floor is the `testcontainers.community.*` namespace: the flat
|
||||
# `testcontainers.minio` / `.postgres` / `.mongodb` paths the e2e suite used
|
||||
# to import are deprecated shims that warn on import.
|
||||
|
||||
# Development Tools
|
||||
ipython>=8.12.0 # Enhanced Python shell
|
||||
|
||||
@@ -3,16 +3,21 @@ psycopg2-binary
|
||||
sqlalchemy
|
||||
asyncua==1.0.6
|
||||
redis
|
||||
git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.1
|
||||
git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.2
|
||||
git+ssh://git@github.com/Aignosi/sientia-model-library.git@0.10.0
|
||||
prometheus-client
|
||||
botocore
|
||||
boto3
|
||||
s3fs
|
||||
# fsspec/s3fs pinned as a pair: s3fs pins fsspec exactly, so a free `s3fs` can make the
|
||||
# resolver fail against evidently's fsspec>=2024.2.0. Versions match the
|
||||
# library-distribution-controller freeze.
|
||||
fsspec==2026.7.0
|
||||
s3fs==2026.7.0
|
||||
# aiobotocore 3.9.0 (pulled by s3fs) requires botocore<1.43.57,>=1.43.3.
|
||||
botocore==1.43.56
|
||||
boto3==1.43.56
|
||||
pyarrow
|
||||
kaleido
|
||||
hyperopt
|
||||
shap
|
||||
pycurl
|
||||
scipy<1.14.0
|
||||
scikit-learn==1.5.2
|
||||
scikit-learn==1.5.2
|
||||
|
||||
@@ -3,16 +3,21 @@ psycopg2-binary
|
||||
sqlalchemy
|
||||
asyncua==1.0.6
|
||||
redis
|
||||
sientia_do>=1.12.1
|
||||
sientia_do>=1.12.2
|
||||
sientia_model>=0.8.2
|
||||
prometheus-client
|
||||
botocore
|
||||
boto3
|
||||
s3fs
|
||||
# fsspec/s3fs pinned as a pair: s3fs pins fsspec exactly, so a free `s3fs` can make the
|
||||
# resolver fail against evidently's fsspec>=2024.2.0. Versions match the
|
||||
# library-distribution-controller freeze.
|
||||
fsspec==2026.7.0
|
||||
s3fs==2026.7.0
|
||||
# aiobotocore 3.9.0 (pulled by s3fs) requires botocore<1.43.57,>=1.43.3.
|
||||
botocore==1.43.56
|
||||
boto3==1.43.56
|
||||
pyarrow
|
||||
kaleido
|
||||
hyperopt
|
||||
shap
|
||||
pycurl
|
||||
scipy<1.14.0
|
||||
scikit-learn
|
||||
scikit-learn
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
sonar.projectKey=Aignosi_sientia-dataops-laborious_temporal_beaec423-6c42-4f26-8134-b676287b499d
|
||||
sonar.projectKey=Aignosi_sientia-dataops-laborious_temporal_ca1a7039-6db9-49e5-be78-54d29bc93e4f
|
||||
sonar.projectName=sientia-dataops-laborious_temporal
|
||||
sonar.sources=laborious
|
||||
sonar.tests=tests
|
||||
@@ -8,4 +8,4 @@ sonar.qualitygate.wait=true
|
||||
sonar.qualitygate.timeout=300
|
||||
sonar.python.coverage.reportPaths=coverage.xml
|
||||
sonar.python.xunit.reportPath=pytest.xml
|
||||
sonar.python.version=3.11
|
||||
sonar.python.version=3.11
|
||||
938
tests.ipynb
938
tests.ipynb
File diff suppressed because one or more lines are too long
@@ -218,6 +218,43 @@ def test_write_pi_web_api_data_success(mock_dataframe, api, base_input_data):
|
||||
}
|
||||
|
||||
|
||||
@patch('laborious.activities.api.DataFrame')
|
||||
def test_write_pi_web_api_data_partial_prediction_mismatch_downgrades_confidence(
|
||||
mock_dataframe, api, base_input_data
|
||||
):
|
||||
"""A partial write (some tags report Errors) doesn't raise — it's caught by the tag-count
|
||||
mismatch check, which still must downgrade confidence and attach the mismatch message."""
|
||||
input_data = {
|
||||
**base_input_data,
|
||||
'pi_web_api_output_config': {
|
||||
'endpoint': 'https://test-pi-server.com/piwebapi',
|
||||
'prediction_tags': {'tag1': 'web_id_1', 'tag2': 'web_id_2'},
|
||||
'confidence_tags': {'tag3': 'web_id_3'},
|
||||
},
|
||||
}
|
||||
mock_df = _create_mock_dataframe()
|
||||
mock_dataframe.return_value = mock_df
|
||||
|
||||
api.pi_web_api_client.write_value.side_effect = [
|
||||
[{'WebId': 'web_id_1', 'Errors': []}, {'WebId': 'web_id_2', 'Errors': ['bad tag']}],
|
||||
[{'WebId': 'web_id_3', 'Errors': []}],
|
||||
]
|
||||
|
||||
api.write_pi_web_api_data(input_data)
|
||||
|
||||
mock_df.__setitem__.assert_any_call(
|
||||
'prediction_confidence', PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
|
||||
)
|
||||
mock_df.__setitem__.assert_any_call('comments', ANY)
|
||||
api.send_notification.assert_called_once_with(
|
||||
metadata=metadata['metadata'],
|
||||
notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR',
|
||||
message=ANY,
|
||||
block='write_pi_web_api_data',
|
||||
level=NotificationLevel.ERROR,
|
||||
)
|
||||
|
||||
|
||||
@patch('laborious.activities.api.DataFrame')
|
||||
def test_write_pi_web_api_data_prediction_error(mock_dataframe, api, base_input_data):
|
||||
mock_dataframe.return_value = _create_mock_dataframe(
|
||||
|
||||
@@ -539,6 +539,38 @@ def test_manage_output_tags_do_nothing(opc):
|
||||
opc._write_tags_from_config.assert_not_called()
|
||||
|
||||
|
||||
@patch('laborious.activities.opc.DataFrame')
|
||||
def test_write_opc_data_propagates_session_bad_and_reconnect_in_progress(mock_dataframe, opc):
|
||||
"""A single server reporting session_bad/reconnect_in_progress must flip the aggregated
|
||||
flags passed to process_confidence, even though nothing raised."""
|
||||
input_data = {
|
||||
**metadata,
|
||||
'data': {'prediction': [0.75], 'prediction_confidence': [0.95]},
|
||||
'opc_output_config': {
|
||||
'server1': {
|
||||
'prediction_tags': {'tag1': {'data_type': 'float'}},
|
||||
'confidence_tags': {'tag2': {'data_type': 'float'}},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
opc.manage_output_tags = MagicMock(
|
||||
return_value=(False, {'tag1': 0.1, 'tag2': 0.2}, True, 'BadSessionClosed', True)
|
||||
)
|
||||
opc.process_confidence = MagicMock(return_value={'data': 'data'})
|
||||
|
||||
opc.write_opc_data(input_data)
|
||||
|
||||
opc.process_confidence.assert_called_once_with(
|
||||
mock_dataframe.return_value,
|
||||
False,
|
||||
metadata['metadata'],
|
||||
session_bad=True,
|
||||
opc_status='BadSessionClosed',
|
||||
reconnect_in_progress=True,
|
||||
)
|
||||
|
||||
|
||||
@patch('laborious.activities.opc.DataFrame')
|
||||
def test_write_opc_data_success(mock_dataframe, opc):
|
||||
# Arrange
|
||||
|
||||
@@ -220,6 +220,26 @@ def test_from_dataframe_offloaded(mock_now):
|
||||
minio.upload_file.assert_called_once()
|
||||
|
||||
|
||||
@patch('laborious.utils.models.minio_dataframe_payload.OFFLOAD_THRESHOLD_BYTES', 10**9)
|
||||
def test_from_dataframe_inline_keeps_a_provided_last_timestamp():
|
||||
"""An explicit last_timestamp must survive as-is, skipping the max() derivation."""
|
||||
minio = MagicMock()
|
||||
logger = MagicMock()
|
||||
df = _mock_dataframe({'timestamp': ['2024-01-01'], 'value': [42]})
|
||||
|
||||
result = MinioDataFramePayload.from_dataframe(
|
||||
dataframe=df,
|
||||
minio_repo=minio,
|
||||
model_name='m',
|
||||
operation='initial',
|
||||
last_timestamp='2030-01-01T00:00:00+00:00',
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
assert result.last_timestamp == '2030-01-01T00:00:00+00:00'
|
||||
logger.custom_debug.assert_called()
|
||||
|
||||
|
||||
def test_from_dict_inline():
|
||||
raw = {
|
||||
'last_timestamp': '2024-01-01T00:00:00+00:00',
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import concurrent.futures
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from unittest.mock import ANY, MagicMock, Mock, patch
|
||||
|
||||
@@ -8,11 +10,13 @@ from asyncua.crypto import security_policies
|
||||
from asyncua.ua.uaerrors import BadNodeIdUnknown, BadSessionIdInvalid
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
|
||||
from laborious import metrics
|
||||
from laborious.utils.repository.opc_repository import (
|
||||
OpcClientAlreadyExistsError,
|
||||
OpcClientNotInitializedError,
|
||||
OpcRepository,
|
||||
OpcSessionAlreadyConnectedError,
|
||||
_opc_authentication_token_str,
|
||||
is_reconnectable_opcua_bad,
|
||||
)
|
||||
|
||||
@@ -86,6 +90,40 @@ def test_init(opc_repository):
|
||||
assert opc_repository.last_reconnection_time is None
|
||||
|
||||
|
||||
def test_opc_authentication_token_str_no_protocol():
|
||||
client = MagicMock()
|
||||
client.aio_obj.uaclient.protocol = None
|
||||
|
||||
assert _opc_authentication_token_str(client) == 'unknown'
|
||||
|
||||
|
||||
def test_opc_authentication_token_str_no_token():
|
||||
client = MagicMock()
|
||||
client.aio_obj.uaclient.protocol = MagicMock(authentication_token=None)
|
||||
|
||||
assert _opc_authentication_token_str(client) == 'unknown'
|
||||
|
||||
|
||||
def test_opc_authentication_token_str_swallows_errors():
|
||||
class _ExplodingClient:
|
||||
@property
|
||||
def aio_obj(self):
|
||||
raise RuntimeError('boom')
|
||||
|
||||
assert _opc_authentication_token_str(_ExplodingClient()) == 'unknown'
|
||||
|
||||
|
||||
def test_is_session_open_swallows_errors(opc_repository):
|
||||
class _ExplodingClient:
|
||||
@property
|
||||
def aio_obj(self):
|
||||
raise RuntimeError('boom')
|
||||
|
||||
opc_repository.client = _ExplodingClient()
|
||||
|
||||
assert opc_repository._is_session_open() is False
|
||||
|
||||
|
||||
def test_set_security(opc_repository, mock_client):
|
||||
opc_repository.client = mock_client
|
||||
opc_repository.set_security()
|
||||
@@ -112,6 +150,13 @@ def test_set_security_missing_certificates(opc_repository):
|
||||
assert str(e) == 'Certificate and private key paths must be provided for secure connection.'
|
||||
|
||||
|
||||
def test_set_security_missing_server_uri(opc_repository):
|
||||
opc_repository.server_uri = None
|
||||
|
||||
with pytest.raises(ValueError, match='server_uri must be provided for secure connection.'):
|
||||
opc_repository.set_security()
|
||||
|
||||
|
||||
def test_set_security_missing_client(opc_repository):
|
||||
opc_repository.client = None
|
||||
try:
|
||||
@@ -160,6 +205,43 @@ def test_create_client_raises_when_client_exists(opc_repository, mock_client):
|
||||
opc_repository._create_client()
|
||||
|
||||
|
||||
def test_create_client_builds_a_secured_client(opc_repository, mock_client):
|
||||
opc_repository.pod_id = 'pod-1'
|
||||
opc_repository.set_security = MagicMock()
|
||||
|
||||
opc_repository._create_client()
|
||||
|
||||
assert opc_repository.client is mock_client
|
||||
assert mock_client.aio_obj.name == 'pod-1'
|
||||
assert mock_client.application_uri == 'pod:1'
|
||||
assert mock_client.aio_obj.product_uri == 'pod:1'
|
||||
opc_repository.set_security.assert_called_once()
|
||||
|
||||
|
||||
def test_create_client_skips_security_without_a_cert_path(opc_repository, mock_client):
|
||||
opc_repository.cert_path = None
|
||||
opc_repository.set_security = MagicMock()
|
||||
|
||||
opc_repository._create_client()
|
||||
|
||||
assert opc_repository.client is mock_client
|
||||
opc_repository.set_security.assert_not_called()
|
||||
|
||||
|
||||
def test_connect_locked_reuses_an_existing_client(opc_repository, mock_client):
|
||||
"""A client already set (e.g. surviving a prior failed open) is not recreated."""
|
||||
opc_repository.client = mock_client
|
||||
opc_repository._is_session_open = MagicMock(return_value=False)
|
||||
opc_repository._create_client = MagicMock()
|
||||
opc_repository._open_session = MagicMock(return_value=(True, {}))
|
||||
|
||||
result = opc_repository._connect_locked()
|
||||
|
||||
opc_repository._create_client.assert_not_called()
|
||||
opc_repository._open_session.assert_called_once()
|
||||
assert result == (True, {})
|
||||
|
||||
|
||||
def test_open_session_success(opc_repository):
|
||||
closed_proto = MagicMock()
|
||||
closed_proto.state = 'closed'
|
||||
@@ -423,6 +505,75 @@ def test_write_data_write_value_failed(opc_repository, mock_client):
|
||||
assert error_data['attachment_content'] is not None
|
||||
|
||||
|
||||
def test_write_data_with_empty_metadata_uses_unknown_labels(opc_repository, mock_client):
|
||||
"""No model_id/model_name in the write metadata: the metric labels fall back to 'unknown'."""
|
||||
opc_repository.validate_connection = MagicMock(return_value=(True, {}))
|
||||
opc_repository.client = mock_client
|
||||
mock_client.get_node = MagicMock(return_value=MagicMock())
|
||||
|
||||
result = opc_repository.write_data('ns=2;s=TestNode', 42.0, 'float', {})
|
||||
|
||||
assert result == (True, {'response_time': ANY})
|
||||
last_call = opc_repository.emit_metric_sync.call_args_list[-1]
|
||||
assert last_call.args[0] is metrics.OPC_WRITE_ATTEMPTS_TOTAL
|
||||
assert last_call.args[1]['model_id'] == 'unknown'
|
||||
assert last_call.args[1]['model_name'] == 'unknown'
|
||||
|
||||
|
||||
def test_write_data_get_node_bad_session_id_schedules_reconnect(opc_repository, mock_client):
|
||||
opc_repository.validate_connection = MagicMock(return_value=(True, {}))
|
||||
opc_repository.client = mock_client
|
||||
opc_repository._start_reconnect = MagicMock()
|
||||
mock_client.get_node = MagicMock(side_effect=BadSessionIdInvalid())
|
||||
|
||||
is_success, error_data = opc_repository.write_data(
|
||||
'ns=2;s=TestNode', 42.0, 'float', metadata['metadata']
|
||||
)
|
||||
|
||||
opc_repository._start_reconnect.assert_called_once()
|
||||
assert is_success is False
|
||||
assert error_data['opc_error_kind'] == 'session_bad'
|
||||
assert error_data['opc_status'] == 'BadSessionIdInvalid'
|
||||
|
||||
|
||||
def test_write_data_logs_inter_arrival_and_flags_a_stale_session(opc_repository, mock_client):
|
||||
"""The second write on the same session logs the gap since the previous one, and flags it
|
||||
when the gap outlives the session's revised timeout."""
|
||||
opc_repository.validate_connection = MagicMock(return_value=(True, {}))
|
||||
opc_repository.client = mock_client
|
||||
mock_client.aio_obj.session_timeout = 0.001 # 1 microsecond: any real gap will exceed it
|
||||
mock_client.get_node = MagicMock(return_value=MagicMock())
|
||||
|
||||
opc_repository.write_data('ns=2;s=TestNode', 1.0, 'float', metadata['metadata'])
|
||||
opc_repository.write_data('ns=2;s=TestNode', 2.0, 'float', metadata['metadata'])
|
||||
|
||||
logged = [c.args[0] for c in opc_repository.info.call_args_list]
|
||||
assert any('OPC write inter-arrival' in message for message in logged)
|
||||
emitted_metrics = [c.args[0] for c in opc_repository.emit_metric_sync.call_args_list]
|
||||
assert metrics.OPC_WRITE_INTER_ARRIVAL_OVER_SESSION_TIMEOUT_TOTAL in emitted_metrics
|
||||
|
||||
|
||||
def test_log_write_inter_arrival_skips_the_metric_without_a_client(opc_repository):
|
||||
opc_repository._last_write_mono = time.monotonic() - 1
|
||||
opc_repository.client = None
|
||||
|
||||
opc_repository._log_write_inter_arrival('session-1', 'ns=2;s=Node')
|
||||
|
||||
opc_repository.emit_metric_sync.assert_not_called()
|
||||
|
||||
|
||||
def test_log_write_inter_arrival_skips_the_metric_within_the_session_timeout(
|
||||
opc_repository, mock_client
|
||||
):
|
||||
opc_repository._last_write_mono = time.monotonic()
|
||||
opc_repository.client = mock_client
|
||||
mock_client.aio_obj.session_timeout = 600_000 # 600s: comfortably longer than this test takes
|
||||
|
||||
opc_repository._log_write_inter_arrival('session-1', 'ns=2;s=Node')
|
||||
|
||||
opc_repository.emit_metric_sync.assert_not_called()
|
||||
|
||||
|
||||
def test_is_reconnectable_opcua_bad():
|
||||
assert is_reconnectable_opcua_bad(BadSessionIdInvalid()) is True
|
||||
assert is_reconnectable_opcua_bad(BadNodeIdUnknown()) is False
|
||||
@@ -472,6 +623,23 @@ def test_start_reconnect_skips_within_interval(opc_repository):
|
||||
assert opc_repository._reconnect_thread is None
|
||||
|
||||
|
||||
def test_start_reconnect_skips_when_already_in_progress(opc_repository):
|
||||
opc_repository.last_reconnection_time = None
|
||||
release = threading.Event()
|
||||
in_progress_thread = threading.Thread(target=release.wait, daemon=True)
|
||||
in_progress_thread.start()
|
||||
opc_repository._reconnect_thread = in_progress_thread
|
||||
|
||||
try:
|
||||
opc_repository._start_reconnect('BadSessionIdInvalid', 'tok')
|
||||
|
||||
assert opc_repository._reconnect_thread is in_progress_thread
|
||||
opc_repository.warning.assert_called_once()
|
||||
finally:
|
||||
release.set()
|
||||
in_progress_thread.join(timeout=5.0)
|
||||
|
||||
|
||||
def test_write_data_protocol_closed_schedules_reconnect(opc_repository):
|
||||
opc_repository.client = MagicMock()
|
||||
opc_repository.client.aio_obj.uaclient.protocol = MagicMock(state='closed')
|
||||
@@ -582,3 +750,21 @@ def test_reconnect_locked_sets_last_reconnection_time(mock_datetime, opc_reposit
|
||||
opc_repository._connect_locked.assert_called_once()
|
||||
assert result == (True, {})
|
||||
assert opc_repository.last_reconnection_time == datetime(2025, 1, 1, 12, 0, 0)
|
||||
|
||||
|
||||
def test_run_reconnect_failed_without_error_detail_logs_once(opc_repository):
|
||||
"""A falsy `error` (no message to add) must not log a second, empty line."""
|
||||
opc_repository._reconnect_locked = MagicMock(return_value=(False, {}))
|
||||
|
||||
opc_repository._run_reconnect('BadSessionIdInvalid', 'tok')
|
||||
|
||||
assert opc_repository.error.call_count == 1
|
||||
|
||||
|
||||
def test_run_reconnect_swallows_an_unexpected_exception(opc_repository):
|
||||
"""A bug inside the reconnect itself must not escape the background thread."""
|
||||
opc_repository._reconnect_locked = MagicMock(side_effect=RuntimeError('boom'))
|
||||
|
||||
opc_repository._run_reconnect('BadSessionIdInvalid', 'tok')
|
||||
|
||||
assert opc_repository.error.call_count >= 2
|
||||
|
||||
@@ -1,12 +1,40 @@
|
||||
"""Debug message formatting: full dataframe when small, a summary when it would flood the log."""
|
||||
|
||||
from pandas import DataFrame
|
||||
|
||||
from laborious.utils.dataframe_debug import build_dataframe_debug_message
|
||||
from laborious.utils.dataframe_debug import (
|
||||
DEFAULT_MAX_DEBUG_DATAFRAME_ROWS,
|
||||
build_dataframe_debug_message,
|
||||
)
|
||||
|
||||
|
||||
def test_build_dataframe_debug_message_skips_large_dataframe():
|
||||
df = DataFrame({'a': [1, 2, 3]})
|
||||
def test_non_dataframe_payloads_are_stringified():
|
||||
assert build_dataframe_debug_message('payload:', {'a': 1}) == "payload: {'a': 1}"
|
||||
|
||||
msg = build_dataframe_debug_message('payload', df, max_rows=1)
|
||||
|
||||
assert 'skipped because dataframe has 3 rows' in msg
|
||||
assert '(max: 1)' in msg
|
||||
def test_a_small_dataframe_is_logged_in_full():
|
||||
data = DataFrame({'a': [1, 2]})
|
||||
|
||||
message = build_dataframe_debug_message('payload:', data)
|
||||
|
||||
assert message.startswith('payload:\n')
|
||||
assert data.to_csv() in message
|
||||
|
||||
|
||||
def test_a_dataframe_over_the_row_limit_is_summarized_instead():
|
||||
data = DataFrame({'a': range(DEFAULT_MAX_DEBUG_DATAFRAME_ROWS + 1)})
|
||||
|
||||
message = build_dataframe_debug_message('payload:', data)
|
||||
|
||||
assert 'skipped because dataframe has' in message
|
||||
assert str(data.shape) in message
|
||||
assert data.to_csv() not in message
|
||||
|
||||
|
||||
def test_the_row_limit_is_configurable():
|
||||
data = DataFrame({'a': [1, 2, 3]})
|
||||
|
||||
message = build_dataframe_debug_message('payload:', data, max_rows=1)
|
||||
|
||||
assert 'skipped because dataframe has 3 rows' in message
|
||||
assert '(max: 1)' in message
|
||||
|
||||
19
tests/laborious/utils/test_time.py
Normal file
19
tests/laborious/utils/test_time.py
Normal file
@@ -0,0 +1,19 @@
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from laborious.utils.time import to_naive
|
||||
|
||||
|
||||
def test_to_naive_returns_naive_input_unchanged() -> None:
|
||||
moment = datetime(2024, 1, 1, 12, 30, 45)
|
||||
|
||||
assert to_naive(moment) == moment
|
||||
assert to_naive(moment).tzinfo is None
|
||||
|
||||
|
||||
def test_to_naive_strips_tzinfo_from_aware_input() -> None:
|
||||
moment = datetime(2024, 1, 1, 12, 30, 45, tzinfo=UTC)
|
||||
|
||||
result = to_naive(moment)
|
||||
|
||||
assert result.tzinfo is None
|
||||
assert result == datetime(2024, 1, 1, 12, 30, 45)
|
||||
@@ -48,21 +48,25 @@ def _build_fake_worker(async_result=None, async_error: Exception | None = None):
|
||||
|
||||
@patch('laborious.worker.worker.start_http_server')
|
||||
def test_start_prometheus_server_success(mock_start_http):
|
||||
logger = MagicMock()
|
||||
with patch.object(worker.metrics.APP_UP, 'labels') as labels:
|
||||
gauge = MagicMock()
|
||||
labels.return_value = gauge
|
||||
with patch('laborious.worker.worker.os.getenv', return_value='9090'):
|
||||
worker.start_prometheus_server()
|
||||
worker.start_prometheus_server(logger, {})
|
||||
mock_start_http.assert_called_once_with(9090)
|
||||
gauge.set.assert_called_once_with(1)
|
||||
logger.custom_info.assert_called_once_with('Prometheus server started on port 9090.', {})
|
||||
|
||||
|
||||
@patch('laborious.worker.worker.start_http_server', side_effect=RuntimeError('nope'))
|
||||
def test_start_prometheus_server_error_exits(_mock_start_http):
|
||||
logger = MagicMock()
|
||||
with patch('laborious.worker.worker.os._exit', side_effect=SystemExit(1)) as m_exit:
|
||||
with raises(SystemExit):
|
||||
worker.start_prometheus_server()
|
||||
worker.start_prometheus_server(logger, {})
|
||||
m_exit.assert_called_once_with(1)
|
||||
logger.custom_critical.assert_called_once_with('Failed to start Prometheus server: nope', {})
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@@ -181,6 +185,9 @@ async def test_main_success_exit_zero(monkeypatch):
|
||||
prepare_calls = m_prepare.call_args_list
|
||||
assert prepare_calls[0].kwargs['runtime'] == 'single'
|
||||
assert prepare_calls[3].kwargs['runtime'] == 'single'
|
||||
# Drift and SimpleMetrics load no model code, so every runtime shares the `core` queues.
|
||||
assert prepare_calls[1].kwargs['runtime'] == 'core'
|
||||
assert prepare_calls[2].kwargs['runtime'] == 'core'
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
|
||||
@@ -791,6 +791,30 @@ async def test_path_flag_handler_unknown(workflow_mock, prediction_process):
|
||||
workflow_mock.execute_child_workflow.assert_not_called()
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock)
|
||||
async def test_run_cleans_up_and_reraises_on_pipeline_failure(workflow_mock, prediction_process):
|
||||
"""A failure anywhere in the pipeline must still release the MinIO offload before the
|
||||
workflow fails — cleanup is not conditional on success."""
|
||||
prediction_process._run_prediction_pipeline = AsyncMock(side_effect=RuntimeError('boom'))
|
||||
input_data = {
|
||||
'metadata': metadata,
|
||||
'data': {'some': 'data'},
|
||||
'model_id': 1,
|
||||
'model_name': 'test_model_name',
|
||||
}
|
||||
|
||||
with raises(RuntimeError, match='boom'):
|
||||
await prediction_process.run(input_data)
|
||||
|
||||
workflow_mock.execute_activity_method.assert_called_once_with(
|
||||
Activities.cleanup_minio_objects_expired,
|
||||
{**metadata, 'data': input_data['data']},
|
||||
retry_policy=ANY,
|
||||
start_to_close_timeout=ANY,
|
||||
)
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock)
|
||||
async def test_run_with_cleanup_prefixes(workflow_mock, prediction_process):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from unittest.mock import ANY, AsyncMock, call, patch
|
||||
|
||||
from pytest import fixture, mark
|
||||
from pytest import fixture, mark, raises
|
||||
from temporalio.exceptions import ApplicationError
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.workflows.minimal_retrain import MinimalRetrain
|
||||
@@ -192,11 +193,12 @@ async def test_run_storage_fail(workflow_mock: AsyncMock, minimal_retrain: Minim
|
||||
]
|
||||
)
|
||||
|
||||
from pytest import raises
|
||||
|
||||
with raises(ValueError, match='No data returned from query'):
|
||||
with raises(ApplicationError, match='No data returned from query') as exc_info:
|
||||
await minimal_retrain.run(input_data)
|
||||
|
||||
assert exc_info.value.non_retryable is True
|
||||
assert exc_info.value.type == 'NoDataReturnedError'
|
||||
|
||||
workflow_mock.execute_activity_method.assert_called_once_with(
|
||||
Activities.load_query_with_minio_offload,
|
||||
{
|
||||
|
||||
319
values.yaml
319
values.yaml
@@ -1,319 +0,0 @@
|
||||
#
|
||||
# Default values for sientia-laborious-worker using the sientia-module chart (0.6.x).
|
||||
# This is a YAML-formatted file.
|
||||
# Declare variables to be passed into your templates.
|
||||
#
|
||||
|
||||
projectName: &projectName "sientia-laborious-worker"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Global configuration shared by all runtimes
|
||||
# -----------------------------------------------------------------------------
|
||||
global:
|
||||
namespace: sientia
|
||||
|
||||
image:
|
||||
repository: aignosi.azurecr.io/sientia-module
|
||||
pullPolicy: Always
|
||||
tag: "1.2.0"
|
||||
|
||||
commonLabels: {}
|
||||
|
||||
resources:
|
||||
# Resource limits and requests are important for ResourceBasedTuner to work correctly.
|
||||
# The tuner monitors system CPU and memory usage, so proper resource limits must be set.
|
||||
limits:
|
||||
cpu: 2000m
|
||||
memory: 20Gi
|
||||
requests:
|
||||
cpu: 1000m
|
||||
memory: 2Gi
|
||||
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
curl -sf http://localhost:9090/metrics | grep -q '^app_up{.*} 1'
|
||||
initialDelaySeconds: 1260
|
||||
periodSeconds: 15
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
curl -sf http://localhost:9090/metrics | grep -q '^app_up{.*} 1'
|
||||
initialDelaySeconds: 1200
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 2
|
||||
|
||||
autoscaling:
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
maxReplicas: 100
|
||||
targetCPUUtilizationPercentage: 80
|
||||
# targetMemoryUtilizationPercentage: 80
|
||||
|
||||
# Environment variables shared by all runtimes.
|
||||
env:
|
||||
# Entrypoint variables
|
||||
- name: GITHUB_REPO_URL
|
||||
value: "git@github.com:Aignosi/sientia-dataops-laborious_temporal.git"
|
||||
- name: GITHUB_BRANCH
|
||||
value: "release/SIENTIAPDE-1646"
|
||||
- name: PYTHON_APP
|
||||
value: "laborious.worker.worker"
|
||||
- name: PYPI_SERVER
|
||||
value: "http://library-distribution-server.library.svc.cluster.local:5000"
|
||||
|
||||
# Application variables
|
||||
- name: POSTGRES_HOST
|
||||
value: "paradedb-rw.paradedb.svc.cluster.local"
|
||||
- name: POSTGRES_PORT
|
||||
value: "5432"
|
||||
- name: POSTGRES_USER
|
||||
value: "postgres"
|
||||
- name: POSTGRES_PASSWORD
|
||||
value: "nFqc81y6kwmr2zuAIx43DhiOosFCVPpeEfTtTWZflkNjB2j1KtEeIANkhFR9mAX3"
|
||||
- name: POSTGRES_DBNAME
|
||||
value: "sientia"
|
||||
- name: POSTGRES_MIN_CONNECTIONS
|
||||
value: "20"
|
||||
# max_connections = number_of_workers * max_concurrent_activities * safety_factor
|
||||
# Example: 4 workers * 50 activities * 0.5 = 100 connections
|
||||
- name: POSTGRES_MAX_CONNECTIONS
|
||||
value: "100"
|
||||
|
||||
- name: MLFLOW_URL
|
||||
value: "http://sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local:80"
|
||||
- name: MLFLOW_USERNAME
|
||||
value: "aignosi"
|
||||
- name: MLFLOW_PASSWORD
|
||||
value: "1L0FP50j3ncp123"
|
||||
|
||||
# Plugin store (model-library-store Git + runtime packages).
|
||||
- name: STORE_BASE_URL
|
||||
value: "http://gitea-http.gitea.svc.cluster.local:3000"
|
||||
- name: STORE_OWNER
|
||||
value: "aignosi"
|
||||
- name: STORE_REPO
|
||||
value: "suse-model-store"
|
||||
- name: STORE_USERNAME
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: sientia-plugin-store-credentials
|
||||
key: username
|
||||
- name: STORE_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: sientia-plugin-store-credentials
|
||||
key: password
|
||||
- name: STORE_CACHE_TTL_SECONDS
|
||||
value: "3600"
|
||||
|
||||
- name: OPC_ID
|
||||
value: "1"
|
||||
- name: OPC_SERVER_NAME
|
||||
value: "default_server"
|
||||
- name: OPC_URL
|
||||
value: "opc.tcp://sientia-opc-simulator-opc.sientia.svc.cluster.local:4840"
|
||||
|
||||
- name: LOG_LEVEL
|
||||
value: "DEBUG"
|
||||
- name: HTTP_METRICS_PORT
|
||||
value: "9090"
|
||||
- name: HTTP_SDK_METRICS_PORT
|
||||
value: "9091"
|
||||
- name: PROJECT_NAME
|
||||
value: "sientia-laborious"
|
||||
|
||||
- name: TEMPORAL_HOST
|
||||
value: "temporal-frontend.temporal.svc.cluster.local:7233"
|
||||
- name: TEMPORAL_NAMESPACE
|
||||
value: "laborious"
|
||||
|
||||
- name: MONGODB_USERNAME
|
||||
value: "root"
|
||||
- name: MONGODB_PASSWORD
|
||||
value: "wKZDbMNU1c"
|
||||
- name: MONGODB_URL
|
||||
value: "my-release-mongodb.mongodb.svc.cluster.local:27017"
|
||||
- name: MONGODB_DATABASE
|
||||
value: "sientia"
|
||||
- name: MONGODB_TTL_INDEX_HOURS
|
||||
value: "1"
|
||||
|
||||
- name: MINIO_ENDPOINT_URL
|
||||
value: "minio.minio.svc.cluster.local:9000"
|
||||
- name: MINIO_ACCESS_KEY
|
||||
value: "admin"
|
||||
- name: MINIO_SECRET_KEY
|
||||
value: "LiArt4eNmJ"
|
||||
- name: MINIO_DEFAULT_BUCKET
|
||||
value: "sientia"
|
||||
- name: MINIO_RETENTION_HOURS
|
||||
value: "24"
|
||||
- name: SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES
|
||||
value: "0.5"
|
||||
|
||||
# Temporal worker tuning for PredictionsBatch.
|
||||
# IMPORTANT: prefix must be PREDICTIONSBATCH_ (from class name PredictionsBatch).
|
||||
- name: PREDICTIONSBATCH_MAX_CONCURRENT_WORKFLOW_TASKS
|
||||
value: "20"
|
||||
- name: PREDICTIONSBATCH_MAX_CONCURRENT_ACTIVITIES
|
||||
value: "60"
|
||||
- name: PREDICTIONSBATCH_ACTIVITY_EXECUTOR_MAX_WORKERS
|
||||
value: "10"
|
||||
- name: PREDICTIONSBATCH_MAX_CONCURRENT_LOCAL_ACTIVITIES
|
||||
value: "20"
|
||||
- name: PREDICTIONSBATCH_MAX_CACHED_WORKFLOWS
|
||||
value: "200"
|
||||
- name: PREDICTIONSBATCH_WORKFLOW_POLLER_BEHAVIOUR_MINIMUM
|
||||
value: "3"
|
||||
- name: PREDICTIONSBATCH_WORKFLOW_POLLER_BEHAVIOUR_INITIAL
|
||||
value: "5"
|
||||
- name: PREDICTIONSBATCH_WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM
|
||||
value: "15"
|
||||
- name: PREDICTIONSBATCH_ACTIVITY_POLLER_BEHAVIOUR_MINIMUM
|
||||
value: "3"
|
||||
- name: PREDICTIONSBATCH_ACTIVITY_POLLER_BEHAVIOUR_INITIAL
|
||||
value: "10"
|
||||
- name: PREDICTIONSBATCH_ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM
|
||||
value: "30"
|
||||
|
||||
- name: MINIMALRETRAIN_MAX_CONCURRENT_ACTIVITIES
|
||||
value: "5"
|
||||
- name: MINIMALRETRAIN_ACTIVITY_EXECUTOR_MAX_WORKERS
|
||||
value: "5"
|
||||
- name: MINIMALRETRAIN_MAX_CONCURRENT_LOCAL_ACTIVITIES
|
||||
value: "5"
|
||||
- name: MINIMALRETRAIN_MAX_CACHED_WORKFLOWS
|
||||
value: "5"
|
||||
- name: MINIMALRETRAIN_WORKFLOW_POLLER_BEHAVIOUR_MINIMUM
|
||||
value: "5"
|
||||
- name: MINIMALRETRAIN_WORKFLOW_POLLER_BEHAVIOUR_INITIAL
|
||||
value: "5"
|
||||
- name: MINIMALRETRAIN_WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM
|
||||
value: "5"
|
||||
- name: MINIMALRETRAIN_ACTIVITY_POLLER_BEHAVIOUR_MINIMUM
|
||||
value: "5"
|
||||
- name: MINIMALRETRAIN_ACTIVITY_POLLER_BEHAVIOUR_INITIAL
|
||||
value: "5"
|
||||
- name: MINIMALRETRAIN_ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM
|
||||
value: "5"
|
||||
|
||||
- name: PI_WEB_API_BASE_URL
|
||||
value: "https://pivision.votorantimcimentos.com/piwebapi"
|
||||
- name: PI_WEB_API_AUTH_TYPE
|
||||
value: "basic"
|
||||
- name: PI_WEB_API_AUTH_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: pi-web-api-auth-token
|
||||
key: token
|
||||
|
||||
# Thread-pool size for non-runtime workers that also use prepare_worker.
|
||||
- name: SIMPLEMETRICS_ACTIVITY_EXECUTOR_MAX_WORKERS
|
||||
value: "20"
|
||||
- name: DRIFT_ACTIVITY_EXECUTOR_MAX_WORKERS
|
||||
value: "20"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Runtimes configuration
|
||||
# -----------------------------------------------------------------------------
|
||||
# IMPORTANT:
|
||||
# - The runtime name is used by the worker bootstrap to resolve plugins and task queues.
|
||||
# - Keep runtime names in sync with the plugin-store runtime names.
|
||||
runtimes:
|
||||
basic:
|
||||
replicas: 1
|
||||
legacy:
|
||||
replicas: 1
|
||||
env:
|
||||
- name: "GITHUB_BRANCH"
|
||||
value: "main"
|
||||
- name: "MLFLOW_HOST"
|
||||
value: "http://sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local"
|
||||
- name: "MLFLOW_PORT"
|
||||
value: "80"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Chart-level configuration (applies to all runtimes)
|
||||
# -----------------------------------------------------------------------------
|
||||
imagePullSecrets:
|
||||
- name: docker-hub-secret
|
||||
|
||||
nameOverride: *projectName
|
||||
fullnameOverride: *projectName
|
||||
|
||||
serviceAccount:
|
||||
create: true
|
||||
automount: true
|
||||
annotations: {}
|
||||
name: *projectName
|
||||
|
||||
podAnnotations: {}
|
||||
podLabels: {}
|
||||
|
||||
podSecurityContext: {}
|
||||
securityContext: {}
|
||||
|
||||
volumes: []
|
||||
volumeMounts: []
|
||||
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
|
||||
services:
|
||||
sdk-metrics:
|
||||
enabled: true
|
||||
type: ClusterIP
|
||||
port: 9091
|
||||
targetPort: 9091
|
||||
name: sdk-metrics
|
||||
metrics:
|
||||
enabled: true
|
||||
type: ClusterIP
|
||||
port: 9090
|
||||
targetPort: 9090
|
||||
name: metrics
|
||||
|
||||
# Configuração do ServiceMonitor para o Prometheus Operator
|
||||
# ref: https://github.com/prometheus-operator/prometheus-operator
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
endpoints:
|
||||
- port: metrics
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
relabelings: []
|
||||
- port: sdk-metrics
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
relabelings: []
|
||||
additionalLabels:
|
||||
release: kube-prometheus-stack
|
||||
|
||||
ssh:
|
||||
enabled: true
|
||||
secretName: git-ssh-key-sientia-laborious-worker
|
||||
sshPath: /mnt/.ssh
|
||||
knownHostsPath: /mnt/known_hosts
|
||||
|
||||
# kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=<pwd>
|
||||
#
|
||||
# helm upgrade --install sientia-laborious-worker /home/grezewave/Documents/projects/sientia/sientia-core-applications/sientia-module -n sientia --create-namespace -f ./values.yaml
|
||||
|
||||
# kubectl create secret generic git-ssh-key-sientia-laborious-worker \
|
||||
# --namespace sientia \
|
||||
# --from-file=ssh-privatekey=git_key \
|
||||
# --type=kubernetes.io/ssh-auth
|
||||
|
||||
# helm upgrade --install sientia-laborious-worker /home/grezewave/Documents/projects/sientia/sientia-core-applications/sientia-module -n sientia --create-namespace -f ./values.yaml
|
||||
Reference in New Issue
Block a user