From 638d5b70b418458890595f69966e2a56813d6873 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 15 May 2026 15:28:28 -0300 Subject: [PATCH 01/64] SIENTIAPDE-1811 Enhance OPC UA communication and metrics tracking - Updated README.md to include new OPC UA Communication section and detailed metrics for session and write diagnostics. - Added new metrics in laborious/metrics.py for tracking OPC UA session states and write attempts. - Refactored OPC activity in laborious/activities/opc.py to handle session errors and improve error reporting. - Updated e2e tests to cover new scenarios for OPC session/channel errors and reconnect handling. - Modified .gitignore to include relatorio files and mlruns directory. - Added ipykernel to requirements-dev.txt for Jupyter notebook support. --- .gitignore | 4 +- README.md | 24 +- docs/opc-communication.md | 160 ++++ e2e/conftest.py | 7 + e2e/helpers.py | 13 +- e2e/scenarios.md | 42 + e2e/test_predictions_batch_format_export.py | 137 +++- inter_arrival.py | 25 + laborious/activities/opc.py | 127 +++- laborious/metrics.py | 34 + laborious/utils/repository/opc_repository.py | 715 +++++++++++++----- laborious/worker/worker.py | 2 +- requirements-dev.txt | 1 + tests/laborious/activities/test_opc.py | 176 ++++- .../utils/repository/test_opc_repository.py | 327 +++++--- 15 files changed, 1370 insertions(+), 424 deletions(-) create mode 100644 docs/opc-communication.md create mode 100644 inter_arrival.py diff --git a/.gitignore b/.gitignore index 7cd11b3..c35c913 100644 --- a/.gitignore +++ b/.gitignore @@ -51,4 +51,6 @@ catboost_info/ .ruff_cache/ .mypy_cache/ -mlruns/ \ No newline at end of file +mlruns/ + +relatorio* \ No newline at end of file diff --git a/README.md b/README.md index b63be57..0de066f 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ A comprehensive, Temporal-based ML orchestration system for industrial data proc - [Prediction Operation Metrics](#prediction-operation-metrics) - [OPC Export Metrics](#opc-export-metrics) - [Data Quality Metrics](#data-quality-metrics) +- [OPC UA Communication](#opc-ua-communication) - [Configuration](#configuration-1) - [Environment Variables](#environment-variables) - [OPC Configuration](#opc-configuration) @@ -164,7 +165,7 @@ Laborious uses a Temporal-based architecture with strong separation of concerns - `connectors_config.py`: Env-driven configuration builders - `models/minio_dataframe_payload.py`: MinIO-offloaded DataFrame payload model - `repository/model_repository.py`: MLFlow operations and retraining -- `repository/opc_repository.py`: OPC communication and writes +- `repository/opc_repository.py`: OPC UA client, writes, session recovery (see [OPC UA Communication](#opc-ua-communication)) - `repository/minio_manager.py`: MinIO object storage operations - `filters/conditional_filters.py` and `filters/mlflow_filters.py` @@ -779,6 +780,15 @@ The Laborious system exposes comprehensive Prometheus metrics for operational vi - Labels: `pod_id`, `model_name`, `workflow_name`, `opc_server_id` - Buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0] +OPC UA session and write diagnostics (Prometheus, `laborious/metrics.py`): + +- `opc_connections_initiated_total`, `opc_connections_failed_total`, `opc_connection_status` +- `opc_session_created_total`, `opc_session_closed_total`, `opc_session_revised_timeout_milliseconds` +- `opc_write_attempts_total` (label `result`: `OK` or exception name, e.g. `BadSessionIdInvalid`) +- `opc_write_inter_arrival_over_session_timeout_total` + +See [OPC UA Communication](#opc-ua-communication) for semantics, concurrency, and confidence codes **12** / **14**. + ### Data Quality Metrics - Filter pass/fail rates through notification system - MLFlow API response validation metrics @@ -810,7 +820,7 @@ The Laborious system exposes comprehensive Prometheus metrics for operational vi | `OPC_CERT_PATH` | OPC client certificate path | `None` | No | | `OPC_PRIVATE_KEY_PATH` | OPC private key path | `None` | No | | `OPC_SERVER_CERT_PATH` | OPC server certificate path | `None` | No | -| `OPC_RECONNECTION_INTERVAL` | OPC reconnection interval (ms) | `120` | No | +| `OPC_RECONNECTION_INTERVAL` | Minimum seconds between OPC reconnects | `120` | No | | `PI_WEB_API_BASE_URL` | PI Web API server base URL | `None` | No | | `PI_WEB_API_AUTH_TYPE` | PI Web API authentication type (basic/bearer) | `None` | No | | `PI_WEB_API_AUTH_TOKEN` | PI Web API authentication token | `None` | No | @@ -903,6 +913,12 @@ Legacy MinIO object layout (relative key): `training_datasets/{model_name}/{object_prefix}_{timestamp}.parquet` where `object_prefix` is sanitized (slashes replaced by underscores) to keep a stable model-level directory. +## OPC UA Communication + +Full reference: **[docs/opc-communication.md](docs/opc-communication.md)** (connection lifecycle, Tier-1 `Bad*` reconnect, connection lock / session readiness, metrics, PostgreSQL confidence **12** vs **14**, tests). + +Implementation plan: [`.cursor/plans/opc_bad_reconnect_ac4c6045.plan.md`](.cursor/plans/opc_bad_reconnect_ac4c6045.plan.md). + ### OPC Configuration For multiple OPC servers, use the `OPC_CONFIG` environment variable: @@ -1126,8 +1142,10 @@ laborious/ - Ensure proper connection pool configuration 4. **OPC Connection Failures** - - Verify OPC server is accessible + - See [docs/opc-communication.md](docs/opc-communication.md) + - Verify OPC server is accessible and `OPC_RECONNECTION_INTERVAL` is appropriate - Check certificate and key file paths + - Correlate `opc_write_attempts_total` with `opc_session_*` metrics; count session errors via `prediction_confidence = 14` - Review OPC server logs for connection issues 5. **PI Web API Connection Failures** diff --git a/docs/opc-communication.md b/docs/opc-communication.md new file mode 100644 index 0000000..ce612b9 --- /dev/null +++ b/docs/opc-communication.md @@ -0,0 +1,160 @@ +# OPC UA communication (Laborious) + +Laborious exports predictions to OPC UA servers through `OpcRepository` ([`laborious/utils/repository/opc_repository.py`](../laborious/utils/repository/opc_repository.py)) and the Temporal activity layer in [`laborious/activities/opc.py`](../laborious/activities/opc.py). + +Implementation plan for session/channel recovery on Tier-1 `Bad*` errors: [`.cursor/plans/opc_bad_reconnect_ac4c6045.plan.md`](../.cursor/plans/opc_bad_reconnect_ac4c6045.plan.md). + +## Architecture + +```text +Worker (long-lived) + └── OpcRepository per OPC server id (from OPC_CONFIG / env) + ├── connect / disconnect / validate_connection (read-only) + ├── _connect_locked / _reconnect_locked (under _connection_lock) + ├── write_data (single attempt per call) + └── background reconnect on Tier-1 Bad* + +Temporal activity write_opc_data + └── OPC.manage_output_tags → write_data per tag (sequential per activity) +``` + +One worker process holds one `OpcRepository` instance per configured server. Multiple Temporal activities can call `write_data` concurrently on the same repository. + +## Connection lifecycle + +| Phase | Behavior | +|-------|----------| +| Startup | `init_opc()` creates repositories and calls `connect()` → `_connect_locked()` | +| Steady state | `validate_connection()` is read-only (`protocol.state` only); `_session_ready` is checked in `write_data` | +| Tier-1 Bad* | `_start_reconnect_on_bad` → `_run_reconnect_on_bad` → `_reconnect_locked()` (respects `reconnection_interval`) | +| Write | `write_data()` checks `_session_ready`, validates, then one `get_node` + `write_value` | +| Shutdown | `close()` disconnects all repositories | + +### Session and channel timeouts + +Requested session and secure-channel lifetime: **10 minutes** (`OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS` in `opc_repository.py`). The server may revise these values; negotiated values are logged after connect and exposed as `opc_session_revised_timeout_milliseconds`. + +### Reconnection interval + +`OPC_RECONNECTION_INTERVAL` is in **seconds** (default `120`). It gates **background** reconnect after Tier-1 `Bad*` (`last_reconnection_time` is updated only in `_reconnect_locked()`). It limits load on the OPC server when many workflows fail at once. + +## Concurrency: connection lock and session readiness + +To allow **multiple concurrent writes** when the session is healthy, but **block all writes** while the connection is being torn down or re-established: + +| Primitive | Role | +|-----------|------| +| `_connection_lock` (`asyncio.Lock`) | Held for the entire `disconnect` → `connect` path. Only one connection-maintenance task at a time. | +| `_session_ready` (`asyncio.Event`) | Set when a session is ready for writes; cleared before reconnect starts and set again after a successful connect. | + +**Connection methods (caller holds `_connection_lock` for `_*_locked` helpers):** + +| Method | Role | +|--------|------| +| `_create_client()` | Create asyncua `Client` + optional `set_security`; raises if `client` already exists | +| `_open_session()` | `client.connect()` + metrics; raises if session already open or client missing | +| `_connect_locked()` | `_create_client()` (when needed) + `_open_session()`; raises if already connected | +| `_disconnect_locked()` | Teardown session and clear `client` | +| `_reconnect_locked()` | `_disconnect_locked()` + `_connect_locked()`; sets `last_reconnection_time` | + +Public `connect()` / `disconnect()` acquire the lock and call `_connect_locked()` / `_disconnect_locked()`. + +**Write path (`write_data`):** + +1. If `_session_ready` is cleared → **fail immediately** (`opc_error_kind=reconnect_in_progress`). +2. `validate_connection()` checks `protocol.state` only (read-only). +3. Single `get_node` + `write_value` (no retry). + +**Reconnect path (`_run_reconnect_on_bad`):** + +1. `_start_reconnect_on_bad` clears `_session_ready` and schedules the task when the interval allows. +2. `async with _connection_lock:` → `_reconnect_locked()`. +3. `_session_ready` is set on successful `_open_session()`. + +A second `_connect_locked()` while a session is already open raises `OpcSessionAlreadyConnectedError` (disconnect first). + +**asyncua note:** Concurrent `write_value` on the same session is only safe if the stack tolerates it. If production shows issues, serialize writes with an optional `asyncio.Semaphore(1)` while keeping the connection lock semantics above. + +**Future threads:** replace `asyncio.Lock` / `Event` with `threading` primitives or route all OPC I/O through one dedicated loop. + +## Tier-1 `Bad*` errors and reconnect + +When the server invalidates the session (e.g. `BadSessionIdInvalid`) but the client still sees transport as open, `write_data` fails once, records the OPC status in metrics, and **schedules** reconnect if: + +- The exception is a `UaStatusCodeError` whose name is in `RECONNECTABLE_OPC_BAD_NAMES` (see plan), and +- `reconnection_interval` has elapsed since `last_reconnection_time`, and +- No reconnect task is already running. + +There is **no write retry**: the failed export is not sent again in the same activity. + +## Prediction confidence and PostgreSQL comments + +| `prediction_confidence` | Meaning | +|-------------------------|---------| +| (unchanged) | Successful OPC export | +| **12** | Generic OPC write failure (`OPC_WRITTING_ERROR_CONFIDENCE`) | +| **14** | Tier-1 session/channel `Bad*` on export (`OPC_SESSION_BAD_CONFIDENCE`) | +| **14** | Write while reconnect in progress (`OPC_SESSION_BAD_CONFIDENCE`, comment `OPC UA reconnect in progress`) | +| **13** | PI Web API write failure (separate path) | + +Session/channel errors use a stable comment for counting: + +```text +OPC UA session/channel error: BadSessionIdInvalid +``` + +Reconnect-in-progress exports use: + +```text +OPC UA reconnect in progress +``` + +Example SQL: + +```sql +SELECT count(*) FROM predictions WHERE prediction_confidence = 14; +SELECT count(*) FROM predictions WHERE comments LIKE 'OPC UA session/channel error:%'; +``` + +## Prometheus metrics (`opc_*`) + +Defined in [`laborious/metrics.py`](../laborious/metrics.py). Do not rename in production without a dashboard migration. + +| Metric | Purpose | +|--------|---------| +| `opc_connections_initiated_total` | Connection attempts | +| `opc_connections_failed_total` | Failed connects | +| `opc_connection_status` | Gauge 1=connected, 0=disconnected | +| `opc_session_created_total` | Session established after connect | +| `opc_session_closed_total` | Disconnect initiated | +| `opc_session_revised_timeout_milliseconds` | Negotiated session timeout (ms) | +| `opc_write_attempts_total` | Per write; label `result` = `OK` or exception name | +| `opc_write_inter_arrival_over_session_timeout_total` | Successful writes spaced longer than revised session timeout | + +Legacy activity metrics: `laborious_prediction_opc_writing_count`, `laborious_prediction_opc_writing_response_time_monitor`. + +## Environment variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `OPC_CONFIG` | — | JSON map of server configs (overrides single-server env) | +| `OPC_ID` | `1` | Server id | +| `OPC_URL` | `opc.tcp://localhost:4840` | Endpoint | +| `OPC_SERVER_NAME` | `default_server` | Label for metrics/logs | +| `OPC_SERVER_URI` | same as URL | Application URI / cert SAN | +| `OPC_CERT_PATH` | — | Client certificate (secure mode) | +| `OPC_PRIVATE_KEY_PATH` | — | Client private key | +| `OPC_SERVER_CERT_PATH` | — | Server certificate | +| `OPC_RECONNECTION_INTERVAL` | `120` | Minimum seconds between reconnects | + +## Operations checklist + +- Correlate `BadSessionIdInvalid` in `opc_write_attempts_total` with `opc_session_closed_total` / `opc_session_created_total` (reconnect may finish after the row is stored with confidence 14). +- Use confidence **14** and comment prefix for session invalidation rates; use **12** for other OPC failures. +- Respect `OPC_RECONNECTION_INTERVAL` under parallel load; bursts of confidence 14 are expected until the next successful cycle. + +## Related tests + +- Unit: [`tests/laborious/utils/repository/test_opc_repository.py`](../tests/laborious/utils/repository/test_opc_repository.py) +- Unit: [`tests/laborious/activities/test_opc.py`](../tests/laborious/activities/test_opc.py) +- E2E: [`e2e/test_predictions_batch_format_export.py`](../e2e/test_predictions_batch_format_export.py), scenarios in [`e2e/scenarios.md`](../e2e/scenarios.md) diff --git a/e2e/conftest.py b/e2e/conftest.py index 7825335..f29ec0d 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -2,7 +2,14 @@ Pytest configuration and fixtures for E2E tests. """ +import sys from unittest.mock import AsyncMock, MagicMock, patch + +# E2E workflows under test do not run ModelAnalysis; stub before Activities import. +_model_analysis_module = MagicMock() +_model_analysis_module.ModelAnalysis = MagicMock +sys.modules.setdefault('sientia', MagicMock()) +sys.modules.setdefault('sientia.ModelAnalysis', _model_analysis_module) from io import BytesIO import pandas as pd diff --git a/e2e/helpers.py b/e2e/helpers.py index 1aaa4f2..bb404ca 100644 --- a/e2e/helpers.py +++ b/e2e/helpers.py @@ -64,7 +64,8 @@ def assert_prediction( prediction: float = 0.5, prediction_confidence: int | Decimal = 0, prediction_status: str = 'Good', - comments: str = '', + comments: str | None = None, + comments_contains: str | None = None, ) -> None: """ Assert exactly one prediction row exists for model_id with expected columns. @@ -75,7 +76,8 @@ def assert_prediction( prediction: Expected prediction value. prediction_confidence: Expected confidence (int or Decimal for numeric column). prediction_status: Expected status string. - comments: Expected comments string. + comments: Expected exact comments string (optional). + comments_contains: Substring expected in comments when queued (optional). """ import pytest @@ -98,7 +100,12 @@ def assert_prediction( str(prediction_confidence) ), f'Expected prediction_confidence={prediction_confidence}, got {row[2]}' assert row[3] == prediction_status, f"Expected prediction_status='{prediction_status}', got {row[3]}" - assert row[4] == comments, f"Expected comments='{comments}', got {row[4]}" + if comments is not None: + assert row[4] == comments, f"Expected comments='{comments}', got {row[4]}" + if comments_contains is not None: + assert comments_contains in row[4], ( + f"Expected comments to contain '{comments_contains}', got {row[4]}" + ) def assert_continue( diff --git a/e2e/scenarios.md b/e2e/scenarios.md index 5732cd3..03f58c2 100644 --- a/e2e/scenarios.md +++ b/e2e/scenarios.md @@ -495,6 +495,48 @@ These paths do **not** rely on Temporal activity retries for export failures: th --- +#### Scenario 3.2.4: OPC Session / Channel Bad* (Tier-1) +**Description**: OPC write fails with a Tier-1 session or channel status (e.g. `BadSessionIdInvalid`) while transport may still appear open on the client + +**Input**: +- Valid prediction and OPC output config +- Mock or server returning Tier-1 `UaStatusCodeError` on write (no write retry in the same activity) + +**Expected Behavior**: +- `write_opc_data` fails forward for affected tags; background reconnect may be scheduled if `OPC_RECONNECTION_INTERVAL` allows +- Workflow **completes** +- PostgreSQL row uses **`prediction_confidence` 14** and comment prefix `OPC UA session/channel error:` (including OPC status name) +- `opc_write_attempts_total` records `result=BadSessionIdInvalid` (or matching status); no second write attempt in the same activity + +**Assertions**: +- Workflow completes +- `prediction_confidence = 14` +- `comments` matches `OPC UA session/channel error:%` +- Generic OPC error confidence **12** is not used for this case + +**Reference**: [docs/opc-communication.md](../docs/opc-communication.md), plan `.cursor/plans/opc_bad_reconnect_ac4c6045.plan.md` + +--- + +#### Scenario 3.2.5: OPC Write Blocked During Reconnect +**Description**: A write is attempted while the repository is reconnecting (session not ready) + +**Input**: +- Valid prediction +- Simulated slow reconnect (e.g. delayed `connect`) or concurrent writes where the first triggers reconnect + +**Expected Behavior**: +- Second write (or parallel write) is rejected **immediately** when reconnect is in progress or `_session_ready` is cleared — **without** calling `write_value` +- No wait/sleep on the write path; no duplicate `connect` from parallel writers (connection lock) +- `prediction_confidence = 14`, `comments = OPC UA reconnect in progress` (distinguish from Tier-1 `Bad*` via comment prefix in SQL) + +**Assertions**: +- At most one reconnect sequence (`disconnect` + `connect`) for the overlapping window +- No write retry after failure +- Tests in `test_opc_repository` (unit) and optional e2e in `test_predictions_batch_format_export.py` + +--- + #### Scenario 3.2.3: PI Web API Partial Write Error **Description**: Two prediction tags attempt to be written to PI Web API, but only one succeeds diff --git a/e2e/test_predictions_batch_format_export.py b/e2e/test_predictions_batch_format_export.py index 35543f8..5582d0c 100644 --- a/e2e/test_predictions_batch_format_export.py +++ b/e2e/test_predictions_batch_format_export.py @@ -153,7 +153,6 @@ async def test_scenario_3_1_1_default_prediction_export( 'addr_1', 0, 'float', - ANY, { 'model_id': 311, 'model_name': 'test_model', @@ -165,7 +164,6 @@ async def test_scenario_3_1_1_default_prediction_export( 'addr_2', 2, 'float', - ANY, { 'model_id': 311, 'model_name': 'test_model', @@ -252,20 +250,28 @@ async def test_scenario_3_1_2_export_with_opc_only( opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data) opc_write_data.assert_has_calls( [ - call('addr_1', 0.5, 'float', ANY, - { - 'model_id': 312, - 'model_name': 'test_model', - 'schedule_name': 'test-schedule', - 'workflow_name': 'predictions_batch', - }), - call('addr_2', 0, 'float', ANY, - { - 'model_id': 312, - 'model_name': 'test_model', - 'schedule_name': 'test-schedule', - 'workflow_name': 'predictions_batch', - }), + call( + 'addr_1', + 0.5, + 'float', + { + 'model_id': 312, + 'model_name': 'test_model', + 'schedule_name': 'test-schedule', + 'workflow_name': 'predictions_batch', + }, + ), + call( + 'addr_2', + 0, + 'float', + { + 'model_id': 312, + 'model_name': 'test_model', + 'schedule_name': 'test-schedule', + 'workflow_name': 'predictions_batch', + }, + ), ] ) @@ -500,20 +506,28 @@ async def test_scenario_3_1_5_export_without_transformed_data( opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data) opc_write_data.assert_has_calls( [ - call('addr_1', 0.5, 'float', ANY, - { - 'model_id': 315, - 'model_name': 'test_model', - 'schedule_name': 'test-schedule', - 'workflow_name': 'predictions_batch', - }), - call('addr_2', 0, 'float', ANY, - { - 'model_id': 315, - 'model_name': 'test_model', - 'schedule_name': 'test-schedule', - 'workflow_name': 'predictions_batch', - }), + call( + 'addr_1', + 0.5, + 'float', + { + 'model_id': 315, + 'model_name': 'test_model', + 'schedule_name': 'test-schedule', + 'workflow_name': 'predictions_batch', + }, + ), + call( + 'addr_2', + 0, + 'float', + { + 'model_id': 315, + 'model_name': 'test_model', + 'schedule_name': 'test-schedule', + 'workflow_name': 'predictions_batch', + }, + ), ] ) @@ -646,7 +660,68 @@ async def test_scenario_3_2_2_opc_write_error( prediction_confidence=12, comments='Some data could not be written to OPC servers', ) - + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_3_2_4_opc_session_bad_error( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + test_activities: Activities, + postgres_engine, +): + """ + Scenario 3.2.4: OPC session/channel Tier-1 Bad* (e.g. BadSessionIdInvalid). + + PostgreSQL stores prediction_confidence 14 and a stable session error comment. + """ + client = temporal_test_env.client + + model_id = 324 + + insert_sample_data(postgres_engine, model_id, [23.5, 78.2]) + + opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data) + opc_write_data.return_value = ( + False, + { + 'notification_id': 'OPC_WRITE_DATA_ERROR_1', + 'message': 'BadSessionIdInvalid', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + 'attachment_content': 'BadSessionIdInvalid', + 'opc_error_kind': 'session_bad', + 'opc_status': 'BadSessionIdInvalid', + }, + ) + + input_data = get_base_input_data(model_id) + input_data['opc_output_config'] = { + '1': { + 'prediction_tags': { + 'addr_1': { + 'data_type': 'float', + } + }, + 'confidence_tags': { + 'addr_2': { + 'data_type': 'float', + } + }, + } + } + input_data['pi_web_api_output_config'] = None + + await start_and_await_workflow( + client, PredictionsBatch.run, input_data, make_workflow_id('test-opc-session-bad') + ) + + assert_prediction( + postgres_engine, + model_id, + prediction_confidence=14, + comments='OPC UA session/channel error: BadSessionIdInvalid', + ) @pytest.mark.asyncio diff --git a/inter_arrival.py b/inter_arrival.py new file mode 100644 index 0000000..bce9cee --- /dev/null +++ b/inter_arrival.py @@ -0,0 +1,25 @@ +# %% + +# Load logs.txt +with open('logs.txt', 'r') as file: + lines = file.readlines() + +# %% +import re +# Grep "inter-arrival_s=number" with regex +intervals = [] +for line in lines: + match = re.search(r'inter-arrival_s=([0-9.]+)', line) + if match: + intervals.append(float(match.group(1))) +# %% + +print(intervals) +# %% +import matplotlib.pyplot as plt +plt.plot(intervals) +plt.ylabel('Inter-arrival time (s)') +plt.xlabel('Sample') +plt.title('Inter-arrival time distribution') +plt.show() +# %% diff --git a/laborious/activities/opc.py b/laborious/activities/opc.py index da36793..524237e 100644 --- a/laborious/activities/opc.py +++ b/laborious/activities/opc.py @@ -15,6 +15,16 @@ with workflow.unsafe.imports_passed_through(): from laborious.utils.repository.opc_repository import OpcRepository OPC_WRITTING_ERROR_CONFIDENCE = 12 +OPC_SESSION_BAD_CONFIDENCE = 14 +OPC_SESSION_BAD_COMMENT_PREFIX = 'OPC UA session/channel error:' +OPC_WRITTING_ERROR_MESSAGE = 'Some data could not be written to OPC servers' +OPC_RECONNECT_IN_PROGRESS_COMMENT = 'OPC UA reconnect in progress' +OPC_COMMENT_SEPARATOR = ' | ' + + +def _opc_session_bad_comment(opc_status: str | None) -> str: + status = opc_status or 'Unknown' + return f'{OPC_SESSION_BAD_COMMENT_PREFIX} {status}' class OPC(SientiaMonitoring): @@ -118,29 +128,18 @@ class OPC(SientiaMonitoring): data_type: str, tag_type: str, metadata: dict[str, Any], - ) -> float | None: + ) -> tuple[float | None, dict[str, Any] | None]: """ Write data to a specific OPC server tag with comprehensive error handling. - This method provides a secure and reliable way to write data to OPC servers - with automatic error handling, notification integration, and detailed logging. - It validates server availability before attempting write operations and - provides comprehensive error reporting for operational monitoring. - - Args: - - server_id (str): The id of the OPC server. - - tag (str): The tag to write to. - - data (Any): The data to write. - - data_type (str): The data type. - - tag_type (str): The tag type. - - Returns: - - bool: True if the data was written successfully, False otherwise. + Return: + tuple[float | None, dict[str, Any] | None]: Response time on success, or + (None, error info_data) on repository failure. """ try: is_success, info_data = await self.opc_repository[server_id].write_data( - tag, data, data_type, self.logger, metadata + tag, data, data_type, metadata ) if not is_success: await self.send_notification_async( @@ -151,8 +150,8 @@ class OPC(SientiaMonitoring): level=info_data.get('level', NotificationLevel.ERROR), attachment_content=info_data.get('attachment_content', None), ) - return None - return info_data['response_time'] + return None, info_data + return info_data['response_time'], None except Exception as e: trace = traceback.format_exc() await self.send_notification_async( @@ -205,7 +204,7 @@ class OPC(SientiaMonitoring): config: dict[str, Any], data: DataFrame, metadata: dict[str, Any], - ) -> tuple[bool, dict[str, float | None]]: + ) -> tuple[bool, dict[str, float | None], bool, str | None, bool]: """ Manage the writing of prediction and confidence data to OPC server tags. @@ -234,10 +233,13 @@ class OPC(SientiaMonitoring): """ response_times: dict[str, float | None] = {} + session_bad_seen = False + session_bad_status: str | None = None + reconnect_in_progress_seen = False if 'prediction_tags' in config: for tag, tag_config in config['prediction_tags'].items(): - response_time = await self.write_data( + response_time, error_info = await self.write_data( server_id=server_id, tag=tag, data=data.head(1)['prediction'].values[0], @@ -245,6 +247,13 @@ class OPC(SientiaMonitoring): tag_type='prediction', metadata=metadata, ) + if error_info: + kind = error_info.get('opc_error_kind') + if kind == 'session_bad': + session_bad_seen = True + session_bad_status = error_info.get('opc_status', session_bad_status) + elif kind == 'reconnect_in_progress': + reconnect_in_progress_seen = True if response_time is not None: self.info( f'Prediction data written to OPC server {server_id} for tag {tag}.', @@ -254,7 +263,7 @@ class OPC(SientiaMonitoring): if 'confidence_tags' in config: for tag, tag_config in config['confidence_tags'].items(): - response_time = await self.write_data( + response_time, error_info = await self.write_data( server_id=server_id, tag=tag, data=data.head(1)['prediction_confidence'].values[0], @@ -262,6 +271,13 @@ class OPC(SientiaMonitoring): tag_type='confidence', metadata=metadata, ) + if error_info: + kind = error_info.get('opc_error_kind') + if kind == 'session_bad': + session_bad_seen = True + session_bad_status = error_info.get('opc_status', session_bad_status) + elif kind == 'reconnect_in_progress': + reconnect_in_progress_seen = True if response_time is not None: self.info( f'Confidence data written to OPC server {server_id} for tag {tag}.', @@ -271,7 +287,13 @@ class OPC(SientiaMonitoring): success = None not in response_times.values() - return success, response_times + return ( + success, + response_times, + session_bad_seen, + session_bad_status, + reconnect_in_progress_seen, + ) @activity.defn(name='write_opc_data') async def write_opc_data( @@ -301,6 +323,9 @@ class OPC(SientiaMonitoring): self.info(f'Data to write: {data.size} rows', metadata) success = True + session_bad_seen = False + session_bad_status: str | None = None + reconnect_in_progress_seen = False metrics: dict[str, dict[str, float | None]] = {} @@ -309,22 +334,48 @@ class OPC(SientiaMonitoring): success = False continue - local_success, local_response_times = await self.manage_output_tags( - server_id, config, data, metadata - ) + ( + local_success, + local_response_times, + local_session_bad, + local_status, + local_reconnect_in_progress, + ) = await self.manage_output_tags(server_id, config, data, metadata) metrics[server_id] = local_response_times local_count = len(local_response_times) success = success and local_success + if local_session_bad: + session_bad_seen = True + session_bad_status = local_status or session_bad_status + if local_reconnect_in_progress: + reconnect_in_progress_seen = True self.info( f'Process completed for OPC server {server_id}: {local_count} of {len(config.get("prediction_tags", []))} prediction tags and {len(config.get("confidence_tags", []))} confidence tags', metadata, ) - return self.process_confidence(data, success, metadata), metrics + return ( + self.process_confidence( + data, + success, + metadata, + session_bad=session_bad_seen, + opc_status=session_bad_status, + reconnect_in_progress=reconnect_in_progress_seen, + ), + metrics, + ) def process_confidence( - self, data: DataFrame, success: bool, metadata: dict[str, Any] + self, + data: DataFrame, + success: bool, + metadata: dict[str, Any], + *, + session_bad: bool = False, + opc_status: str | None = None, + reconnect_in_progress: bool = False, ) -> dict[Hashable, Any]: """ Process prediction confidence based on OPC write operation success. @@ -352,16 +403,26 @@ class OPC(SientiaMonitoring): This allows downstream systems to handle data quality appropriately. """ - message = 'Some data could not be written to OPC servers' - if not success: - data['prediction_confidence'] = OPC_WRITTING_ERROR_CONFIDENCE - data['comments'] = message + comment_parts: list[str] = [] + confidence = OPC_WRITTING_ERROR_CONFIDENCE + + if session_bad: + comment_parts.append(_opc_session_bad_comment(opc_status)) + confidence = OPC_SESSION_BAD_CONFIDENCE + if reconnect_in_progress: + comment_parts.append(OPC_RECONNECT_IN_PROGRESS_COMMENT) + confidence = OPC_SESSION_BAD_CONFIDENCE + if not comment_parts: + comment_parts.append(OPC_WRITTING_ERROR_MESSAGE) + + comments = OPC_COMMENT_SEPARATOR.join(comment_parts) + data['prediction_confidence'] = confidence + data['comments'] = comments self.debug( - f'{message}, setting confidence to {OPC_WRITTING_ERROR_CONFIDENCE}.', + f'OPC write issues, confidence={confidence}, comments={comments}', metadata, ) - else: self.debug('Data written to OPC servers successfully.', metadata) diff --git a/laborious/metrics.py b/laborious/metrics.py index 776b826..74a9189 100644 --- a/laborious/metrics.py +++ b/laborious/metrics.py @@ -89,6 +89,40 @@ OPC_CONNECTION_STATUS = Gauge( ['pod_id', 'server_name', 'server_url'], ) +_OPC_SESSION_DEBUG_LABELS = ['pod_id', 'server_name', 'runtime', 'opc_server_id', 'session_id'] + +OPC_SESSION_CREATED_TOTAL = Counter( + 'opc_session_created_total', + 'OPC UA sessions established (after successful connect)', + _OPC_SESSION_DEBUG_LABELS, +) + +OPC_SESSION_CLOSED_TOTAL = Counter( + 'opc_session_closed_total', + 'OPC UA client disconnects completed (session tear-down initiated)', + _OPC_SESSION_DEBUG_LABELS, +) + +OPC_SESSION_REVISED_TIMEOUT_MS = Gauge( + 'opc_session_revised_timeout_milliseconds', + 'Server-revised OPC UA session timeout (RevisedSessionTimeout) in ms after connect', + _OPC_SESSION_DEBUG_LABELS, +) + +OPC_WRITE_ATTEMPT_LABELS = [*_OPC_SESSION_DEBUG_LABELS, 'model_id', 'model_name', 'result'] + +OPC_WRITE_ATTEMPTS_TOTAL = Counter( + 'opc_write_attempts_total', + 'OPC UA write attempts with session and outcome (result=OK or exception class name)', + OPC_WRITE_ATTEMPT_LABELS, +) + +OPC_WRITE_INTER_ARRIVAL_OVER_SESSION_TIMEOUT_TOTAL = Counter( + 'opc_write_inter_arrival_over_session_timeout_total', + 'Successful writes where seconds since the previous successful write exceeded RevisedSessionTimeout (ms)', + _OPC_SESSION_DEBUG_LABELS, +) + # ================== Model metrics ================== MODEL_READ_LAG = Histogram( diff --git a/laborious/utils/repository/opc_repository.py b/laborious/utils/repository/opc_repository.py index 385b256..b164ce1 100644 --- a/laborious/utils/repository/opc_repository.py +++ b/laborious/utils/repository/opc_repository.py @@ -9,6 +9,7 @@ from typing import Any from asyncua import Client from asyncua.crypto.security_policies import SecurityPolicyBasic256 from asyncua.ua import DataValue, Variant, VariantType +from asyncua.ua.uaerrors import UaStatusCodeError from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger @@ -17,6 +18,119 @@ from sientia_do.observability.sientia_monitoring import SientiaMonitoring from laborious import metrics +# Requested session and secure channel lifetime (ms) before server revision; 10 minutes. +OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS = 10 * 60 * 1000 + + +class OpcClientAlreadyExistsError(RuntimeError): + """Raised when _create_client is called while self.client is already set.""" + + +class OpcSessionAlreadyConnectedError(RuntimeError): + """Raised when _open_session is called while a UA session is already open.""" + + +class OpcClientNotInitializedError(RuntimeError): + """Raised when _open_session is called before _create_client.""" + + +RECONNECTABLE_OPC_BAD_NAMES: frozenset[str] = frozenset( + { + 'BadSessionIdInvalid', + 'BadSessionClosed', + 'BadSessionNotActivated', + 'BadSecureChannelIdInvalid', + 'BadSecureChannelClosed', + 'BadSecureChannelTokenUnknown', + 'BadTcpSecureChannelUnknown', + 'BadServerNotConnected', + 'BadConnectionClosed', + 'BadDisconnect', + 'BadConnectionRejected', + 'BadCommunicationError', + 'BadRequestInterrupted', + 'BadUnknownResponse', + 'BadTimeout', + 'BadRequestTimeout', + 'BadSequenceNumberInvalid', + 'BadSequenceNumberUnknown', + 'BadSecurityModeInsufficient', + 'BadRequestHeaderInvalid', + 'BadInvalidState', + } +) + + +def _opc_authentication_token_str(client: Client | None) -> str: + """ + Serialize the current OPC UA authentication token (session handle) for logging and metrics. + + Return: + str: Token string, or "unknown" if unavailable. + """ + if client is None: + return 'unknown' + try: + proto = client.uaclient.protocol + if proto is None: + return 'unknown' + tok = getattr(proto, 'authentication_token', None) + if tok is None: + return 'unknown' + return str(tok) + except Exception: + return 'unknown' + + +def _opc_status_from_exception(exc: BaseException) -> str: + """ + Resolve OPC UA status name from an exception, including chained UaStatusCodeError causes. + + Args: + exc (BaseException): Raised error from asyncua. + + Return: + str: Status class name or generic Python exception name. + """ + current: BaseException | None = exc + while current is not None: + if isinstance(current, UaStatusCodeError): + return type(current).__name__ + current = current.__cause__ + return type(exc).__name__ + + +def is_reconnectable_opcua_bad(exc: BaseException) -> bool: + """ + Return whether the exception is a Tier-1 OPC UA Bad* that should trigger reconnect. + + Args: + exc (BaseException): Raised error from get_node or write_value. + + Return: + bool: True if reconnect should be scheduled. + """ + return _opc_status_from_exception(exc) in RECONNECTABLE_OPC_BAD_NAMES + + +def _model_labels_from_write_metadata(metadata: dict[str, Any] | None) -> dict[str, str]: + """ + Extract model_id and model_name from write metadata for Prometheus labels. + + Args: + metadata (dict[str, Any] | None): Context passed into write_data; may omit keys. + + Return: + dict[str, str]: Labels model_id and model_name, defaulting to "unknown". + """ + if not metadata: + return {'model_id': 'unknown', 'model_name': 'unknown'} + return { + 'model_id': str(metadata.get('model_id', 'unknown')), + 'model_name': str(metadata.get('model_name', 'unknown')), + } + + data_type_map = { 'float': { 'converter': float, @@ -63,8 +177,6 @@ class OpcRepository(SientiaMonitoring): self.cert_path = cert_path self.private_key_path = private_key_path self.server_cert_path = server_cert_path - self.logger = logger - self.error_count = 0 self.reconnection_interval = reconnection_interval self.last_reconnection_time: None | datetime = None self.disconnection_interval = 10.0 @@ -79,27 +191,63 @@ class OpcRepository(SientiaMonitoring): 'workflow_name': 'opc_repository', 'schedule_name': '-', } + self._last_write_mono: float | None = None + self._connection_lock = asyncio.Lock() + self._session_ready = asyncio.Event() + self._reconnect_task: asyncio.Task[None] | None = None - async def set_security(self): + def _opc_debug_tags(self, session_id: str) -> dict[str, str]: + return { + 'pod_id': str(getattr(self, 'pod_id', 'unknown')), + 'server_name': self.server_name, + 'runtime': str(getattr(self, 'runtime', 'unknown')), + 'opc_server_id': self.id, + 'session_id': session_id, + } + + def _is_session_open(self) -> bool: """ - Configures the security settings for the OPC UA client. - This method sets up the security policy, certificates, and timeouts - required for establishing a secure connection with the OPC UA server. + Return whether the asyncua client has an open transport session. + + Return: + bool: True when protocol exists and is not closed. + """ + if self.client is None: + return False + try: + proto = self.client.uaclient.protocol + return proto is not None and proto.state != 'closed' + except Exception: + return False + + def _reconnection_window_elapsed(self) -> bool: + """ + Return whether enough time has passed since the last reconnect attempt. + + Return: + bool: True if a new reconnect is allowed. + """ + if self.last_reconnection_time is None: + return True + return ( + datetime.now() - self.last_reconnection_time + ).total_seconds() > self.reconnection_interval + + def _not_connected_error(self) -> dict[str, Any]: + return { + 'notification_id': f'OPC_CONNECTION_NOT_READY_{self.id}', + 'message': f'OPC server {self.id} is not connected', + 'block': 'opc_repository', + 'level': NotificationLevel.WARNING, + } + + async def set_security(self) -> None: + """ + Configure certificates and timeouts on the asyncua client. + Raises: - ValueError: If either the certificate path or private key path is not provided. - Attributes: - - cert_path (str): Path to the client's certificate file. - - private_key_path (str): Path to the client's private key file. - - server_cert_path (str, optional): Path to the server's certificate file. - - server_uri (str): The URI of the server to be used as the application URI. - - client (opcua.Client): The OPC UA client instance. - - logger (logging.Logger): Logger instance for logging information. - Security Settings: - - Security Policy: Basic256 - - Secure Channel Timeout: 10,000,000 ms - - Session Timeout: 10,000,000 ms + ValueError: If cert paths or client are missing. """ - if self.cert_path is None or self.private_key_path is None: raise ValueError( 'Certificate and private key paths must be provided for secure connection.' @@ -113,91 +261,107 @@ class OpcRepository(SientiaMonitoring): raise ValueError('Client must be initialized before setting security') self.client.application_uri = self.server_uri - self.logger.custom_info('Setting security...', self.metadata) + self.info('Setting security...', self.metadata) await self.client.set_security( SecurityPolicyBasic256, certificate=str(cert), private_key=str(private_key), server_certificate=str(server_cert) if server_cert else None, ) - self.client.secure_channel_timeout = 10000000 - self.client.session_timeout = 10000000 + self.client.secure_channel_timeout = OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS + self.client.session_timeout = OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS - async def connect(self) -> tuple[bool, dict[str, Any]]: + async def _create_client(self) -> None: """ - Establishes a connection to the OPC server. - This method initializes the OPC client using the provided URL and - sets up security if a certificate path is specified. It then - attempts to connect to the server and logs the connection status. + Instantiate the asyncua Client and apply security when configured. + + Caller must hold _connection_lock. Does not open a UA session. + Raises: - Exception: If the connection to the OPC server fails. + OpcClientAlreadyExistsError: If self.client is already set. """ + if self.client is not None: + raise OpcClientAlreadyExistsError( + f'OPC client already exists for server {self.id}; ' + 'call disconnect() before creating a new client' + ) - self.client = Client(self.url, timeout=10, watchdog_intervall=3600000) # type: ignore[attr-defined] - + self.client = Client(self.url, timeout=10, watchdog_intervall=50) # type: ignore[attr-defined] self.client.name = self.pod_id self.client.application_name = self.pod_id pod_uri = self.pod_id.replace('-', ':') self.client.application_uri = pod_uri self.client.product_uri = pod_uri - if self.cert_path: await self.set_security() - self.logger.custom_info( - f'Starting connection to OPC server {self.id}:{self.server_name}...', self.metadata - ) - return await self.try_connect() - async def try_connect(self) -> tuple[bool, dict[str, Any]]: + async def _open_session(self) -> tuple[bool, dict[str, Any]]: """ - Attempt to establish connection to the OPC server. + Open the OPC UA session on the existing client. - This method performs the actual connection attempt to the OPC server - and handles connection failures with comprehensive error reporting. - It updates reconnection timing and provides detailed error information - for operational monitoring and debugging. + Caller must hold _connection_lock. - Returns: - tuple[bool, dict[str, Any]]: Connection result - - bool: True if connection successful, False otherwise - - dict: Error information if connection failed + Raises: + OpcClientNotInitializedError: If self.client is None. + OpcSessionAlreadyConnectedError: If a session is already open. + + Return: + tuple[bool, dict[str, Any]]: Success flag and error payload on connect failure. """ + if self.client is None: + raise OpcClientNotInitializedError( + f'OPC client is not initialized for server {self.id}; ' + 'call _create_client() before opening a session' + ) + if self._is_session_open(): + raise OpcSessionAlreadyConnectedError( + f'OPC session already connected for server {self.id}; ' + 'call disconnect() before connecting again' + ) tags = { 'pod_id': self.pod_id, 'server_name': self.server_name, } await self.emit_metric(metrics.OPC_CONNECTIONS_TOTAL, tags) + try: - self.last_reconnection_time = datetime.now() - if self.client is None: - return False, { - 'notification_id': f'OPC_CONNECTION_ERROR_{self.id}', - 'message': 'Client is not initialized', - 'block': 'opc_repository', - 'level': NotificationLevel.ERROR, - } await self.client.connect() + session_id = _opc_authentication_token_str(self.client) + revised_session_timeout_ms = int(self.client.session_timeout) + revised_secure_channel_timeout_ms = int(self.client.secure_channel_timeout) + self.info( + f'OPC new session connected opc_server_id={self.id} session_id={session_id} ' + f'revised_session_timeout_ms={revised_session_timeout_ms} ' + f'revised_secure_channel_timeout_ms={revised_secure_channel_timeout_ms}', + self.metadata, + ) + await self.emit_metric( + metrics.OPC_SESSION_CREATED_TOTAL, self._opc_debug_tags(session_id) + ) + await self.emit_metric( + metric_object=metrics.OPC_SESSION_REVISED_TIMEOUT_MS, + method='set', + tags=self._opc_debug_tags(session_id), + value=revised_session_timeout_ms, + ) await self.emit_metric( metric_object=metrics.OPC_CONNECTION_STATUS, method='set', - tags={ - **tags, - 'server_url': self.url, - }, + tags={**tags, 'server_url': self.url}, value=1, ) + self._last_write_mono = None + self._session_ready.set() return True, {} + except Exception as e: - await self.disconnect() - + await self._disconnect_locked() trace = traceback.format_exc() - self.logger.custom_error(trace, self.metadata) - + self.error(trace, self.metadata) await self.emit_metric(metrics.OPC_CONNECTIONS_FAILED, tags) - return False, { 'notification_id': f'OPC_CONNECTION_ERROR_{self.id}', 'message': f'Failed to connect to OPC server: {e}', @@ -206,21 +370,45 @@ class OpcRepository(SientiaMonitoring): 'attachment_content': trace, } - async def disconnection_fallback(self) -> list: - """ - Tries 5 times to disconnect from the OPC UA server, with a delay of 100ms x try. + async def _connect_locked(self) -> tuple[bool, dict[str, Any]]: """ + Create the client when absent, then open a UA session. + Caller must hold _connection_lock. + + Raises: + OpcSessionAlreadyConnectedError: If a session is already open. + + Return: + tuple[bool, dict[str, Any]]: Result from _open_session on connect failure. + """ + if self._is_session_open(): + raise OpcSessionAlreadyConnectedError( + f'OPC session already connected for server {self.id}; ' + 'call disconnect() before connecting again' + ) + if self.client is None: + await self._create_client() + return await self._open_session() + + async def _disconnection_fallback(self) -> list[dict[str, Any]]: + """ + Try up to five times to disconnect from the OPC UA server. + """ assert self.client is not None - error_stack = [] + error_stack: list[dict[str, Any]] = [] for i in range(5): try: - self.logger.info(f'Disconnecting from OPC UA server, attempt {i + 1} of 5') + self.info( + f'Disconnecting from OPC UA server, attempt {i + 1} of 5', + self.metadata, + ) await self.client.disconnect() return [] except Exception as e: - self.logger.error( - f'Failed to disconnect from OPC UA server in attempt {i + 1} of 5: {e}' + self.error( + f'Failed to disconnect from OPC UA server in attempt {i + 1} of 5: {e}', + self.metadata, ) error_stack.append( { @@ -232,18 +420,26 @@ class OpcRepository(SientiaMonitoring): await asyncio.sleep(self.disconnection_interval * i) return error_stack - async def disconnect(self): + async def _disconnect_locked(self) -> None: """ - Gracefully disconnect from the OPC server. + Tear down the current session and client. - This method safely terminates the connection to the OPC server - and cleans up client resources. It handles disconnection errors - gracefully and ensures proper resource cleanup. + Caller must hold _connection_lock. """ + self._last_write_mono = None + self._session_ready.clear() + if self.client is None: return - errors = await self.disconnection_fallback() + session_id = _opc_authentication_token_str(self.client) + self.info( + f'OPC disconnecting opc_server_id={self.id} session_id={session_id}', + self.metadata, + ) + await self.emit_metric(metrics.OPC_SESSION_CLOSED_TOTAL, self._opc_debug_tags(session_id)) + + errors = await self._disconnection_fallback() if errors: await self.send_notification_async( metadata=self.metadata, @@ -254,7 +450,8 @@ class OpcRepository(SientiaMonitoring): attachment_content=json.dumps(errors, indent=4), ) else: - self.logger.warning(f'Disconnected from OPC server {self.id} successfully') + self.warning(f'Disconnected from OPC server {self.id} successfully', self.metadata) + await self.emit_metric( metric_object=metrics.OPC_CONNECTION_STATUS, method='set', @@ -265,186 +462,286 @@ class OpcRepository(SientiaMonitoring): }, value=0, ) - self.client = None + async def _reconnect_locked(self) -> tuple[bool, dict[str, Any]]: + """ + Close the current session and open a new one. + + Caller must hold _connection_lock. Records last_reconnection_time for interval gating. + + Return: + tuple[bool, dict[str, Any]]: Result from _connect_locked after teardown. + """ + self.last_reconnection_time = datetime.now() + await self._disconnect_locked() + return await self._connect_locked() + + async def connect(self) -> tuple[bool, dict[str, Any]]: + """ + Open an OPC UA session under the connection lock (worker initialization). + """ + async with self._connection_lock: + self.info( + f'Starting connection to OPC server {self.id}:{self.server_name}...', + self.metadata, + ) + return await self._connect_locked() + + async def disconnect(self) -> None: + """ + Gracefully disconnect from the OPC server under the connection lock. + """ + async with self._connection_lock: + await self._disconnect_locked() + async def validate_connection(self) -> tuple[bool, dict[str, Any]]: """ - Validate and maintain OPC server connection health. + Read-only check that the asyncua protocol is open. - This method performs comprehensive connection validation and - implements automatic reconnection logic for production reliability. - It handles various connection states and implements intelligent - reconnection strategies with error counting and timing controls. + Caller must ensure _session_ready before writing. Does not connect or reconnect. - Connection Validation: - 1. Checks client existence and connection state - 2. Implements error counting with automatic disconnection - 3. Enforces reconnection timing windows - 4. Provides detailed error reporting and notifications + Return: + tuple[bool, dict[str, Any]]: (True, {}) when open, otherwise (False, error). + """ + if self._is_session_open(): + return True, {} + self.error(f'OPC server {self.id} is not connected', self.metadata) + return False, self._not_connected_error() - Reconnection Strategy: - - Error Count Threshold: Disconnects after 5 consecutive errors - - Reconnection Window: Enforces minimum intervals between attempts - - Automatic Recovery: Attempts reconnection when conditions allow - - State Monitoring: Continuously monitors connection health + async def _start_reconnect_on_bad(self, opc_status: str, session_id: str) -> None: + """ + Schedule a background reconnect if interval and task state allow it. Args: - None - - Returns: - tuple[bool, dict[str, Any]]: Connection validation result - - bool: True if connection is healthy, False otherwise - - dict: Error information if validation fails + opc_status (str): OPC UA status name that triggered reconnect. + session_id (str): Session token before failure. """ - if self.client is None: - return await self.connect() + if not self._reconnection_window_elapsed(): + self.warning( + f'OPC reconnect skipped reason=reconnection_window opc_server_id={self.id} ' + f'opc_status={opc_status}', + self.metadata, + ) + return + if self._reconnect_task is not None and not self._reconnect_task.done(): + self.warning( + f'OPC reconnect skipped reason=in_progress opc_server_id={self.id} ' + f'opc_status={opc_status}', + self.metadata, + ) + return - # if self.error_count > 5: # NOSONAR - # self.logger.custom_warning( - # f'OPC server {self.id} will be disconnected due to multiple errors', self.metadata - # ) - # try: - # await self.disconnect() - # except Exception as e: - # trace = traceback.format_exc() - # self.logger.custom_error( - # f'Failed to disconnect from OPC server: {e}', self.metadata - # ) - # self.logger.custom_error(trace, self.metadata) - # self.logger.custom_info( - # f'Attempting to reconnect to OPC server {self.id}...', self.metadata - # ) - # return await self.connect() + self._session_ready.clear() + self.info( + f'OPC reconnect scheduled after opc_status={opc_status} opc_server_id={self.id} ' + f'old_session_id={session_id}', + self.metadata, + ) + self._reconnect_task = asyncio.create_task( + self._run_reconnect_on_bad(opc_status, session_id) + ) - # Check if client is connected using asyncua's connection state + async def _run_reconnect_on_bad(self, opc_status: str, session_id: str) -> None: + """ + Tear down and re-establish the OPC UA session under the connection lock. + + Args: + opc_status (str): OPC UA status that triggered reconnect. + session_id (str): Previous session token string. + """ try: - if ( - self.client.uaclient.protocol is None - or self.client.uaclient.protocol.state == 'closed' - ): - # OPC server is not connected - self.logger.custom_error(f'OPC server {self.id} is not connected', self.metadata) - if ( - self.last_reconnection_time is None - or (datetime.now() - self.last_reconnection_time).total_seconds() - > self.reconnection_interval - ): - await self.disconnect() - self.logger.custom_info( - f'Trying to reconnect to OPC server {self.id}...', self.metadata + async with self._connection_lock: + self.info( + f'OPC reconnect started opc_status={opc_status} opc_server_id={self.id} ' + f'old_session_id={session_id}', + self.metadata, + ) + await self._reconnect_locked() + except Exception: + self.error( + f'OPC reconnect task failed opc_server_id={self.id} opc_status={opc_status}', + self.metadata, + ) + self.error(traceback.format_exc(), self.metadata) + + async def _log_write_inter_arrival(self, session_id: str, node: str) -> None: + """ + Log elapsed wall time since the previous successful OPC write on this repository. + + Args: + session_id (str): Current OPC UA session token string. + node (str): Node id written in this operation. + """ + now = time.monotonic() + if self._last_write_mono is not None: + delta_s = now - self._last_write_mono + self.info( + f'OPC write inter-arrival_s={delta_s:.6f} opc_server_id={self.id} ' + f'session_id={session_id} node={node}', + self.metadata, + ) + if self.client is not None: + session_timeout_ms = float(self.client.session_timeout) + if session_timeout_ms > 0 and delta_s > (session_timeout_ms / 1000.0): + await self.emit_metric( + metrics.OPC_WRITE_INTER_ARRIVAL_OVER_SESSION_TIMEOUT_TOTAL, + self._opc_debug_tags(session_id), ) - return await self.connect() + self._last_write_mono = now - return False, { - 'notification_id': f'OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{self.id}', - 'message': f'OPC server {self.id} is not connected, waiting for next reconnection window...', - 'block': 'opc_repository', - 'level': NotificationLevel.WARNING, - } - return True, {} - except Exception as e: - trace = traceback.format_exc() - message = f'Failed to validate connection to OPC server: {e}' - self.logger.custom_error(message, self.metadata) - return False, { - 'notification_id': f'OPC_CONNECTION_CHECK_ERROR_{self.id}', - 'message': message, - 'block': 'opc_repository', - 'level': NotificationLevel.ERROR, - 'attachment_content': trace, - } + async def _emit_opc_write_metric( + self, session_id: str, result: str, metadata: dict[str, Any] | None + ) -> None: + await self.emit_metric( + metrics.OPC_WRITE_ATTEMPTS_TOTAL, + { + **self._opc_debug_tags(session_id), + **_model_labels_from_write_metadata(metadata), + 'result': result, + }, + ) - async def write_data( - self, node: str, value: Any, data_type: str, logger: Logger, metadata: dict[str, Any] + def _write_failure_payload( + self, + notification_id: str, + message: str, + level: NotificationLevel = NotificationLevel.ERROR, + attachment_content: str | None = None, + opc_error_kind: str | None = None, + opc_status: str | None = None, + ) -> dict[str, Any]: + payload: dict[str, Any] = { + 'notification_id': notification_id, + 'message': message, + 'block': 'opc_repository', + 'level': level, + } + if attachment_content is not None: + payload['attachment_content'] = attachment_content + if opc_error_kind is not None: + payload['opc_error_kind'] = opc_error_kind + if opc_status is not None: + payload['opc_status'] = opc_status + return payload + + async def _handle_tier1_bad( + self, + exc: BaseException, + session_id: str, + node: str, + metadata: dict[str, Any], + phase: str, ) -> tuple[bool, dict[str, Any]]: """ - Write data to OPC server with comprehensive validation and monitoring. - - This method provides secure and reliable data writing to OPC servers - with automatic connection validation, data type conversion, and - comprehensive error handling. It implements performance monitoring - and metrics collection for operational visibility. - - Data Writing Process: - 1. Connection validation and automatic reconnection - 2. Node validation and error handling - 3. Data type conversion and validation - 4. OPC data writing with timestamp - 5. Performance metrics collection - 6. Error handling and notification + Record metrics/logs and schedule reconnect after a Tier-1 Bad* error. Args: - node (str): OPC node identifier to write data to - value (Any): Data value to write to the OPC node - data_type (str): Data type for OPC conversion - logger (Logger): Logger instance for operation logging - metadata (dict[str, Any]): Context metadata for logging and metrics + exc (BaseException): Tier-1 OPC UA error. + session_id (str): Session token at failure time. + node (str): Node id being written. + metadata (dict[str, Any]): Write context. + phase (str): get_node or write_value. - Returns: - tuple[bool, dict[str, Any]]: Write operation result - - bool: True if write successful, False otherwise - - dict: Error information if write failed + Return: + tuple[bool, dict[str, Any]]: Always (False, error payload). """ + opc_status = _opc_status_from_exception(exc) + trace = traceback.format_exc() + self.error(trace, metadata) + await self._emit_opc_write_metric(session_id, opc_status, metadata) + self.error( + f'OPC write failed opc_status={opc_status} opc_server_id={self.id} ' + f'session_id={session_id} model_id={metadata.get("model_id", "unknown")} ' + f'model_name={metadata.get("model_name", "unknown")} node={node} phase={phase}', + metadata, + ) + await self._start_reconnect_on_bad(opc_status, session_id) + return False, self._write_failure_payload( + notification_id=f'OPC_WRITE_DATA_ERROR_{self.id}', + message=f'Failed to {phase} on OPC server: {exc} | metadata: {metadata}', + attachment_content=trace, + opc_error_kind='session_bad', + opc_status=opc_status, + ) + + async def write_data( + self, node: str, value: Any, data_type: str, metadata: dict[str, Any] + ) -> tuple[bool, dict[str, Any]]: + """ + Write data to OPC server with a single attempt and Tier-1 Bad* reconnect scheduling. + """ + if not self._session_ready.is_set(): + await self._emit_opc_write_metric('unknown', 'ReconnectInProgress', metadata) + self.warning( + f'OPC write rejected reconnect_in_progress opc_server_id={self.id} ' + f'model_id={metadata.get("model_id", "unknown")} ' + f'model_name={metadata.get("model_name", "unknown")}', + metadata, + ) + return False, { + 'notification_id': f'OPC_WRITE_RECONNECT_IN_PROGRESS_{self.id}', + 'message': f'OPC write skipped: reconnect in progress | metadata: {metadata}', + 'block': 'opc_repository', + 'level': NotificationLevel.WARNING, + 'opc_error_kind': 'reconnect_in_progress', + } is_connected, error = await self.validate_connection() - if not is_connected: + await self._emit_opc_write_metric('unknown', 'NotConnected', metadata) return False, error start_time = time.time() + session_id = _opc_authentication_token_str(self.client) try: - # ignored because self.validate_connection is called before, so we know self.client is not None node_obj = self.client.get_node(node) # type: ignore[union-attr] except Exception as e: + if is_reconnectable_opcua_bad(e): + return await self._handle_tier1_bad(e, session_id, node, metadata, 'get_node') trace = traceback.format_exc() - logger.custom_error(trace, metadata.get('schedule_name', 'N/A')) - self.error_count += 1 - return False, { - 'notification_id': f'OPC_WRITE_GET_NODE_ERROR_{self.id}', - 'message': f'Failed to get node from OPC server: {e} | metadata: {metadata}', - 'block': 'opc_repository', - 'level': NotificationLevel.ERROR, - 'attachment_content': trace, - } + self.error(trace, metadata) + await self._emit_opc_write_metric( + session_id, f'GetNodeError:{type(e).__name__}', metadata + ) + return False, self._write_failure_payload( + notification_id=f'OPC_WRITE_GET_NODE_ERROR_{self.id}', + message=f'Failed to get node from OPC server: {e} | metadata: {metadata}', + attachment_content=trace, + ) if data_type not in data_type_map: - return False, { - 'notification_id': f'OPC_WRITE_DATA_TYPE_ERROR_{self.id}', - 'message': f'Unsupported data type: {data_type} | metadata: {metadata}', - 'block': 'opc_repository', - 'level': NotificationLevel.ERROR, - } + await self._emit_opc_write_metric(session_id, 'UnsupportedDataType', metadata) + return False, self._write_failure_payload( + notification_id=f'OPC_WRITE_DATA_TYPE_ERROR_{self.id}', + message=f'Unsupported data type: {data_type} | metadata: {metadata}', + ) data = data_type_map[data_type]['converter'](value) - logger.custom_info(f'Writing {data} - {type(data)} to {node}', metadata) - # now = datetime.now() # NOSONAR + self.info(f'Writing {data} - {type(data)} to {node}', metadata) ua_data = DataValue( Variant(data, data_type_map[data_type]['opc_type']), - # SourceTimestamp=DateTime( # NOSONAR - # now.year, now.month, now.day, now.hour, now.minute, now.second, now.microsecond # NOSONAR - # ), # NOSONAR ) try: await node_obj.write_value(ua_data) - end_time = time.time() response_time = end_time - start_time - except Exception as e: + if is_reconnectable_opcua_bad(e): + return await self._handle_tier1_bad(e, session_id, node, metadata, 'write_value') trace = traceback.format_exc() - logger.custom_error(trace, metadata) - self.error_count += 1 - return False, { - 'notification_id': f'OPC_WRITE_DATA_ERROR_{self.id}', - 'message': f'Failed to write data to OPC server: {e} | metadata: {metadata}', - 'block': 'opc_repository', - 'level': NotificationLevel.ERROR, - 'attachment_content': trace, - } - self.error_count = 0 + self.error(trace, metadata) + await self._emit_opc_write_metric(session_id, type(e).__name__, metadata) + return False, self._write_failure_payload( + notification_id=f'OPC_WRITE_DATA_ERROR_{self.id}', + message=f'Failed to write data to OPC server: {e} | metadata: {metadata}', + attachment_content=trace, + ) + + await self._emit_opc_write_metric(session_id, 'OK', metadata) + await self._log_write_inter_arrival(session_id, node) return True, { 'response_time': response_time, diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index fa09259..fe78753 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -63,7 +63,7 @@ with workflow.unsafe.imports_passed_through(): ) from laborious.workflows.sub_workflows.prediction_process import PredictionProcess -POD_ID = os.getenv('POD_ID') +POD_ID = os.getenv('HOSTNAME') SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091')) diff --git a/requirements-dev.txt b/requirements-dev.txt index c4f4ef6..4d0f89d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,3 +18,4 @@ testcontainers[postgres,minio] # PostgreSQL and MinIO containers for E2E tests # Development Tools ipython>=8.12.0 # Enhanced Python shell ipdb>=0.13.13 # IPython debugger +ipykernel==6.30.1 # IPython kernel for Jupyter notebooks diff --git a/tests/laborious/activities/test_opc.py b/tests/laborious/activities/test_opc.py index 0da8214..eaf946f 100644 --- a/tests/laborious/activities/test_opc.py +++ b/tests/laborious/activities/test_opc.py @@ -5,7 +5,15 @@ from pandas import DataFrame from pytest import mark from sientia_do.notifications.models import NotificationLevel -from laborious.activities.opc import OPC +from laborious.activities.opc import ( + OPC, + OPC_COMMENT_SEPARATOR, + OPC_RECONNECT_IN_PROGRESS_COMMENT, + OPC_SESSION_BAD_COMMENT_PREFIX, + OPC_SESSION_BAD_CONFIDENCE, + OPC_WRITTING_ERROR_CONFIDENCE, + OPC_WRITTING_ERROR_MESSAGE, +) metadata = { 'metadata': { @@ -206,7 +214,7 @@ WRITE_DATA_CASES = [ async def test_write_data_success(opc, tag, data_type, data): opc.opc_repository['server1'].write_data.return_value = (True, {'response_time': 0.1}) - result = await opc.write_data( + response_time, error_info = await opc.write_data( server_id='server1', tag=tag, data=data, @@ -214,10 +222,9 @@ async def test_write_data_success(opc, tag, data_type, data): tag_type='prediction', metadata=metadata, ) - assert result == 0.1 - opc.opc_repository['server1'].write_data.assert_called_once_with( - tag, data, data_type, opc.logger, metadata - ) + assert response_time == 0.1 + assert error_info is None + opc.opc_repository['server1'].write_data.assert_called_once_with(tag, data, data_type, metadata) @mark.asyncio @@ -233,7 +240,7 @@ async def test_write_data_failed(opc): }, ) - result = await opc.write_data( + response_time, error_info = await opc.write_data( server_id='server1', tag='tag1', data=50, @@ -241,7 +248,8 @@ async def test_write_data_failed(opc): tag_type='prediction', metadata=metadata, ) - assert result is None + assert response_time is None + assert error_info is not None opc.send_notification_async.assert_called_once_with( metadata=metadata, @@ -283,7 +291,7 @@ async def test_write_data_exception(opc): @mark.asyncio async def test_manage_output_tags_success(opc): - opc.write_data = AsyncMock(return_value=0.1) + opc.write_data = AsyncMock(return_value=(0.1, None)) data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) config = { @@ -291,7 +299,7 @@ async def test_manage_output_tags_success(opc): 'confidence_tags': {'tag2': {'data_type': 'float'}}, } - output_data, opc_metrics = await opc.manage_output_tags( + output_data, opc_metrics, _, _, _ = await opc.manage_output_tags( server_id='server1', config=config, data=data, @@ -323,7 +331,13 @@ async def test_manage_output_tags_success(opc): @mark.asyncio -@mark.parametrize('side_effect', [[0.1, None], [None, 0.2]]) +@mark.parametrize( + 'side_effect', + [ + [(0.1, None), (None, {})], + [(None, {}), (0.2, None)], + ], +) async def test_manage_output_tags_failed(opc, side_effect): opc.write_data = AsyncMock(side_effect=side_effect) data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) @@ -331,14 +345,14 @@ async def test_manage_output_tags_failed(opc, side_effect): 'prediction_tags': {'tag1': {'data_type': 'float'}}, 'confidence_tags': {'tag2': {'data_type': 'float'}}, } - output_data, opc_metrics = await opc.manage_output_tags( + output_data, opc_metrics, _, _, _ = await opc.manage_output_tags( server_id='server1', config=config, data=data, metadata=metadata['metadata'], ) assert output_data is False - assert opc_metrics == {'tag1': side_effect[0], 'tag2': side_effect[1]} + assert opc_metrics == {'tag1': side_effect[0][0], 'tag2': side_effect[1][0]} opc.write_data.assert_has_calls( [ call( @@ -363,12 +377,12 @@ async def test_manage_output_tags_failed(opc, side_effect): @mark.asyncio async def test_manage_output_tags_do_nothing(opc): - opc.write_data = AsyncMock(return_value=0.1) + opc.write_data = AsyncMock(return_value=(0.1, None)) data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) config = { '_invalid_key': {'tag1': {'data_type': 'float'}}, } - output_data, opc_metrics = await opc.manage_output_tags( + output_data, opc_metrics, _, _, _ = await opc.manage_output_tags( server_id='server1', config=config, data=data, @@ -395,7 +409,9 @@ async def test_write_opc_data_success(mock_dataframe, opc): } # Act - opc.manage_output_tags = AsyncMock(return_value=(True, {'tag1': 0.1, 'tag2': 0.2})) + opc.manage_output_tags = AsyncMock( + return_value=(True, {'tag1': 0.1, 'tag2': 0.2}, False, None, False) + ) opc.process_confidence = MagicMock(return_value={'data': 'data'}) output_data, opc_metrics = await opc.write_opc_data(input_data) @@ -413,6 +429,9 @@ async def test_write_opc_data_success(mock_dataframe, opc): mock_dataframe.return_value, True, metadata['metadata'], + session_bad=False, + opc_status=None, + reconnect_in_progress=False, ) @@ -462,13 +481,130 @@ async def test_write_opc_data_no_validate_server(opc): ], ) def test_process_confidence(opc, data, success, expected): - # Act - result = opc.process_confidence(data, success, metadata) - - # Assert + result = opc.process_confidence(data, success, metadata['metadata']) assert result['prediction_confidence'][0] == expected +def test_process_confidence_session_bad(opc): + data = DataFrame({'prediction_confidence': [0.9]}) + result = opc.process_confidence( + data, + False, + metadata['metadata'], + session_bad=True, + opc_status='BadSessionIdInvalid', + ) + assert result['prediction_confidence'][0] == OPC_SESSION_BAD_CONFIDENCE + assert result['comments'][0].startswith(OPC_SESSION_BAD_COMMENT_PREFIX) + assert 'BadSessionIdInvalid' in result['comments'][0] + + +def test_process_confidence_generic_failure(opc): + data = DataFrame({'prediction_confidence': [0.9]}) + result = opc.process_confidence(data, False, metadata['metadata']) + assert result['prediction_confidence'][0] == OPC_WRITTING_ERROR_CONFIDENCE + assert result['comments'][0] == OPC_WRITTING_ERROR_MESSAGE + + +@mark.asyncio +async def test_manage_output_tags_session_bad(opc): + opc.write_data = AsyncMock( + side_effect=[ + ( + None, + { + 'opc_error_kind': 'session_bad', + 'opc_status': 'BadSessionIdInvalid', + 'notification_id': 'OPC_WRITE_DATA_ERROR_1', + 'message': 'bad', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + }, + ), + (0.2, None), + ] + ) + data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) + config = { + 'prediction_tags': {'tag1': {'data_type': 'float'}}, + 'confidence_tags': {'tag2': {'data_type': 'float'}}, + } + ( + success, + metrics, + session_bad_seen, + opc_status, + reconnect_in_progress, + ) = await opc.manage_output_tags('server1', config, data, metadata['metadata']) + assert success is False + assert session_bad_seen is True + assert reconnect_in_progress is False + assert opc_status == 'BadSessionIdInvalid' + assert metrics['tag1'] is None + assert metrics['tag2'] == 0.2 + + +@mark.asyncio +async def test_manage_output_tags_reconnect_in_progress(opc): + opc.write_data = AsyncMock( + return_value=( + None, + { + 'opc_error_kind': 'reconnect_in_progress', + 'notification_id': 'OPC_WRITE_RECONNECT_IN_PROGRESS_1', + 'message': 'skipped', + 'block': 'opc_repository', + 'level': NotificationLevel.WARNING, + }, + ) + ) + data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) + config = {'prediction_tags': {'tag1': {'data_type': 'float'}}} + ( + success, + metrics, + session_bad_seen, + opc_status, + reconnect_in_progress, + ) = await opc.manage_output_tags('server1', config, data, metadata['metadata']) + assert success is False + assert session_bad_seen is False + assert reconnect_in_progress is True + assert opc_status is None + assert metrics['tag1'] is None + + +def test_process_confidence_reconnect_in_progress(opc): + data = DataFrame({'prediction_confidence': [0.9]}) + result = opc.process_confidence( + data, + False, + metadata['metadata'], + reconnect_in_progress=True, + ) + assert result['prediction_confidence'][0] == OPC_SESSION_BAD_CONFIDENCE + assert result['comments'][0] == OPC_RECONNECT_IN_PROGRESS_COMMENT + + +def test_process_confidence_concatenates_multiple_comments(opc): + data = DataFrame({'prediction_confidence': [0.9]}) + session_comment = f'{OPC_SESSION_BAD_COMMENT_PREFIX} BadSessionIdInvalid' + + result = opc.process_confidence( + data, + False, + metadata['metadata'], + session_bad=True, + opc_status='BadSessionIdInvalid', + reconnect_in_progress=True, + ) + + assert result['prediction_confidence'][0] == OPC_SESSION_BAD_CONFIDENCE + assert result['comments'][0] == OPC_COMMENT_SEPARATOR.join( + [session_comment, OPC_RECONNECT_IN_PROGRESS_COMMENT] + ) + + @mark.asyncio async def test_validate_server(opc): assert await opc.validate_server('server1', metadata) is True diff --git a/tests/laborious/utils/repository/test_opc_repository.py b/tests/laborious/utils/repository/test_opc_repository.py index 8be96a7..f1fd81a 100644 --- a/tests/laborious/utils/repository/test_opc_repository.py +++ b/tests/laborious/utils/repository/test_opc_repository.py @@ -1,12 +1,20 @@ +import asyncio import json from datetime import datetime from unittest.mock import ANY, AsyncMock, MagicMock, Mock, patch import pytest from asyncua.crypto.security_policies import SecurityPolicyBasic256 +from asyncua.ua.uaerrors import BadNodeIdUnknown, BadSessionIdInvalid from sientia_do.notifications.models import NotificationLevel -from laborious.utils.repository.opc_repository import OpcRepository +from laborious.utils.repository.opc_repository import ( + OpcClientAlreadyExistsError, + OpcClientNotInitializedError, + OpcRepository, + OpcSessionAlreadyConnectedError, + is_reconnectable_opcua_bad, +) @pytest.fixture @@ -33,6 +41,11 @@ def opc_repository(mock_logger): repository.send_notification = MagicMock() repository.send_notification_async = AsyncMock() repository.emit_metric = AsyncMock() + repository.info = MagicMock() + repository.error = MagicMock() + repository.warning = MagicMock() + repository.debug = MagicMock() + repository._session_ready.set() return repository @@ -65,7 +78,6 @@ def test_init(opc_repository): assert opc_repository.reconnection_interval == 60 assert opc_repository.client is None assert opc_repository.last_reconnection_time is None - assert opc_repository.error_count == 0 @pytest.mark.asyncio @@ -80,8 +92,8 @@ async def test_set_security(opc_repository, mock_client): private_key='/path/to/key.pem', server_certificate='/path/to/server_cert.pem', ) - assert mock_client.secure_channel_timeout == 10000000 - assert mock_client.session_timeout == 10000000 + assert mock_client.secure_channel_timeout == 600_000 + assert mock_client.session_timeout == 600_000 @pytest.mark.asyncio @@ -106,48 +118,95 @@ async def test_set_security_missing_client(opc_repository): @pytest.mark.asyncio async def test_connect_with_security(opc_repository, mock_client): - opc_repository.try_connect = AsyncMock(return_value=(True, {})) + opc_repository._create_client = AsyncMock() + opc_repository._open_session = AsyncMock(return_value=(True, {})) result = await opc_repository.connect() - opc_repository.try_connect.assert_called_once() - assert opc_repository.client == mock_client + opc_repository._create_client.assert_called_once() + opc_repository._open_session.assert_called_once() assert result == (True, {}) @pytest.mark.asyncio async def test_connect_without_security(opc_repository, mock_client): opc_repository.cert_path = None - opc_repository.try_connect = AsyncMock(return_value=(True, {})) + opc_repository._create_client = AsyncMock() + opc_repository._open_session = AsyncMock(return_value=(True, {})) opc_repository.set_security = AsyncMock() result = await opc_repository.connect() - opc_repository.try_connect.assert_called_once() + opc_repository._create_client.assert_called_once() + opc_repository._open_session.assert_called_once() opc_repository.set_security.assert_not_called() - assert opc_repository.client == mock_client assert result == (True, {}) @pytest.mark.asyncio -async def test_try_connect_success(opc_repository): - opc_repository.last_reconnection_time = None +async def test_connect_raises_when_session_already_open(opc_repository, mock_client): + opc_repository.client = mock_client + proto = MagicMock() + proto.state = 'open' + mock_client.uaclient = MagicMock(protocol=proto) + + with pytest.raises(OpcSessionAlreadyConnectedError, match='disconnect'): + await opc_repository.connect() + + +@pytest.mark.asyncio +async def test_create_client_raises_when_client_exists(opc_repository, mock_client): + opc_repository.client = mock_client + + with pytest.raises(OpcClientAlreadyExistsError, match='already exists'): + await opc_repository._create_client() + + +@pytest.mark.asyncio +async def test_open_session_success(opc_repository): + closed_proto = MagicMock() + closed_proto.state = 'closed' opc_repository.client = AsyncMock() - result = await opc_repository.try_connect() + opc_repository.client.uaclient = MagicMock(protocol=closed_proto) + opc_repository.client.session_timeout = 600_000 + opc_repository.client.secure_channel_timeout = 600_000 + + open_proto = MagicMock() + open_proto.state = 'open' + open_proto.authentication_token = 'tok' + + async def connect_side_effect(): + opc_repository.client.uaclient.protocol = open_proto + + opc_repository.client.connect = AsyncMock(side_effect=connect_side_effect) + + result = await opc_repository._open_session() opc_repository.client.connect.assert_called_once() - assert opc_repository.last_reconnection_time is not None + assert opc_repository.last_reconnection_time is None assert result == (True, {}) + assert opc_repository._session_ready.is_set() @pytest.mark.asyncio -async def test_try_connect_fail(opc_repository): - opc_repository.last_reconnection_time = None - opc_repository.disconnect = AsyncMock() +async def test_open_session_raises_when_already_connected(opc_repository, mock_client): + opc_repository.client = mock_client + proto = MagicMock() + proto.state = 'open' + mock_client.uaclient = MagicMock(protocol=proto) + + with pytest.raises(OpcSessionAlreadyConnectedError, match='disconnect'): + await opc_repository._open_session() + + +@pytest.mark.asyncio +async def test_open_session_fail(opc_repository): + opc_repository._disconnect_locked = AsyncMock() opc_repository.client = MagicMock() - opc_repository.client.connect.side_effect = Exception('Test error') + opc_repository.client.uaclient = MagicMock(protocol=MagicMock(state='closed')) + opc_repository.client.connect = AsyncMock(side_effect=Exception('Test error')) - is_connected, error_data = await opc_repository.try_connect() + is_connected, error_data = await opc_repository._open_session() - opc_repository.disconnect.assert_called_once() + opc_repository._disconnect_locked.assert_called_once() opc_repository.client.connect.assert_called_once() assert is_connected is False assert error_data['notification_id'] == f'OPC_CONNECTION_ERROR_{opc_repository.id}' @@ -158,25 +217,18 @@ async def test_try_connect_fail(opc_repository): @pytest.mark.asyncio -async def test_try_connect_no_client(opc_repository): +async def test_open_session_raises_when_no_client(opc_repository): opc_repository.client = None - result = await opc_repository.try_connect() - assert result == ( - False, - { - 'notification_id': f'OPC_CONNECTION_ERROR_{opc_repository.id}', - 'message': 'Client is not initialized', - 'block': 'opc_repository', - 'level': NotificationLevel.ERROR, - }, - ) + + with pytest.raises(OpcClientNotInitializedError, match='not initialized'): + await opc_repository._open_session() @pytest.mark.asyncio async def test_disconnection_fallback_success(opc_repository, mock_client): opc_repository.client = mock_client mock_client.disconnect.return_value = True - result = await opc_repository.disconnection_fallback() + result = await opc_repository._disconnection_fallback() mock_client.disconnect.assert_called_once() assert result == [] @@ -186,7 +238,7 @@ async def test_disconnection_fallback_success(opc_repository, mock_client): async def test_disconnection_fallback_fail(opc_repository, mock_client): opc_repository.client = mock_client mock_client.disconnect.side_effect = Exception('Test error') - result = await opc_repository.disconnection_fallback() + result = await opc_repository._disconnection_fallback() assert result == [ {'attempt': 1, 'error': 'Test error', 'traceback': ANY}, {'attempt': 2, 'error': 'Test error', 'traceback': ANY}, @@ -200,10 +252,10 @@ async def test_disconnection_fallback_fail(opc_repository, mock_client): @pytest.mark.asyncio async def test_disconnect(opc_repository, mock_client): opc_repository.client = mock_client - opc_repository.disconnection_fallback = AsyncMock(return_value=[]) + opc_repository._disconnection_fallback = AsyncMock(return_value=[]) await opc_repository.disconnect() - opc_repository.disconnection_fallback.assert_called_once() + opc_repository._disconnection_fallback.assert_called_once() assert opc_repository.client is None @@ -216,12 +268,12 @@ async def test_disconnect_no_client(opc_repository): @pytest.mark.asyncio async def test_disconnect_error(opc_repository, mock_client): opc_repository.client = mock_client - opc_repository.disconnection_fallback = AsyncMock( + opc_repository._disconnection_fallback = AsyncMock( return_value=[{'attempt': 1, 'error': 'Test error', 'traceback': 'text'}] ) await opc_repository.disconnect() - opc_repository.disconnection_fallback.assert_called_once() + opc_repository._disconnection_fallback.assert_called_once() opc_repository.send_notification_async.assert_called_once_with( metadata=opc_repository.metadata, notification_id=f'OPC_DISCONNECTION_ERROR_{opc_repository.id}', @@ -238,91 +290,24 @@ async def test_disconnect_error(opc_repository, mock_client): @pytest.mark.asyncio async def test_validate_connection_none_client(opc_repository): opc_repository.client = None - opc_repository.connect = AsyncMock(return_value=(True, {})) response = await opc_repository.validate_connection() - assert response == (True, {}) - opc_repository.connect.assert_called_once() - - -# @pytest.mark.asyncio -# async def test_validate_connection_error_count_disconnect_error(opc_repository): -# opc_repository.error_count = 6 -# opc_repository.client = AsyncMock() -# opc_repository.disconnect = AsyncMock(side_effect=Exception('Test error')) -# opc_repository.connect = AsyncMock(return_value=(True, {})) - -# response = await opc_repository.validate_connection() -# assert response == opc_repository.connect.return_value -# opc_repository.disconnect.assert_called_once() -# opc_repository.connect.assert_called_once() -# opc_repository.logger.custom_error.assert_has_calls( -# [ -# call('Failed to disconnect from OPC server: Test error', ANY), -# ] -# ) + assert response == (False, opc_repository._not_connected_error()) @pytest.mark.asyncio -async def test_validate_connection_error_validate_connection_error(opc_repository): - opc_repository.client = MagicMock(uaclient=Exception('Test error')) - opc_repository.error_count = 0 - - response = await opc_repository.validate_connection() - - assert response == ( - False, - { - 'notification_id': f'OPC_CONNECTION_CHECK_ERROR_{opc_repository.id}', - 'message': "Failed to validate connection to OPC server: 'Exception' object has no attribute 'protocol'", - 'block': 'opc_repository', - 'level': NotificationLevel.ERROR, - 'attachment_content': ANY, - }, - ) - - -@pytest.mark.asyncio -@patch('laborious.utils.repository.opc_repository.datetime') -async def test_validate_connection_lost_not_time_to_reconnect(_mock_datetime, opc_repository): - _mock_datetime.now = MagicMock(return_value=datetime(2025, 1, 1, 0, 0, 0)) - opc_repository.error_count = 0 +async def test_validate_connection_session_not_open(opc_repository): opc_repository.client = MagicMock() opc_repository.client.uaclient.protocol = None - opc_repository.last_reconnection_time = datetime(2025, 1, 1, 0, 0, 0) - opc_repository.connect = MagicMock(return_value=(True, {})) response = await opc_repository.validate_connection() - opc_repository.connect.assert_not_called() - assert response == ( - False, - { - 'notification_id': f'OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{opc_repository.id}', - 'message': f'OPC server {opc_repository.id} is not connected, waiting for next reconnection window...', - 'block': 'opc_repository', - 'level': NotificationLevel.WARNING, - }, - ) - -@pytest.mark.asyncio -@patch('laborious.utils.repository.opc_repository.datetime') -async def test_validate_connection_lost_time_to_reconnect(mock_datetime, opc_repository): - mock_datetime.now = MagicMock(return_value=datetime(2025, 1, 1, 1, 0, 0)) - opc_repository.error_count = 0 - opc_repository.client = AsyncMock() - opc_repository.client.uaclient.protocol = None - opc_repository.last_reconnection_time = datetime(2025, 1, 1, 0, 0, 0) - opc_repository.connect = AsyncMock(return_value=(True, {})) - - response = await opc_repository.validate_connection() - opc_repository.connect.assert_called_once() - assert response == opc_repository.connect.return_value + assert response == (False, opc_repository._not_connected_error()) + opc_repository.error.assert_called_once() @pytest.mark.asyncio async def test_validate_connection_success(opc_repository): opc_repository.client = MagicMock() - opc_repository.error_count = 0 opc_repository.client.uaclient.protocol = MagicMock() opc_repository.client.uaclient.protocol.state = 'open' @@ -337,9 +322,7 @@ async def test_write_data_validate_connection_do_nothing(opc_repository): mock_node = AsyncMock() opc_repository.client.get_node.return_value = mock_node - result = await opc_repository.write_data( - 'ns=2;s=TestNode', 42.0, 'float', opc_repository.logger, metadata['metadata'] - ) + result = await opc_repository.write_data('ns=2;s=TestNode', 42.0, 'float', metadata['metadata']) opc_repository.validate_connection.assert_called_once() opc_repository.client.get_node.assert_called_once_with('ns=2;s=TestNode') @@ -350,11 +333,8 @@ async def test_write_data_validate_connection_do_nothing(opc_repository): async def test_write_data_validate_connection_failed(opc_repository): opc_repository.validate_connection = AsyncMock(return_value=(False, {})) opc_repository.client = AsyncMock() - opc_repository.error_count = 0 - result = await opc_repository.write_data( - 'ns=2;s=TestNode', 42.0, 'float', opc_repository.logger, metadata['metadata'] - ) + result = await opc_repository.write_data('ns=2;s=TestNode', 42.0, 'float', metadata['metadata']) opc_repository.validate_connection.assert_called_once() opc_repository.client.get_node.assert_not_called() @@ -365,11 +345,10 @@ async def test_write_data_validate_connection_failed(opc_repository): async def test_write_data_get_node_failed(opc_repository): opc_repository.validate_connection = AsyncMock(return_value=(True, {})) opc_repository.client = AsyncMock() - opc_repository.error_count = 0 opc_repository.client.get_node = MagicMock(side_effect=Exception('Test error')) is_success, error_data = await opc_repository.write_data( - 'ns=2;s=TestNode', 42.0, 'float', opc_repository.logger, metadata['metadata'] + 'ns=2;s=TestNode', 42.0, 'float', metadata['metadata'] ) opc_repository.validate_connection.assert_called_once() @@ -393,7 +372,7 @@ async def test_write_data_invalid_data_type(opc_repository, mock_client): mock_client.get_node = MagicMock(return_value=mock_node) is_success, error_data = await opc_repository.write_data( - 'ns=2;s=TestNode', 42.0, 'invalid_type', opc_repository.logger, metadata['metadata'] + 'ns=2;s=TestNode', 42.0, 'invalid_type', metadata['metadata'] ) opc_repository.validate_connection.assert_called_once() @@ -417,9 +396,7 @@ async def test_write_data(opc_repository, mock_client): mock_node = AsyncMock() mock_client.get_node = MagicMock(return_value=mock_node) - result = await opc_repository.write_data( - 'ns=2;s=TestNode', 42.0, 'float', opc_repository.logger, metadata['metadata'] - ) + result = await opc_repository.write_data('ns=2;s=TestNode', 42.0, 'float', metadata['metadata']) mock_client.get_node.assert_called_once_with('ns=2;s=TestNode') mock_node.write_value.assert_called_once() @@ -431,12 +408,11 @@ async def test_write_data_write_value_failed(opc_repository, mock_client): opc_repository.validate_connection = AsyncMock(return_value=(True, {})) opc_repository.client = mock_client mock_node = AsyncMock() - opc_repository.error_count = 0 mock_client.get_node = MagicMock(return_value=mock_node) mock_node.write_value.side_effect = Exception('Test error') is_success, error_data = await opc_repository.write_data( - 'ns=2;s=TestNode', 42.0, 'float', opc_repository.logger, metadata['metadata'] + 'ns=2;s=TestNode', 42.0, 'float', metadata['metadata'] ) opc_repository.validate_connection.assert_called_once() @@ -451,3 +427,108 @@ async def test_write_data_write_value_failed(opc_repository, mock_client): assert error_data['block'] == 'opc_repository' assert error_data['level'] == NotificationLevel.ERROR assert error_data['attachment_content'] is not None + + +def test_is_reconnectable_opcua_bad(): + assert is_reconnectable_opcua_bad(BadSessionIdInvalid()) is True + assert is_reconnectable_opcua_bad(BadNodeIdUnknown()) is False + assert is_reconnectable_opcua_bad(Exception('other')) is False + + +@pytest.mark.asyncio +async def test_write_data_bad_session_id_invalid_schedules_reconnect(opc_repository, mock_client): + opc_repository.validate_connection = AsyncMock(return_value=(True, {})) + opc_repository.client = mock_client + opc_repository._start_reconnect_on_bad = AsyncMock() + mock_node = AsyncMock() + mock_client.get_node = MagicMock(return_value=mock_node) + mock_node.write_value.side_effect = BadSessionIdInvalid() + + is_success, error_data = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'float', metadata['metadata'] + ) + + mock_node.write_value.assert_called_once() + opc_repository._start_reconnect_on_bad.assert_called_once() + assert is_success is False + assert error_data['opc_error_kind'] == 'session_bad' + assert error_data['opc_status'] == 'BadSessionIdInvalid' + + +@pytest.mark.asyncio +async def test_write_data_reconnect_in_progress_immediate(opc_repository): + opc_repository._session_ready.clear() + opc_repository._reconnect_task = asyncio.create_task(asyncio.sleep(60)) + opc_repository.validate_connection = AsyncMock() + + is_success, error_data = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'float', metadata['metadata'] + ) + + opc_repository._reconnect_task.cancel() + with pytest.raises(asyncio.CancelledError): + await opc_repository._reconnect_task + opc_repository._reconnect_task = None + + opc_repository.validate_connection.assert_not_called() + assert is_success is False + assert error_data['opc_error_kind'] == 'reconnect_in_progress' + + +@pytest.mark.asyncio +async def test_start_reconnect_on_bad_skips_within_interval(opc_repository): + opc_repository.last_reconnection_time = datetime.now() + opc_repository.reconnection_interval = 3600 + + await opc_repository._start_reconnect_on_bad('BadSessionIdInvalid', 'tok') + + assert opc_repository._reconnect_task is None + + +@pytest.mark.asyncio +async def test_parallel_bad_writes_single_reconnect_task(opc_repository, mock_client): + opc_repository.validate_connection = AsyncMock(return_value=(True, {})) + opc_repository.client = mock_client + opc_repository.reconnection_interval = 0 + opc_repository.last_reconnection_time = None + mock_node = AsyncMock() + mock_client.get_node = MagicMock(return_value=mock_node) + mock_node.write_value.side_effect = BadSessionIdInvalid() + + connect_count = 0 + + async def slow_reconnect(): + nonlocal connect_count + connect_count += 1 + await asyncio.sleep(0.05) + opc_repository._session_ready.set() + return True, {} + + opc_repository._reconnect_locked = slow_reconnect + + results = await asyncio.gather( + opc_repository.write_data('ns=2;s=TestNode', 1.0, 'float', metadata['metadata']), + opc_repository.write_data('ns=2;s=TestNode2', 2.0, 'float', metadata['metadata']), + ) + await asyncio.sleep(0.15) + + assert connect_count <= 1 + assert 1 <= mock_node.write_value.call_count <= 2 + error_kinds = [r[1].get('opc_error_kind') for r in results] + assert error_kinds.count('session_bad') >= 1 + assert all(k in ('session_bad', 'reconnect_in_progress') for k in error_kinds) + + +@pytest.mark.asyncio +@patch('laborious.utils.repository.opc_repository.datetime') +async def test_reconnect_locked_sets_last_reconnection_time(mock_datetime, opc_repository): + mock_datetime.now = MagicMock(return_value=datetime(2025, 1, 1, 12, 0, 0)) + opc_repository._disconnect_locked = AsyncMock() + opc_repository._connect_locked = AsyncMock(return_value=(True, {})) + + result = await opc_repository._reconnect_locked() + + opc_repository._disconnect_locked.assert_called_once() + opc_repository._connect_locked.assert_called_once() + assert result == (True, {}) + assert opc_repository.last_reconnection_time == datetime(2025, 1, 1, 12, 0, 0) From e3636d4b880f54d1c3ebf64aca2e6ba20c1f7ffc Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 15 May 2026 15:46:39 -0300 Subject: [PATCH 02/64] SIENTIAPDE-1811 Enhance OPC UA testing framework and documentation - Added a new marker in pyproject.toml for tests using the in-process OPC UA server. - Updated opc-communication.md to clarify E2E test scenarios involving the real OPC server and mock server. - Introduced an in-process asyncua OPC UA server fixture in conftest.py for E2E tests. - Created a new fixture for activities using the real OpcRepository connected to the in-process server. - Updated scenarios.md to include instructions for running OPC real-server tests. --- docs/opc-communication.md | 4 +- e2e/conftest.py | 95 ++++++++++++++++++ e2e/opc_test_server.py | 189 +++++++++++++++++++++++++++++++++++ e2e/scenarios.md | 2 + e2e/test_opc_real_server.py | 194 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + 6 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 e2e/opc_test_server.py create mode 100644 e2e/test_opc_real_server.py diff --git a/docs/opc-communication.md b/docs/opc-communication.md index ce612b9..3084d05 100644 --- a/docs/opc-communication.md +++ b/docs/opc-communication.md @@ -157,4 +157,6 @@ Legacy activity metrics: `laborious_prediction_opc_writing_count`, `laborious_pr - Unit: [`tests/laborious/utils/repository/test_opc_repository.py`](../tests/laborious/utils/repository/test_opc_repository.py) - Unit: [`tests/laborious/activities/test_opc.py`](../tests/laborious/activities/test_opc.py) -- E2E: [`e2e/test_predictions_batch_format_export.py`](../e2e/test_predictions_batch_format_export.py), scenarios in [`e2e/scenarios.md`](../e2e/scenarios.md) +- E2E (mock OPC): [`e2e/test_predictions_batch_format_export.py`](../e2e/test_predictions_batch_format_export.py) +- E2E (in-process asyncua server + real `OpcRepository`): [`e2e/test_opc_real_server.py`](../e2e/test_opc_real_server.py) — scenarios 3.1.2, 3.2.2, 3.2.4, 3.2.5 +- Scenarios: [`e2e/scenarios.md`](../e2e/scenarios.md) diff --git a/e2e/conftest.py b/e2e/conftest.py index f29ec0d..cc49d9b 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -21,6 +21,7 @@ from testcontainers.postgres import PostgresContainer from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker +from e2e.opc_test_server import OpcE2ETestServer from laborious.activities.activities import Activities from laborious.workflows.predictions_batch import PredictionsBatch from laborious.workflows.sub_workflows.prediction_process import PredictionProcess @@ -304,6 +305,19 @@ def mock_opc_repository(): mock_repo.disconnect = AsyncMock() return mock_repo + +@pytest_asyncio.fixture +async def opc_e2e_server(): + """ + In-process asyncua OPC UA server for E2E tests against OpcRepository. + """ + server = OpcE2ETestServer() + await server.start() + try: + yield server + finally: + await server.stop() + @pytest_asyncio.fixture def patch_create_engine(postgres_engine): """Patch create_engine to return test postgres_engine.""" @@ -584,3 +598,84 @@ async def temporal_worker_real_minio(temporal_test_env, test_activities_real_min activities=_worker_activity_list(test_activities_real_minio), ) as worker: yield worker + + +@pytest_asyncio.fixture(scope='function') +async def test_activities_real_opc( + postgres_engine, + postgres_container, + opc_e2e_server: OpcE2ETestServer, + mock_logger, + notification_handler, + metrics_controller, + patch_create_engine, + patch_minio_repository, + patch_mlflow, + patch_pi_web_api_repository, +): + """ + Activities with a real OpcRepository connected to the in-process OPC UA server. + """ + activities = Activities( + postgres_config={ + 'host': 'localhost', + 'port': postgres_container.get_exposed_port(5432), + 'user': 'test', + 'password': 'test', + 'dbname': 'test', + 'min_connections': 1, + 'max_connections': 5, + }, + mlflow_config={ + 'host': 'http://localhost', + 'port': '5000', + 'username': 'test', + 'password': 'test', + }, + minio_config={ + 'endpoint_url': 'localhost:9000', + 'access_key': 'test', + 'secret_key': 'test', + 'default_bucket': 'test-bucket', + 'retention_hours': 24, + 'secure': False, + }, + opc_config={ + '1': { + 'id': '1', + 'server_name': 'e2e-opc', + 'url': opc_e2e_server.url, + 'server_uri': opc_e2e_server.url, + 'cert_path': None, + 'private_key_path': None, + 'server_cert_path': None, + 'reconnection_interval': 0, + } + }, + pi_web_api_config={ + 'base_url': 'http://localhost:8080', + 'auth_type': 'bearer', + 'auth_token': 'test_token', + }, + logger=mock_logger, + notification_handler=notification_handler, + ) + await activities.init_opc() + repo = activities.opc_repository['1'] + assert repo._session_ready.is_set(), 'OPC E2E server connection failed during init_opc' + try: + yield activities + finally: + await activities.shutdown() + + +@pytest_asyncio.fixture(scope='function') +async def temporal_worker_real_opc(temporal_test_env, test_activities_real_opc): + """Temporal worker backed by Activities using the in-process OPC UA server.""" + async with Worker( + temporal_test_env.client, + task_queue='test-queue', + workflows=[PredictionsBatch, PredictionProcess, FormatAndExportPrediction], + activities=_worker_activity_list(test_activities_real_opc), + ) as worker: + yield worker diff --git a/e2e/opc_test_server.py b/e2e/opc_test_server.py new file mode 100644 index 0000000..69b02c8 --- /dev/null +++ b/e2e/opc_test_server.py @@ -0,0 +1,189 @@ +""" +In-process OPC UA server for E2E tests (asyncua). + +Provides writable prediction/confidence nodes and optional write faults +(Tier-1 BadSessionIdInvalid via PreWrite callback). +""" + +from __future__ import annotations + +import socket +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from asyncua import Server, ua +from asyncua.common.callback import CallbackType +from asyncua.common.utils import ServiceError + +if TYPE_CHECKING: + from asyncua.common.node import Node + + +UNKNOWN_NODE_ID = 'ns=99;i=9999' + + +@dataclass(frozen=True) +class OpcE2ENodeIds: + """NodeId strings used in opc_output_config for E2E workflows.""" + + prediction: str + confidence: str + unknown: str = UNKNOWN_NODE_ID + + +class OpcE2ETestServer: + """ + Ephemeral asyncua server with Laborious E2E variables and controllable faults. + + Args: + host: Bind address (default 127.0.0.1). + """ + + def __init__(self, host: str = '127.0.0.1') -> None: + self._host = host + self._server: Server | None = None + self._prediction_node: Node | None = None + self._confidence_node: Node | None = None + self._session_bad_on_write = False + self._url: str | None = None + self._node_ids: OpcE2ENodeIds | None = None + + @property + def url(self) -> str: + if self._url is None: + raise RuntimeError('OPC E2E server is not started') + return self._url + + @property + def node_ids(self) -> OpcE2ENodeIds: + if self._node_ids is None: + raise RuntimeError('OPC E2E server is not started') + return self._node_ids + + def set_session_bad_on_write(self, enabled: bool) -> None: + """ + When enabled, every client Write is rejected with BadSessionIdInvalid. + + Args: + enabled (bool): Turn Tier-1 session fault injection on or off. + """ + self._session_bad_on_write = enabled + + async def start(self) -> OpcE2ENodeIds: + """ + Start the OPC UA server on a free TCP port. + + Return: + OpcE2ENodeIds: NodeId strings for prediction and confidence tags. + """ + port = _free_port(self._host) + self._url = f'opc.tcp://{self._host}:{port}/freeopcua/server/' + + server = Server() + server.set_endpoint(self._url) + await server.init() + server.iserver.callback_service.addListener( + CallbackType.PreWrite, + self._pre_write_callback, + ) + + idx = await server.register_namespace('http://sientia.test/laborious-e2e') + e2e_object = await server.nodes.objects.add_object(idx, 'LaboriousE2E') + prediction = await e2e_object.add_variable( + idx, + 'Prediction', + ua.Variant(0.0, ua.VariantType.Float), + ) + confidence = await e2e_object.add_variable( + idx, + 'Confidence', + ua.Variant(0.0, ua.VariantType.Float), + ) + await prediction.set_writable() + await confidence.set_writable() + + await server.start() + self._server = server + self._prediction_node = prediction + self._confidence_node = confidence + self._node_ids = OpcE2ENodeIds( + prediction=prediction.nodeid.to_string(), + confidence=confidence.nodeid.to_string(), + ) + return self._node_ids + + async def stop(self) -> None: + """Stop the OPC UA server and release the listening port.""" + if self._server is not None: + await self._server.stop() + self._server = None + self._prediction_node = None + self._confidence_node = None + self._url = None + self._node_ids = None + self._session_bad_on_write = False + + async def read_prediction(self) -> float: + """ + Read the current prediction variable value from the address space. + + Return: + float: Stored prediction value. + """ + if self._prediction_node is None: + raise RuntimeError('OPC E2E server is not started') + value = await self._prediction_node.read_value() + return float(value) + + async def read_confidence(self) -> float: + """ + Read the current confidence variable value from the address space. + + Return: + float: Stored confidence value. + """ + if self._confidence_node is None: + raise RuntimeError('OPC E2E server is not started') + value = await self._confidence_node.read_value() + return float(value) + + async def _pre_write_callback(self, _event, _service) -> None: + if self._session_bad_on_write: + raise ServiceError(ua.StatusCodes.BadSessionIdInvalid) + + +def _free_port(host: str) -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind((host, 0)) + return int(sock.getsockname()[1]) + + +def build_opc_output_config( + node_ids: OpcE2ENodeIds, + *, + prediction_tag: str | None = None, + confidence_tag: str | None = None, + prediction_only: bool = False, + server_key: str = '1', +) -> dict[str, dict]: + """ + Build opc_output_config for PredictionsBatch using real server NodeIds. + + Args: + node_ids (OpcE2ENodeIds): Node ids from OpcE2ETestServer. + prediction_tag (str | None): Override prediction NodeId (default: node_ids.prediction). + confidence_tag (str | None): Override confidence NodeId (default: node_ids.confidence). + prediction_only (bool): When True, omit confidence_tags (single write per activity). + server_key (str): OPC server id key in opc_output_config. + + Return: + dict: opc_output_config payload for workflow input. + """ + pred = prediction_tag if prediction_tag is not None else node_ids.prediction + conf = confidence_tag if confidence_tag is not None else node_ids.confidence + server_config: dict = { + 'prediction_tags': {pred: {'data_type': 'float'}}, + } + if not prediction_only: + server_config['confidence_tags'] = {conf: {'data_type': 'float'}} + return {server_key: server_config} diff --git a/e2e/scenarios.md b/e2e/scenarios.md index 03f58c2..8841424 100644 --- a/e2e/scenarios.md +++ b/e2e/scenarios.md @@ -8,6 +8,8 @@ This document describes all possible test scenarios for the `predictions_batch` - **Dependencies**: install dev requirements (includes `testcontainers[postgres,minio]`). - **Invocation**: run only integration-marked tests, for example: `pytest e2e/ -m integration`. - **MinIO tests**: `e2e/test_minio_offload.py` exercises real S3 uploads; other E2E modules continue to mock MinIO on the worker used by most scenarios. +- **OPC tests (real server)**: `e2e/test_opc_real_server.py` uses an in-process **asyncua** server and real `OpcRepository` (`test_activities_real_opc`). Scenarios 3.1.2, 3.2.2, 3.2.4, and 3.2.5 are covered there. Other E2E modules keep the OPC mock. +- Run only OPC real-server tests: `pytest e2e/test_opc_real_server.py -m "integration and opc"`. ## Workflow Overview diff --git a/e2e/test_opc_real_server.py b/e2e/test_opc_real_server.py new file mode 100644 index 0000000..200a8a4 --- /dev/null +++ b/e2e/test_opc_real_server.py @@ -0,0 +1,194 @@ +""" +E2E tests for OPC export using an in-process asyncua server and real OpcRepository. + +Covers scenarios 3.1.2, 3.2.2, 3.2.4, and 3.2.5 from e2e/scenarios.md. +Mock-based OPC tests remain in test_predictions_batch_format_export.py. +""" + +import asyncio + +import pytest +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + +from e2e.helpers import assert_prediction, insert_sample_data, make_workflow_id, start_and_await_workflow +from e2e.opc_test_server import UNKNOWN_NODE_ID, OpcE2ETestServer, build_opc_output_config +from e2e.test_predictions_batch_format_export import get_base_input_data +from laborious.activities.activities import Activities +from laborious.activities.opc import OPC_RECONNECT_IN_PROGRESS_COMMENT +from laborious.utils.repository.opc_repository import OpcRepository +from laborious.workflows.predictions_batch import PredictionsBatch + + +async def _slow_reconnect_under_lock(repo: OpcRepository, hold_seconds: float = 0.75) -> None: + """ + Hold the connection lock briefly so concurrent writes see reconnect_in_progress. + + Args: + repo (OpcRepository): Connected repository. + hold_seconds (float): Time to keep the lock before reconnecting. + """ + async with repo._connection_lock: + await asyncio.sleep(hold_seconds) + await repo._reconnect_locked() + + +@pytest.mark.asyncio +@pytest.mark.integration +@pytest.mark.opc +async def test_scenario_3_1_2_export_with_opc_only_real_server( + temporal_test_env: WorkflowEnvironment, + temporal_worker_real_opc: Worker, + test_activities_real_opc: Activities, + opc_e2e_server: OpcE2ETestServer, + postgres_engine, +): + """ + Scenario 3.1.2 (real OPC): connect, write prediction and confidence, verify server values. + """ + client = temporal_test_env.client + model_id = 412 + insert_sample_data(postgres_engine, model_id, [23.5, 78.2]) + + input_data = get_base_input_data(model_id) + input_data['opc_output_config'] = build_opc_output_config(opc_e2e_server.node_ids) + input_data['pi_web_api_output_config'] = None + + await start_and_await_workflow( + client, + PredictionsBatch.run, + input_data, + make_workflow_id('test-opc-real-happy'), + ) + + test_activities_real_opc.pi_web_api_client.write_value.assert_not_called() + assert await opc_e2e_server.read_prediction() == pytest.approx(0.5) + assert await opc_e2e_server.read_confidence() == pytest.approx(0.0) + assert_prediction(postgres_engine, model_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +@pytest.mark.opc +async def test_scenario_3_2_2_opc_write_error_real_server( + temporal_test_env: WorkflowEnvironment, + temporal_worker_real_opc: Worker, + test_activities_real_opc: Activities, + opc_e2e_server: OpcE2ETestServer, + postgres_engine, +): + """ + Scenario 3.2.2 (real OPC): unknown NodeId yields generic write failure (confidence 12). + """ + client = temporal_test_env.client + model_id = 422 + insert_sample_data(postgres_engine, model_id, [23.5, 78.2]) + + node_ids = opc_e2e_server.node_ids + input_data = get_base_input_data(model_id) + input_data['opc_output_config'] = build_opc_output_config( + node_ids, + prediction_tag=UNKNOWN_NODE_ID, + confidence_tag=UNKNOWN_NODE_ID, + ) + input_data['pi_web_api_output_config'] = None + + await start_and_await_workflow( + client, + PredictionsBatch.run, + input_data, + make_workflow_id('test-opc-real-bad-node'), + ) + + assert_prediction( + postgres_engine, + model_id, + prediction_confidence=12, + comments='Some data could not be written to OPC servers', + ) + + +@pytest.mark.asyncio +@pytest.mark.integration +@pytest.mark.opc +async def test_scenario_3_2_4_opc_session_bad_real_server( + temporal_test_env: WorkflowEnvironment, + temporal_worker_real_opc: Worker, + test_activities_real_opc: Activities, + opc_e2e_server: OpcE2ETestServer, + postgres_engine, +): + """ + Scenario 3.2.4 (real OPC): server PreWrite fault injects BadSessionIdInvalid (confidence 14). + """ + client = temporal_test_env.client + model_id = 424 + insert_sample_data(postgres_engine, model_id, [23.5, 78.2]) + + opc_e2e_server.set_session_bad_on_write(True) + try: + input_data = get_base_input_data(model_id) + input_data['opc_output_config'] = build_opc_output_config( + opc_e2e_server.node_ids, + prediction_only=True, + ) + input_data['pi_web_api_output_config'] = None + + await start_and_await_workflow( + client, + PredictionsBatch.run, + input_data, + make_workflow_id('test-opc-real-session-bad'), + ) + finally: + opc_e2e_server.set_session_bad_on_write(False) + + assert_prediction( + postgres_engine, + model_id, + prediction_confidence=14, + comments_contains='OPC UA session/channel error: BadSessionIdInvalid', + ) + + +@pytest.mark.asyncio +@pytest.mark.integration +@pytest.mark.opc +async def test_scenario_3_2_5_opc_write_blocked_during_reconnect_real_server( + temporal_test_env: WorkflowEnvironment, + temporal_worker_real_opc: Worker, + test_activities_real_opc: Activities, + opc_e2e_server: OpcE2ETestServer, + postgres_engine, +): + """ + Scenario 3.2.5 (real OPC): writes rejected while reconnect holds the connection lock. + """ + client = temporal_test_env.client + model_id = 425 + insert_sample_data(postgres_engine, model_id, [23.5, 78.2]) + + repo = test_activities_real_opc.opc_repository['1'] + repo._session_ready.clear() + reconnect_task = asyncio.create_task(_slow_reconnect_under_lock(repo)) + + input_data = get_base_input_data(model_id) + input_data['opc_output_config'] = build_opc_output_config(opc_e2e_server.node_ids) + input_data['pi_web_api_output_config'] = None + + try: + await start_and_await_workflow( + client, + PredictionsBatch.run, + input_data, + make_workflow_id('test-opc-real-reconnect-block'), + ) + finally: + await reconnect_task + + assert_prediction( + postgres_engine, + model_id, + prediction_confidence=14, + comments_contains=OPC_RECONNECT_IN_PROGRESS_COMMENT, + ) diff --git a/pyproject.toml b/pyproject.toml index 5123123..a878882 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,6 +121,7 @@ markers = [ "asyncio: marks tests as async", "integration: marks tests as integration tests", "unit: marks tests as unit tests", + "opc: marks tests that use the in-process OPC UA server (OpcRepository E2E)", ] [tool.coverage.run] From 7d59fd7c8ce46bddf6555c88431b4c498d461a26 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 18 May 2026 10:17:52 -0300 Subject: [PATCH 03/64] SIENTIAPDE-1811 Update values.yaml to rename worker references and adjust GitHub branch for SIENTIAPDE-1811. Changed nameOverride, fullnameOverride, and service account name to "sientia-laborious-legacy-worker" and updated the GITHUB_BRANCH value to "fix/SIENTIAPDE-1811". --- values.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/values.yaml b/values.yaml index 31dd4fa..b07d60b 100644 --- a/values.yaml +++ b/values.yaml @@ -17,8 +17,8 @@ image: imagePullSecrets: - name: docker-hub-secret # This is to override the chart name. -nameOverride: "sientia-laborious-worker" -fullnameOverride: "sientia-laborious-worker" +nameOverride: "sientia-laborious-legacy-worker" +fullnameOverride: "sientia-laborious-legacy-worker" namespace: sientia # This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ @@ -31,7 +31,7 @@ serviceAccount: annotations: {} # The name of the service account to use. # If not set and create is true, a name is generated using the fullname template - name: "sientia-laborious-worker" + name: "sientia-laborious-legacy-worker" # This is for setting Kubernetes Annotations to a Pod. # For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ @@ -157,7 +157,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-laborious_temporal.git" - name: GITHUB_BRANCH - value: "feature/SIENTIAPDE-1712" + value: "fix/SIENTIAPDE-1811" - name: PYTHON_APP value: "laborious.worker.worker" @@ -307,7 +307,7 @@ ssh: # kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp -# helm upgrade --install sientia-laborious-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.6.0 +# helm upgrade --install sientia-laborious-legacy-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.6.0 # kubectl create secret generic git-ssh-key-sientia-laborious-worker \ # --namespace sientia \ From 2a6def4056c5ad69611693053f3b8d7776056133 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 18 May 2026 10:36:57 -0300 Subject: [PATCH 04/64] SIENTIAPDE-1811 SIENTIAPDE-1811 Implement OPC write error handling and refactor tag writing logic - Introduced a new function `_apply_opc_write_error` to manage session and reconnect flags based on OPC write error responses. - Refactored the `_write_tags_from_config` method to streamline the writing of OPC tags for both prediction and confidence data. - Enhanced unit tests to cover various scenarios for OPC write errors, including session bad and reconnect in progress cases. - Updated existing tests to validate the new logic and ensure robust error handling. --- laborious/activities/opc.py | 161 ++++++++---- tests/laborious/activities/test_opc.py | 330 +++++++++++++++++-------- 2 files changed, 337 insertions(+), 154 deletions(-) diff --git a/laborious/activities/opc.py b/laborious/activities/opc.py index 524237e..be5a941 100644 --- a/laborious/activities/opc.py +++ b/laborious/activities/opc.py @@ -27,6 +27,35 @@ def _opc_session_bad_comment(opc_status: str | None) -> str: return f'{OPC_SESSION_BAD_COMMENT_PREFIX} {status}' +def _apply_opc_write_error( + error_info: dict[str, Any] | None, + session_bad_seen: bool, + session_bad_status: str | None, + reconnect_in_progress_seen: bool, +) -> tuple[bool, str | None, bool]: + """ + Update session/reconnect flags from an OPC write error payload. + + Args: + error_info: Repository error details, or None when the write succeeded. + session_bad_seen: Whether a session_bad error was seen so far. + session_bad_status: Last known OPC status for session errors. + reconnect_in_progress_seen: Whether reconnect_in_progress was seen so far. + + Return: + Updated (session_bad_seen, session_bad_status, reconnect_in_progress_seen). + """ + if not error_info: + return session_bad_seen, session_bad_status, reconnect_in_progress_seen + + kind = error_info.get('opc_error_kind') + if kind == 'session_bad': + return True, error_info.get('opc_status', session_bad_status), reconnect_in_progress_seen + if kind == 'reconnect_in_progress': + return session_bad_seen, session_bad_status, True + return session_bad_seen, session_bad_status, reconnect_in_progress_seen + + class OPC(SientiaMonitoring): """ OPC server integration activities for real-time data export. @@ -198,6 +227,62 @@ class OPC(SientiaMonitoring): return False return True + async def _write_tags_from_config( + self, + server_id: str, + tags_config: dict[str, dict[str, Any]], + data: DataFrame, + data_column: str, + tag_type: str, + log_label: str, + metadata: dict[str, Any], + ) -> tuple[dict[str, float | None], bool, str | None, bool]: + """ + Write a group of OPC tags and collect response times and error flags. + + Args: + server_id: Target OPC server identifier. + tags_config: Tag name to configuration mapping. + data: DataFrame with prediction/confidence columns. + data_column: Column name whose first row value is written. + tag_type: Tag category passed to write_data ('prediction' or 'confidence'). + log_label: Human-readable label for success logs. + metadata: Context metadata for logging and notifications. + + Return: + (response_times, session_bad_seen, session_bad_status, reconnect_in_progress_seen) + """ + response_times: dict[str, float | None] = {} + session_bad_seen = False + session_bad_status: str | None = None + reconnect_in_progress_seen = False + + for tag, tag_config in tags_config.items(): + response_time, error_info = await self.write_data( + server_id=server_id, + tag=tag, + data=data.head(1)[data_column].values[0], + data_type=tag_config['data_type'], + tag_type=tag_type, + metadata=metadata, + ) + session_bad_seen, session_bad_status, reconnect_in_progress_seen = ( + _apply_opc_write_error( + error_info, + session_bad_seen, + session_bad_status, + reconnect_in_progress_seen, + ) + ) + if response_time is not None: + self.info( + f'{log_label} written to OPC server {server_id} for tag {tag}.', + metadata, + ) + response_times[tag] = response_time + + return response_times, session_bad_seen, session_bad_status, reconnect_in_progress_seen + async def manage_output_tags( self, server_id: str, @@ -231,62 +316,40 @@ class OPC(SientiaMonitoring): - overall_success: True if all configured tags were written successfully - total_tags_written: Count of successfully written tags """ - response_times: dict[str, float | None] = {} session_bad_seen = False session_bad_status: str | None = None reconnect_in_progress_seen = False - if 'prediction_tags' in config: - for tag, tag_config in config['prediction_tags'].items(): - response_time, error_info = await self.write_data( - server_id=server_id, - tag=tag, - data=data.head(1)['prediction'].values[0], - data_type=tag_config['data_type'], - tag_type='prediction', - metadata=metadata, - ) - if error_info: - kind = error_info.get('opc_error_kind') - if kind == 'session_bad': - session_bad_seen = True - session_bad_status = error_info.get('opc_status', session_bad_status) - elif kind == 'reconnect_in_progress': - reconnect_in_progress_seen = True - if response_time is not None: - self.info( - f'Prediction data written to OPC server {server_id} for tag {tag}.', - metadata, - ) - response_times[tag] = response_time - - if 'confidence_tags' in config: - for tag, tag_config in config['confidence_tags'].items(): - response_time, error_info = await self.write_data( - server_id=server_id, - tag=tag, - data=data.head(1)['prediction_confidence'].values[0], - data_type=tag_config['data_type'], - tag_type='confidence', - metadata=metadata, - ) - if error_info: - kind = error_info.get('opc_error_kind') - if kind == 'session_bad': - session_bad_seen = True - session_bad_status = error_info.get('opc_status', session_bad_status) - elif kind == 'reconnect_in_progress': - reconnect_in_progress_seen = True - if response_time is not None: - self.info( - f'Confidence data written to OPC server {server_id} for tag {tag}.', - metadata, - ) - response_times[tag] = response_time + tag_groups = ( + ('prediction_tags', 'prediction', 'prediction', 'Prediction data'), + ('confidence_tags', 'prediction_confidence', 'confidence', 'Confidence data'), + ) + for config_key, data_column, tag_type, log_label in tag_groups: + if config_key not in config: + continue + ( + group_times, + group_session_bad, + group_status, + group_reconnect, + ) = await self._write_tags_from_config( + server_id=server_id, + tags_config=config[config_key], + data=data, + data_column=data_column, + tag_type=tag_type, + log_label=log_label, + metadata=metadata, + ) + response_times.update(group_times) + if group_session_bad: + session_bad_seen = True + session_bad_status = group_status or session_bad_status + if group_reconnect: + reconnect_in_progress_seen = True success = None not in response_times.values() - return ( success, response_times, diff --git a/tests/laborious/activities/test_opc.py b/tests/laborious/activities/test_opc.py index eaf946f..c2d37dd 100644 --- a/tests/laborious/activities/test_opc.py +++ b/tests/laborious/activities/test_opc.py @@ -13,6 +13,7 @@ from laborious.activities.opc import ( OPC_SESSION_BAD_CONFIDENCE, OPC_WRITTING_ERROR_CONFIDENCE, OPC_WRITTING_ERROR_MESSAGE, + _apply_opc_write_error, ) metadata = { @@ -289,17 +290,211 @@ async def test_write_data_exception(opc): raise AssertionError('Expected an exception to be raised') +@mark.parametrize( + 'error_info,initial_seen,initial_status,initial_reconnect,expected', + [ + (None, False, None, False, (False, None, False)), + ({}, False, None, False, (False, None, False)), + ( + {'opc_error_kind': 'session_bad', 'opc_status': 'BadSessionIdInvalid'}, + False, + None, + False, + (True, 'BadSessionIdInvalid', False), + ), + ( + {'opc_error_kind': 'session_bad', 'opc_status': 'NewStatus'}, + True, + 'OldStatus', + False, + (True, 'NewStatus', False), + ), + ( + {'opc_error_kind': 'session_bad'}, + True, + 'KeptStatus', + False, + (True, 'KeptStatus', False), + ), + ( + {'opc_error_kind': 'reconnect_in_progress'}, + False, + None, + False, + (False, None, True), + ), + ( + {'opc_error_kind': 'other'}, + True, + 'Status', + True, + (True, 'Status', True), + ), + ], +) +def test_apply_opc_write_error( + error_info, initial_seen, initial_status, initial_reconnect, expected +): + result = _apply_opc_write_error( + error_info, + initial_seen, + initial_status, + initial_reconnect, + ) + assert result == expected + + +@mark.asyncio +async def test_write_tags_from_config_prediction_success(opc): + opc.write_data = AsyncMock(return_value=(0.1, None)) + data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) + tags_config = {'tag1': {'data_type': 'float'}} + + response_times, session_bad, opc_status, reconnect = await opc._write_tags_from_config( + server_id='server1', + tags_config=tags_config, + data=data, + data_column='prediction', + tag_type='prediction', + log_label='Prediction data', + metadata=metadata['metadata'], + ) + + assert response_times == {'tag1': 0.1} + assert session_bad is False + assert opc_status is None + assert reconnect is False + opc.write_data.assert_called_once_with( + server_id='server1', + tag='tag1', + data=0.75, + data_type='float', + tag_type='prediction', + metadata=metadata['metadata'], + ) + + +@mark.asyncio +async def test_write_tags_from_config_confidence_success(opc): + opc.write_data = AsyncMock(return_value=(0.2, None)) + data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) + tags_config = {'tag2': {'data_type': 'float'}} + + response_times, session_bad, opc_status, reconnect = await opc._write_tags_from_config( + server_id='server1', + tags_config=tags_config, + data=data, + data_column='prediction_confidence', + tag_type='confidence', + log_label='Confidence data', + metadata=metadata['metadata'], + ) + + assert response_times == {'tag2': 0.2} + assert session_bad is False + assert opc_status is None + assert reconnect is False + opc.write_data.assert_called_once_with( + server_id='server1', + tag='tag2', + data=0.95, + data_type='float', + tag_type='confidence', + metadata=metadata['metadata'], + ) + + +@mark.asyncio +async def test_write_tags_from_config_write_failure(opc): + opc.write_data = AsyncMock(return_value=(None, {})) + data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) + + response_times, session_bad, opc_status, reconnect = await opc._write_tags_from_config( + server_id='server1', + tags_config={'tag1': {'data_type': 'float'}}, + data=data, + data_column='prediction', + tag_type='prediction', + log_label='Prediction data', + metadata=metadata['metadata'], + ) + + assert response_times == {'tag1': None} + assert session_bad is False + assert opc_status is None + assert reconnect is False + + +@mark.asyncio +async def test_write_tags_from_config_session_bad(opc): + opc.write_data = AsyncMock( + return_value=( + None, + { + 'opc_error_kind': 'session_bad', + 'opc_status': 'BadSessionIdInvalid', + }, + ) + ) + data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) + + response_times, session_bad, opc_status, reconnect = await opc._write_tags_from_config( + server_id='server1', + tags_config={'tag1': {'data_type': 'float'}}, + data=data, + data_column='prediction', + tag_type='prediction', + log_label='Prediction data', + metadata=metadata['metadata'], + ) + + assert response_times == {'tag1': None} + assert session_bad is True + assert opc_status == 'BadSessionIdInvalid' + assert reconnect is False + + +@mark.asyncio +async def test_write_tags_from_config_reconnect_in_progress(opc): + opc.write_data = AsyncMock( + return_value=( + None, + {'opc_error_kind': 'reconnect_in_progress'}, + ) + ) + data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) + + response_times, session_bad, opc_status, reconnect = await opc._write_tags_from_config( + server_id='server1', + tags_config={'tag1': {'data_type': 'float'}}, + data=data, + data_column='prediction', + tag_type='prediction', + log_label='Prediction data', + metadata=metadata['metadata'], + ) + + assert response_times == {'tag1': None} + assert session_bad is False + assert opc_status is None + assert reconnect is True + + @mark.asyncio async def test_manage_output_tags_success(opc): - opc.write_data = AsyncMock(return_value=(0.1, None)) - + opc._write_tags_from_config = AsyncMock( + side_effect=[ + ({'tag1': 0.1}, False, None, False), + ({'tag2': 0.1}, False, None, False), + ] + ) data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) config = { 'prediction_tags': {'tag1': {'data_type': 'float'}}, 'confidence_tags': {'tag2': {'data_type': 'float'}}, } - output_data, opc_metrics, _, _, _ = await opc.manage_output_tags( + output_data, opc_metrics, session_bad, opc_status, reconnect = await opc.manage_output_tags( server_id='server1', config=config, data=data, @@ -308,89 +503,53 @@ async def test_manage_output_tags_success(opc): assert output_data is True assert opc_metrics == {'tag1': 0.1, 'tag2': 0.1} - opc.write_data.assert_has_calls( - [ - call( - server_id='server1', - tag='tag1', - data=0.75, - data_type='float', - tag_type='prediction', - metadata=metadata['metadata'], - ), - call( - server_id='server1', - tag='tag2', - data=0.95, - data_type='float', - tag_type='confidence', - metadata=metadata['metadata'], - ), - ] - ) + assert session_bad is False + assert opc_status is None + assert reconnect is False + assert opc._write_tags_from_config.await_count == 2 @mark.asyncio -@mark.parametrize( - 'side_effect', - [ - [(0.1, None), (None, {})], - [(None, {}), (0.2, None)], - ], -) -async def test_manage_output_tags_failed(opc, side_effect): - opc.write_data = AsyncMock(side_effect=side_effect) +async def test_manage_output_tags_failed(opc): + opc._write_tags_from_config = AsyncMock( + side_effect=[ + ({'tag1': 0.1}, False, None, False), + ({'tag2': None}, False, None, False), + ] + ) data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) config = { 'prediction_tags': {'tag1': {'data_type': 'float'}}, 'confidence_tags': {'tag2': {'data_type': 'float'}}, } + output_data, opc_metrics, _, _, _ = await opc.manage_output_tags( server_id='server1', config=config, data=data, metadata=metadata['metadata'], ) + assert output_data is False - assert opc_metrics == {'tag1': side_effect[0][0], 'tag2': side_effect[1][0]} - opc.write_data.assert_has_calls( - [ - call( - server_id='server1', - tag='tag1', - data=0.75, - data_type='float', - tag_type='prediction', - metadata=metadata['metadata'], - ), - call( - server_id='server1', - tag='tag2', - data=0.95, - data_type='float', - tag_type='confidence', - metadata=metadata['metadata'], - ), - ] - ) + assert opc_metrics == {'tag1': 0.1, 'tag2': None} @mark.asyncio async def test_manage_output_tags_do_nothing(opc): - opc.write_data = AsyncMock(return_value=(0.1, None)) + opc._write_tags_from_config = AsyncMock() data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) - config = { - '_invalid_key': {'tag1': {'data_type': 'float'}}, - } + config = {'_invalid_key': {'tag1': {'data_type': 'float'}}} + output_data, opc_metrics, _, _, _ = await opc.manage_output_tags( server_id='server1', config=config, data=data, metadata=metadata['metadata'], ) + assert output_data is True assert opc_metrics == {} - opc.write_data.assert_not_called() + opc._write_tags_from_config.assert_not_called() @mark.asyncio @@ -507,21 +666,11 @@ def test_process_confidence_generic_failure(opc): @mark.asyncio -async def test_manage_output_tags_session_bad(opc): - opc.write_data = AsyncMock( +async def test_manage_output_tags_merges_error_flags(opc): + opc._write_tags_from_config = AsyncMock( side_effect=[ - ( - None, - { - 'opc_error_kind': 'session_bad', - 'opc_status': 'BadSessionIdInvalid', - 'notification_id': 'OPC_WRITE_DATA_ERROR_1', - 'message': 'bad', - 'block': 'opc_repository', - 'level': NotificationLevel.ERROR, - }, - ), - (0.2, None), + ({'tag1': None}, True, 'BadSessionIdInvalid', False), + ({'tag2': 0.2}, False, None, True), ] ) data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) @@ -529,6 +678,7 @@ async def test_manage_output_tags_session_bad(opc): 'prediction_tags': {'tag1': {'data_type': 'float'}}, 'confidence_tags': {'tag2': {'data_type': 'float'}}, } + ( success, metrics, @@ -536,42 +686,12 @@ async def test_manage_output_tags_session_bad(opc): opc_status, reconnect_in_progress, ) = await opc.manage_output_tags('server1', config, data, metadata['metadata']) + assert success is False assert session_bad_seen is True - assert reconnect_in_progress is False - assert opc_status == 'BadSessionIdInvalid' - assert metrics['tag1'] is None - assert metrics['tag2'] == 0.2 - - -@mark.asyncio -async def test_manage_output_tags_reconnect_in_progress(opc): - opc.write_data = AsyncMock( - return_value=( - None, - { - 'opc_error_kind': 'reconnect_in_progress', - 'notification_id': 'OPC_WRITE_RECONNECT_IN_PROGRESS_1', - 'message': 'skipped', - 'block': 'opc_repository', - 'level': NotificationLevel.WARNING, - }, - ) - ) - data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]}) - config = {'prediction_tags': {'tag1': {'data_type': 'float'}}} - ( - success, - metrics, - session_bad_seen, - opc_status, - reconnect_in_progress, - ) = await opc.manage_output_tags('server1', config, data, metadata['metadata']) - assert success is False - assert session_bad_seen is False assert reconnect_in_progress is True - assert opc_status is None - assert metrics['tag1'] is None + assert opc_status == 'BadSessionIdInvalid' + assert metrics == {'tag1': None, 'tag2': 0.2} def test_process_confidence_reconnect_in_progress(opc): From cd1be2430acf871600778f5006650a54ad8046ff Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 19 May 2026 11:45:08 -0300 Subject: [PATCH 05/64] SIENTIAPDE-1811 Update .gitignore, requirements, and enhance OPC UA error handling - Added new entries to .gitignore for openspec and cursor directories. - Updated sientia-dataops-library dependency version in requirements-light.txt from 1.10.4 to 1.12.0. - Enhanced OPC UA communication by refining reconnect logic and error handling in opc_repository.py, including the introduction of a reconnect flag and improved session management. - Updated tests to cover new reconnect scenarios and ensure robust error handling for protocol states. --- .gitignore | 4 +- docs/opc-communication.md | 30 ++- laborious/utils/repository/opc_repository.py | 194 ++++++++++++++---- laborious/worker/worker.py | 1 + requirements-light.txt | 2 +- .../utils/repository/test_opc_repository.py | 96 ++++++++- 6 files changed, 266 insertions(+), 61 deletions(-) diff --git a/.gitignore b/.gitignore index c35c913..c849e0a 100644 --- a/.gitignore +++ b/.gitignore @@ -53,4 +53,6 @@ catboost_info/ .mypy_cache/ mlruns/ -relatorio* \ No newline at end of file +relatorio* +openspec/ +.cursor/ \ No newline at end of file diff --git a/docs/opc-communication.md b/docs/opc-communication.md index 3084d05..15d78e7 100644 --- a/docs/opc-communication.md +++ b/docs/opc-communication.md @@ -12,7 +12,7 @@ Worker (long-lived) ├── connect / disconnect / validate_connection (read-only) ├── _connect_locked / _reconnect_locked (under _connection_lock) ├── write_data (single attempt per call) - └── background reconnect on Tier-1 Bad* + └── background reconnect on Tier-1 Bad*, protocol closed, or stale session Temporal activity write_opc_data └── OPC.manage_output_tags → write_data per tag (sequential per activity) @@ -26,8 +26,8 @@ One worker process holds one `OpcRepository` instance per configured server. Mul |-------|----------| | Startup | `init_opc()` creates repositories and calls `connect()` → `_connect_locked()` | | Steady state | `validate_connection()` is read-only (`protocol.state` only); `_session_ready` is checked in `write_data` | -| Tier-1 Bad* | `_start_reconnect_on_bad` → `_run_reconnect_on_bad` → `_reconnect_locked()` (respects `reconnection_interval`) | -| Write | `write_data()` checks `_session_ready`, validates, then one `get_node` + `write_value` | +| Tier-1 Bad* / protocol closed | `_start_reconnect` → `_run_reconnect` → `_reconnect_locked()` (respects `reconnection_interval`) | +| Write | `write_data()` checks reconnect task, `_session_ready`, validates protocol, then one `get_node` + `write_value` | | Shutdown | `close()` disconnects all repositories | ### Session and channel timeouts @@ -36,7 +36,7 @@ Requested session and secure-channel lifetime: **10 minutes** (`OPC_UA_SESSION_A ### Reconnection interval -`OPC_RECONNECTION_INTERVAL` is in **seconds** (default `120`). It gates **background** reconnect after Tier-1 `Bad*` (`last_reconnection_time` is updated only in `_reconnect_locked()`). It limits load on the OPC server when many workflows fail at once. +`OPC_RECONNECTION_INTERVAL` is in **seconds** (default `120`). It gates **background** reconnect after Tier-1 `Bad*`, closed protocol, or stale session (`last_reconnection_time` is updated only in `_reconnect_locked()`). It limits load on the OPC server when many workflows fail at once. ## Concurrency: connection lock and session readiness @@ -61,15 +61,17 @@ Public `connect()` / `disconnect()` acquire the lock and call `_connect_locked() **Write path (`write_data`):** -1. If `_session_ready` is cleared → **fail immediately** (`opc_error_kind=reconnect_in_progress`). -2. `validate_connection()` checks `protocol.state` only (read-only). -3. Single `get_node` + `write_value` (no retry). +1. If a reconnect task is **in flight** → **fail immediately** (`opc_error_kind=reconnect_in_progress`). +2. If `_session_ready` is cleared and no task is running → schedule reconnect (`SessionNotReady`); fail with `connection_lost` or `reconnect_in_progress` if a task started. +3. `validate_connection()` checks `protocol.state` only (read-only). If closed → schedule reconnect (`ProtocolClosed`) and fail with `opc_error_kind=connection_lost`. +4. Single `get_node` + `write_value` (no retry). Tier-1 `Bad*` on write also schedules reconnect. -**Reconnect path (`_run_reconnect_on_bad`):** +**Reconnect path (`_run_reconnect`):** -1. `_start_reconnect_on_bad` clears `_session_ready` and schedules the task when the interval allows. +1. `_start_reconnect` clears `_session_ready` and schedules the task when the interval allows and `_allow_reconnect` is true. 2. `async with _connection_lock:` → `_reconnect_locked()`. 3. `_session_ready` is set on successful `_open_session()`. +4. `disconnect()` sets `_allow_reconnect=False` so shutdown does not respawn sessions. A second `_connect_locked()` while a session is already open raises `OpcSessionAlreadyConnectedError` (disconnect first). @@ -77,9 +79,15 @@ A second `_connect_locked()` while a session is already open raises `OpcSessionA **Future threads:** replace `asyncio.Lock` / `Event` with `threading` primitives or route all OPC I/O through one dedicated loop. -## Tier-1 `Bad*` errors and reconnect +## Reconnect triggers -When the server invalidates the session (e.g. `BadSessionIdInvalid`) but the client still sees transport as open, `write_data` fails once, records the OPC status in metrics, and **schedules** reconnect if: +Background reconnect is scheduled when: + +- `validate_connection()` sees a closed or missing protocol (`ProtocolClosed`). +- `_session_ready` is clear after a failed reconnect (`SessionNotReady`). +- A write raises a Tier-1 `UaStatusCodeError` in `RECONNECTABLE_OPC_BAD_NAMES`. + +For Tier-1 `Bad*` when the server invalidates the session (e.g. `BadSessionIdInvalid`) but the client still sees transport as open, `write_data` fails once, records the OPC status in metrics, and **schedules** reconnect if: - The exception is a `UaStatusCodeError` whose name is in `RECONNECTABLE_OPC_BAD_NAMES` (see plan), and - `reconnection_interval` has elapsed since `last_reconnection_time`, and diff --git a/laborious/utils/repository/opc_repository.py b/laborious/utils/repository/opc_repository.py index b164ce1..2ecc27a 100644 --- a/laborious/utils/repository/opc_repository.py +++ b/laborious/utils/repository/opc_repository.py @@ -195,8 +195,18 @@ class OpcRepository(SientiaMonitoring): self._connection_lock = asyncio.Lock() self._session_ready = asyncio.Event() self._reconnect_task: asyncio.Task[None] | None = None + self._allow_reconnect = True def _opc_debug_tags(self, session_id: str) -> dict[str, str]: + """ + Build Prometheus/log label tags for OPC session-scoped metrics. + + Args: + session_id (str): OPC UA session token string. + + Return: + dict[str, str]: Labels pod_id, server_name, runtime, opc_server_id, session_id. + """ return { 'pod_id': str(getattr(self, 'pod_id', 'unknown')), 'server_name': self.server_name, @@ -234,6 +244,12 @@ class OpcRepository(SientiaMonitoring): ).total_seconds() > self.reconnection_interval def _not_connected_error(self) -> dict[str, Any]: + """ + Build the standard error payload when validate_connection finds no open protocol. + + Return: + dict[str, Any]: Notification fields for OPC_CONNECTION_NOT_READY. + """ return { 'notification_id': f'OPC_CONNECTION_NOT_READY_{self.id}', 'message': f'OPC server {self.id} is not connected', @@ -491,8 +507,12 @@ class OpcRepository(SientiaMonitoring): async def disconnect(self) -> None: """ Gracefully disconnect from the OPC server under the connection lock. + + Disables background reconnect so late writes during worker shutdown do not + respawn sessions. """ async with self._connection_lock: + self._allow_reconnect = False await self._disconnect_locked() async def validate_connection(self) -> tuple[bool, dict[str, Any]]: @@ -509,58 +529,77 @@ class OpcRepository(SientiaMonitoring): self.error(f'OPC server {self.id} is not connected', self.metadata) return False, self._not_connected_error() - async def _start_reconnect_on_bad(self, opc_status: str, session_id: str) -> None: + def _reconnect_task_in_progress(self) -> bool: """ - Schedule a background reconnect if interval and task state allow it. + Return whether a background reconnect task is currently running. + + Return: + bool: True when a reconnect task exists and has not finished. + """ + return self._reconnect_task is not None and not self._reconnect_task.done() + + async def _start_reconnect(self, reason: str, session_id: str) -> None: + """ + Schedule a background reconnect when allowed by interval and task state. + + Clears _session_ready before starting the task. No-op when _allow_reconnect is + False, the reconnection window has not elapsed, or a reconnect is already running. Args: - opc_status (str): OPC UA status name that triggered reconnect. + reason (str): Trigger for reconnect (OPC status name or synthetic reason). session_id (str): Session token before failure. """ + if not self._allow_reconnect: + return if not self._reconnection_window_elapsed(): self.warning( f'OPC reconnect skipped reason=reconnection_window opc_server_id={self.id} ' - f'opc_status={opc_status}', + f'reconnect_reason={reason}', self.metadata, ) return - if self._reconnect_task is not None and not self._reconnect_task.done(): + if self._reconnect_task_in_progress(): self.warning( f'OPC reconnect skipped reason=in_progress opc_server_id={self.id} ' - f'opc_status={opc_status}', + f'reconnect_reason={reason}', self.metadata, ) return self._session_ready.clear() self.info( - f'OPC reconnect scheduled after opc_status={opc_status} opc_server_id={self.id} ' + f'OPC reconnect scheduled reconnect_reason={reason} opc_server_id={self.id} ' f'old_session_id={session_id}', self.metadata, ) - self._reconnect_task = asyncio.create_task( - self._run_reconnect_on_bad(opc_status, session_id) - ) + self._reconnect_task = asyncio.create_task(self._run_reconnect(reason, session_id)) - async def _run_reconnect_on_bad(self, opc_status: str, session_id: str) -> None: + async def _run_reconnect(self, reason: str, session_id: str) -> None: """ - Tear down and re-establish the OPC UA session under the connection lock. + Background task that tears down and re-establishes the OPC UA session. Args: - opc_status (str): OPC UA status that triggered reconnect. - session_id (str): Previous session token string. + reason (str): Trigger for reconnect (OPC status or ProtocolClosed). + session_id (str): Previous session token string for logging. """ try: async with self._connection_lock: self.info( - f'OPC reconnect started opc_status={opc_status} opc_server_id={self.id} ' + f'OPC reconnect started reconnect_reason={reason} opc_server_id={self.id} ' f'old_session_id={session_id}', self.metadata, ) - await self._reconnect_locked() + success, error = await self._reconnect_locked() + if not success: + self.error( + f'OPC reconnect failed reconnect_reason={reason} opc_server_id={self.id}', + self.metadata, + ) + if error: + self.error(error.get('message', ''), self.metadata) except Exception: self.error( - f'OPC reconnect task failed opc_server_id={self.id} opc_status={opc_status}', + f'OPC reconnect task failed opc_server_id={self.id} reconnect_reason={reason}', self.metadata, ) self.error(traceback.format_exc(), self.metadata) @@ -593,6 +632,14 @@ class OpcRepository(SientiaMonitoring): async def _emit_opc_write_metric( self, session_id: str, result: str, metadata: dict[str, Any] | None ) -> None: + """ + Emit opc_write_attempts_total for a single write attempt outcome. + + Args: + session_id (str): OPC UA session token string, or "unknown". + result (str): Outcome label (OK, OPC status name, ProtocolClosed, etc.). + metadata (dict[str, Any] | None): Write context for model_id/model_name labels. + """ await self.emit_metric( metrics.OPC_WRITE_ATTEMPTS_TOTAL, { @@ -611,6 +658,20 @@ class OpcRepository(SientiaMonitoring): opc_error_kind: str | None = None, opc_status: str | None = None, ) -> dict[str, Any]: + """ + Build a structured error dict returned from failed write_data paths. + + Args: + notification_id (str): Stable notification identifier. + message (str): Human-readable failure message. + level (NotificationLevel): Severity for downstream notifications. + attachment_content (str | None): Optional traceback or diagnostic text. + opc_error_kind (str | None): Classifier (session_bad, connection_lost, etc.). + opc_status (str | None): OPC UA status name or synthetic reason. + + Return: + dict[str, Any]: Error payload consumed by the OPC activity layer. + """ payload: dict[str, Any] = { 'notification_id': notification_id, 'message': message, @@ -656,7 +717,7 @@ class OpcRepository(SientiaMonitoring): f'model_name={metadata.get("model_name", "unknown")} node={node} phase={phase}', metadata, ) - await self._start_reconnect_on_bad(opc_status, session_id) + await self._start_reconnect(opc_status, session_id) return False, self._write_failure_payload( notification_id=f'OPC_WRITE_DATA_ERROR_{self.id}', message=f'Failed to {phase} on OPC server: {exc} | metadata: {metadata}', @@ -665,32 +726,89 @@ class OpcRepository(SientiaMonitoring): opc_status=opc_status, ) + async def _write_reconnect_in_progress( + self, metadata: dict[str, Any] + ) -> tuple[bool, dict[str, Any]]: + """ + Fail a write because a background reconnect task is already running. + + Args: + metadata (dict[str, Any]): Write context passed through to the activity. + + Return: + tuple[bool, dict[str, Any]]: (False, error info with opc_error_kind reconnect_in_progress). + """ + await self._emit_opc_write_metric('unknown', 'ReconnectInProgress', metadata) + self.warning( + f'OPC write rejected reconnect_in_progress opc_server_id={self.id} ' + f'model_id={metadata.get("model_id", "unknown")} ' + f'model_name={metadata.get("model_name", "unknown")}', + metadata, + ) + return False, { + 'notification_id': f'OPC_WRITE_RECONNECT_IN_PROGRESS_{self.id}', + 'message': f'OPC write skipped: reconnect in progress | metadata: {metadata}', + 'block': 'opc_repository', + 'level': NotificationLevel.WARNING, + 'opc_error_kind': 'reconnect_in_progress', + } + + async def _write_connection_lost( + self, metadata: dict[str, Any], opc_status: str + ) -> tuple[bool, dict[str, Any]]: + """ + Fail a write after scheduling reconnect for a closed or stale session. + + Args: + metadata (dict[str, Any]): Write context passed through to the activity. + opc_status (str): Synthetic reason (ProtocolClosed, SessionNotReady). + + Return: + tuple[bool, dict[str, Any]]: (False, error info with opc_error_kind connection_lost). + """ + await self._emit_opc_write_metric('unknown', opc_status, metadata) + return False, self._write_failure_payload( + notification_id=f'OPC_WRITE_CONNECTION_LOST_{self.id}', + message=f'OPC write skipped: connection lost ({opc_status}) | metadata: {metadata}', + level=NotificationLevel.WARNING, + opc_error_kind='connection_lost', + opc_status=opc_status, + ) + async def write_data( self, node: str, value: Any, data_type: str, metadata: dict[str, Any] ) -> tuple[bool, dict[str, Any]]: """ - Write data to OPC server with a single attempt and Tier-1 Bad* reconnect scheduling. - """ - if not self._session_ready.is_set(): - await self._emit_opc_write_metric('unknown', 'ReconnectInProgress', metadata) - self.warning( - f'OPC write rejected reconnect_in_progress opc_server_id={self.id} ' - f'model_id={metadata.get("model_id", "unknown")} ' - f'model_name={metadata.get("model_name", "unknown")}', - metadata, - ) - return False, { - 'notification_id': f'OPC_WRITE_RECONNECT_IN_PROGRESS_{self.id}', - 'message': f'OPC write skipped: reconnect in progress | metadata: {metadata}', - 'block': 'opc_repository', - 'level': NotificationLevel.WARNING, - 'opc_error_kind': 'reconnect_in_progress', - } + Write data to OPC server with a single attempt and background reconnect scheduling. - is_connected, error = await self.validate_connection() + Reconnect is scheduled on Tier-1 Bad*, closed protocol, or stale session readiness. + There is no retry within the same call. + + Args: + node (str): OPC UA node id to write. + value (Any): Value to convert and send. + data_type (str): Logical type key (float, int, bool, str, double). + metadata (dict[str, Any]): Activity context (model_id, model_name, etc.). + + Return: + tuple[bool, dict[str, Any]]: (True, {response_time}) on success, or + (False, structured error info) on failure. + """ + if self._reconnect_task_in_progress(): + return await self._write_reconnect_in_progress(metadata) + + if not self._session_ready.is_set(): + session_id = _opc_authentication_token_str(self.client) + await self._start_reconnect('SessionNotReady', session_id) + if self._reconnect_task_in_progress(): + return await self._write_reconnect_in_progress(metadata) + return await self._write_connection_lost(metadata, 'SessionNotReady') + + is_connected, _error = await self.validate_connection() if not is_connected: - await self._emit_opc_write_metric('unknown', 'NotConnected', metadata) - return False, error + session_id = _opc_authentication_token_str(self.client) + await self._start_reconnect('ProtocolClosed', session_id) + return await self._write_connection_lost(metadata, 'ProtocolClosed') start_time = time.time() session_id = _opc_authentication_token_str(self.client) diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index fe78753..69e6d2f 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -205,6 +205,7 @@ async def main(): activities.cleanup_minio_objects_expired, activities.repeat_last_prediction, activities.export_data_to_postgres, + activities.export_payload_to_postgres, activities.write_metrics, # Pi Web API activities.write_pi_web_api_data, diff --git a/requirements-light.txt b/requirements-light.txt index 91a9c33..f105052 100644 --- a/requirements-light.txt +++ b/requirements-light.txt @@ -3,7 +3,7 @@ psycopg2-binary sqlalchemy asyncua redis -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.10.4 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.0 prometheus-client botocore boto3 diff --git a/tests/laborious/utils/repository/test_opc_repository.py b/tests/laborious/utils/repository/test_opc_repository.py index f1fd81a..365a400 100644 --- a/tests/laborious/utils/repository/test_opc_repository.py +++ b/tests/laborious/utils/repository/test_opc_repository.py @@ -257,6 +257,7 @@ async def test_disconnect(opc_repository, mock_client): opc_repository._disconnection_fallback.assert_called_once() assert opc_repository.client is None + assert opc_repository._allow_reconnect is False @pytest.mark.asyncio @@ -331,14 +332,19 @@ async def test_write_data_validate_connection_do_nothing(opc_repository): @pytest.mark.asyncio async def test_write_data_validate_connection_failed(opc_repository): - opc_repository.validate_connection = AsyncMock(return_value=(False, {})) - opc_repository.client = AsyncMock() + opc_repository.client = MagicMock() + opc_repository.client.uaclient.protocol = MagicMock(state='closed') + opc_repository._start_reconnect = AsyncMock() - result = await opc_repository.write_data('ns=2;s=TestNode', 42.0, 'float', metadata['metadata']) + is_success, error_data = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'float', metadata['metadata'] + ) - opc_repository.validate_connection.assert_called_once() - opc_repository.client.get_node.assert_not_called() - assert result == (False, {}) + opc_repository._start_reconnect.assert_called_once() + assert opc_repository._start_reconnect.call_args.args[0] == 'ProtocolClosed' + assert is_success is False + assert error_data['opc_error_kind'] == 'connection_lost' + assert error_data['opc_status'] == 'ProtocolClosed' @pytest.mark.asyncio @@ -439,7 +445,7 @@ def test_is_reconnectable_opcua_bad(): async def test_write_data_bad_session_id_invalid_schedules_reconnect(opc_repository, mock_client): opc_repository.validate_connection = AsyncMock(return_value=(True, {})) opc_repository.client = mock_client - opc_repository._start_reconnect_on_bad = AsyncMock() + opc_repository._start_reconnect = AsyncMock() mock_node = AsyncMock() mock_client.get_node = MagicMock(return_value=mock_node) mock_node.write_value.side_effect = BadSessionIdInvalid() @@ -449,7 +455,7 @@ async def test_write_data_bad_session_id_invalid_schedules_reconnect(opc_reposit ) mock_node.write_value.assert_called_once() - opc_repository._start_reconnect_on_bad.assert_called_once() + opc_repository._start_reconnect.assert_called_once() assert is_success is False assert error_data['opc_error_kind'] == 'session_bad' assert error_data['opc_status'] == 'BadSessionIdInvalid' @@ -476,15 +482,85 @@ async def test_write_data_reconnect_in_progress_immediate(opc_repository): @pytest.mark.asyncio -async def test_start_reconnect_on_bad_skips_within_interval(opc_repository): +async def test_start_reconnect_skips_within_interval(opc_repository): opc_repository.last_reconnection_time = datetime.now() opc_repository.reconnection_interval = 3600 - await opc_repository._start_reconnect_on_bad('BadSessionIdInvalid', 'tok') + await opc_repository._start_reconnect('BadSessionIdInvalid', 'tok') assert opc_repository._reconnect_task is None +@pytest.mark.asyncio +async def test_write_data_protocol_closed_schedules_reconnect(opc_repository): + opc_repository.client = MagicMock() + opc_repository.client.uaclient.protocol = MagicMock(state='closed') + opc_repository._start_reconnect = AsyncMock() + + is_success, error_data = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'float', metadata['metadata'] + ) + + opc_repository._start_reconnect.assert_called_once() + assert opc_repository._start_reconnect.call_args.args[0] == 'ProtocolClosed' + assert is_success is False + assert error_data['opc_error_kind'] == 'connection_lost' + assert error_data['opc_status'] == 'ProtocolClosed' + + +@pytest.mark.asyncio +async def test_write_data_protocol_closed_skips_reconnect_within_interval(opc_repository): + opc_repository.client = MagicMock() + opc_repository.client.uaclient.protocol = MagicMock(state='closed') + opc_repository.last_reconnection_time = datetime.now() + opc_repository.reconnection_interval = 3600 + + is_success, error_data = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'float', metadata['metadata'] + ) + + assert opc_repository._reconnect_task is None + assert is_success is False + assert error_data['opc_error_kind'] == 'connection_lost' + + +@pytest.mark.asyncio +async def test_write_data_after_failed_reconnect_schedules_again(opc_repository): + opc_repository._session_ready.clear() + opc_repository.reconnection_interval = 0 + opc_repository.last_reconnection_time = None + opc_repository._reconnect_locked = AsyncMock( + return_value=(False, {'message': 'connect failed'}) + ) + + await opc_repository.write_data('ns=2;s=TestNode', 42.0, 'float', metadata['metadata']) + await asyncio.sleep(0.1) + assert opc_repository._reconnect_locked.call_count == 1 + assert not opc_repository._reconnect_task_in_progress() + + await opc_repository.write_data('ns=2;s=TestNode', 42.0, 'float', metadata['metadata']) + await asyncio.sleep(0.1) + assert opc_repository._reconnect_locked.call_count == 2 + + +@pytest.mark.asyncio +async def test_write_data_after_disconnect_does_not_schedule_reconnect(opc_repository, mock_client): + opc_repository.client = mock_client + proto = MagicMock() + proto.state = 'closed' + mock_client.uaclient = MagicMock(protocol=proto) + opc_repository._disconnection_fallback = AsyncMock(return_value=[]) + await opc_repository.disconnect() + + is_success, error_data = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'float', metadata['metadata'] + ) + + assert opc_repository._reconnect_task is None + assert is_success is False + assert error_data['opc_error_kind'] == 'connection_lost' + + @pytest.mark.asyncio async def test_parallel_bad_writes_single_reconnect_task(opc_repository, mock_client): opc_repository.validate_connection = AsyncMock(return_value=(True, {})) From 7153f1da0dad21d70b8b483cbe72c6318bb0a062 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 19 May 2026 14:13:23 -0300 Subject: [PATCH 06/64] SIENTIAPDE-1646 Remove requirements-light.txt and update requirements.txt to specify versions for asyncua and new sientia dependencies. --- requirements-light.txt | 12 ------------ requirements-local.txt | 18 ++++++++++++++++++ requirements.txt | 6 +++--- 3 files changed, 21 insertions(+), 15 deletions(-) delete mode 100644 requirements-light.txt create mode 100644 requirements-local.txt diff --git a/requirements-light.txt b/requirements-light.txt deleted file mode 100644 index f105052..0000000 --- a/requirements-light.txt +++ /dev/null @@ -1,12 +0,0 @@ -temporalio -psycopg2-binary -sqlalchemy -asyncua -redis -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.0 -prometheus-client -botocore -boto3 -s3fs -pyarrow -mlflow \ No newline at end of file diff --git a/requirements-local.txt b/requirements-local.txt new file mode 100644 index 0000000..73a9a47 --- /dev/null +++ b/requirements-local.txt @@ -0,0 +1,18 @@ +temporalio +psycopg2-binary +sqlalchemy +asyncua==1.0.6 +redis +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.1 +git+ssh://git@github.com/Aignosi/sientia-model-library.git@0.10.0 +prometheus-client +botocore +boto3 +s3fs +pyarrow +kaleido +hyperopt +shap +pycurl +scipy<1.14.0 +scikit-learn==1.5.2 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6c5c242..eff54bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ temporalio psycopg2-binary sqlalchemy -asyncua +asyncua==1.0.6 redis -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.10.4 -git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.41.0 +sientia_do>=1.12.1 +sientia_model>=0.8.2 prometheus-client botocore boto3 From 65698107562d2404c37cd171349099e536ee7e36 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 19 May 2026 14:41:58 -0300 Subject: [PATCH 07/64] Update requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index eff54bd..cea671f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ asyncua==1.0.6 redis sientia_do>=1.12.1 sientia_model>=0.8.2 +sientia>0.40.0 prometheus-client botocore boto3 @@ -15,4 +16,4 @@ hyperopt shap pycurl scipy<1.14.0 -scikit-learn==1.5.2 \ No newline at end of file +scikit-learn==1.5.2 From d856150e240c1012ceb2722cdb088343fe577213 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 19 May 2026 14:45:26 -0300 Subject: [PATCH 08/64] Update requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cea671f..4ad005d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ sqlalchemy asyncua==1.0.6 redis sientia_do>=1.12.1 -sientia_model>=0.8.2 sientia>0.40.0 prometheus-client botocore From 2ccda3e44055eba3bbfb77d9fddeb8274b76059c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 19 May 2026 17:07:18 -0300 Subject: [PATCH 09/64] SIENTIAPDE-1646 Refactor worker task queue management and update README - Introduced runtime-scoped task queues for workflows, replacing legacy queue names. - Updated worker implementation to utilize `sientia_do.temporal.worker.prepare_worker`. - Added `RUNTIME` environment variable to configure task queue suffixes. - Enhanced README documentation to reflect changes in task queue structure and worker setup. --- README.md | 15 +++- laborious/worker/prepare_worker.py | 73 ------------------- laborious/worker/worker.py | 51 +++++++++---- .../worker/test_runtime_task_queues.py | 14 ++++ values.yaml | 3 + 5 files changed, 64 insertions(+), 92 deletions(-) delete mode 100644 laborious/worker/prepare_worker.py create mode 100644 tests/laborious/worker/test_runtime_task_queues.py diff --git a/README.md b/README.md index 0de066f..0b5c844 100644 --- a/README.md +++ b/README.md @@ -126,10 +126,15 @@ Laborious uses a Temporal-based architecture with strong separation of concerns ### Key Components #### **Worker (`laborious/worker/worker.py`)** -- Temporal client setup, worker lifecycle, task queues +- Temporal client setup, four workers via `sientia_do.temporal.worker.prepare_worker` +- Runtime-scoped task queues: `{workflow}-{RUNTIME}-queue` for all workflows - Metrics server initialization, notification handler setup - Graceful shutdown and autoscaling-friendly behavior +**Breaking (schedulers):** drift and simple_metrics queues are no longer `drift-queue` / +`simple_metrics-queue`. Use `drift-{RUNTIME}-queue` and `simple_metrics-{RUNTIME}-queue` +matching the worker pod `RUNTIME` env (same as `predictions_batch` / `minimal_retrain`). + #### **Workflows (`laborious/workflows/`)** - `predictions_batch.py`: Batch prediction entry point - `sub_workflows/prediction_process.py`: Core prediction pipeline @@ -802,6 +807,7 @@ See [OPC UA Communication](#opc-ua-communication) for semantics, concurrency, an |----------|-------------|---------|----------| | `TEMPORAL_HOST` | Temporal server address | `localhost:7233` | Yes | | `TEMPORAL_NAMESPACE` | Temporal namespace | `laborious` | No | +| `RUNTIME` | Task queue suffix for all workflows (`{workflow}-{RUNTIME}-queue`) | _(none)_ | Yes | | `POSTGRES_HOST` | PostgreSQL hostname | `localhost` | Yes | | `POSTGRES_PORT` | PostgreSQL port | `5432` | Yes | | `POSTGRES_USER` | PostgreSQL username | `sientia` | Yes | @@ -1097,8 +1103,7 @@ laborious/ │ ├── prediction_process.py # Core prediction workflow │ └── format_and_export_prediction.py # Export workflow ├── worker/ # Worker implementation -│ ├── worker.py # Main worker orchestrator -│ └── prepare_worker.py # Worker factory with autoscaling config +│ └── worker.py # Main worker orchestrator (uses sientia_do prepare_worker) ├── utils/ # Utility functions │ ├── connectors_config.py # Environment-driven config builders │ ├── models/ # Data models @@ -1179,7 +1184,9 @@ export LOG_LEVEL=DEBUG ### Scaling Considerations - **Horizontal Scaling**: Deploy multiple worker instances -- **Task Queue Distribution**: Use multiple task queues for different workflow types +- **Task Queue Distribution**: One worker pod per `RUNTIME`; queues are + `predictions_batch-{RUNTIME}-queue`, `minimal_retrain-{RUNTIME}-queue`, + `drift-{RUNTIME}-queue`, `simple_metrics-{RUNTIME}-queue` - **Database Performance**: Optimize indexes and connection pooling - **MLFlow Performance**: Configure appropriate model serving resources diff --git a/laborious/worker/prepare_worker.py b/laborious/worker/prepare_worker.py deleted file mode 100644 index 9766af2..0000000 --- a/laborious/worker/prepare_worker.py +++ /dev/null @@ -1,73 +0,0 @@ -import os -import re -from collections.abc import Sequence -from typing import Any - -from sientia_do.observability.logger import Logger -from temporalio.client import Client -from temporalio.worker import PollerBehaviorAutoscaling, Worker - -parameters = [ - ('MAX_CONCURRENT_WORKFLOW_TASKS', '200'), - ('MAX_CONCURRENT_ACTIVITIES', '200'), - ('MAX_CONCURRENT_LOCAL_ACTIVITIES', '200'), - ('MAX_CACHED_WORKFLOWS', '200'), - ('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10'), - ('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100'), - ('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200'), - ('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10'), - ('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100'), - ('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200'), -] - - -def camel_to_snake(text: str) -> str: - """Convert camelCase or PascalCase to snake_case.""" - text = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', text) - text = re.sub('([a-z0-9])([A-Z])', r'\1_\2', text) - return text.lower() - - -def prepare_worker( - main_workflow: type, - other_workflows: Sequence[type], - activities: Sequence[Any], - temporal_client: Client, - logger: Logger, -) -> Worker: - main_workflow_name = main_workflow.__name__.upper() - - queue_name = f'{camel_to_snake(main_workflow.__name__)}-queue' - - local_workflow_parameters = {} - - for parameter in parameters: - local_workflow_parameters[parameter[0]] = int( - os.getenv(main_workflow_name + '_' + parameter[0], parameter[1]) - ) - - logger.info(f'Preparing worker for {main_workflow_name} with queue {queue_name}') - logger.info(f'Worker runtime config: {local_workflow_parameters}') - - return Worker( - temporal_client, - task_queue=queue_name, - workflows=[main_workflow, *other_workflows], - activities=[*activities], - max_concurrent_workflow_tasks=local_workflow_parameters['MAX_CONCURRENT_WORKFLOW_TASKS'], - max_concurrent_activities=local_workflow_parameters['MAX_CONCURRENT_ACTIVITIES'], - max_concurrent_local_activities=local_workflow_parameters[ - 'MAX_CONCURRENT_LOCAL_ACTIVITIES' - ], - max_cached_workflows=local_workflow_parameters['MAX_CACHED_WORKFLOWS'], - workflow_task_poller_behavior=PollerBehaviorAutoscaling( - minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MINIMUM'], - initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_INITIAL'], - maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM'], - ), - activity_task_poller_behavior=PollerBehaviorAutoscaling( - minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MINIMUM'], - initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_INITIAL'], - maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM'], - ), - ) diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index 69e6d2f..7a420ba 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -5,12 +5,14 @@ This module provides the main worker implementation for the Sientia DataOps Labo It orchestrates Temporal workers, manages task queues, and handles the lifecycle of prediction and retraining workflows. -The worker supports multiple task queues: -- predictions_batch-queue: Handles batch prediction workflows (heavy workload) - Includes activities for MLFlow, data quality gates, OPC export, PI Web API export, and PostgreSQL -- minimal_retrain-queue: Handles model retraining workflows -- drift-queue: Handles drift detection workflows -- simple_metrics-queue: Handles simple metrics calculation workflows +The worker supports multiple runtime-scoped task queues (via ``sientia_do.temporal.worker.prepare_worker``): +- predictions_batch-{runtime}-queue: Batch prediction workflows (heavy workload) +- minimal_retrain-{runtime}-queue: Model retraining workflows +- drift-{runtime}-queue: Drift detection workflows +- simple_metrics-{runtime}-queue: Simple metrics workflows + +``RUNTIME`` must be set; it is passed to every ``prepare_worker`` call. Schedulers must use the +same queue names (breaking change vs legacy ``drift-queue`` / ``simple_metrics-queue``). Key Features: - Resource-based scaling with WorkerTuner (CPU and memory aware) @@ -21,6 +23,7 @@ Key Features: - Multiple worker instances for different workflow types Environment Variables: +- RUNTIME: Required non-empty string; suffix for all task queue names - TEMPORAL_HOST: Temporal server address (default: localhost:7233) - TEMPORAL_NAMESPACE: Temporal namespace (default: laborious) - POD_ID: Kubernetes pod identifier for metrics @@ -40,6 +43,7 @@ with workflow.unsafe.imports_passed_through(): from prometheus_client import start_http_server from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import get_logger + from sientia_do.temporal.worker.prepare_worker import prepare_worker from sientia_do.utils.connectors_config import ( build_api_config, build_mongodb_config, @@ -53,7 +57,6 @@ with workflow.unsafe.imports_passed_through(): build_mlflow_config, build_opc_config, ) - from laborious.worker.prepare_worker import prepare_worker from laborious.workflows.drift import Drift from laborious.workflows.minimal_retrain import MinimalRetrain from laborious.workflows.predictions_batch import PredictionsBatch @@ -100,10 +103,21 @@ async def main(): logger.custom_info(f'Starting Worker with POD_ID: {POD_ID}', metadata) - logger.custom_info('Starting prometheus client...', metadata) + runtime = os.getenv('RUNTIME', '').strip() + if not runtime: + logger.custom_critical( + 'RUNTIME environment variable is required and must be non-empty', + metadata, + ) + metrics.APP_UP.labels(pod_id=POD_ID).set(0) + sys.exit(1) + + metadata_runtime = {**metadata, 'runtime': runtime} + + logger.custom_info('Starting prometheus client...', metadata_runtime) start_prometheus_server() - logger.custom_info('Starting Notification Handler...', metadata) + logger.custom_info('Starting Notification Handler...', metadata_runtime) mongo_config = build_mongodb_config() notification_handler = NotificationHandler( @@ -113,7 +127,7 @@ async def main(): project_name=os.getenv('PROJECT_NAME', 'laborious'), ) - logger.custom_info('Starting Activities...', metadata) + logger.custom_info('Starting Activities...', metadata_runtime) activities = Activities( postgres_config=build_postgres_config(), @@ -125,10 +139,13 @@ async def main(): notification_handler=notification_handler, ) - logger.custom_info('Initializing OPC...', metadata) + logger.custom_info('Initializing OPC...', metadata_runtime) await activities.init_opc() - logger.custom_info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata) + logger.custom_info( + f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', + metadata_runtime, + ) new_runtime = Runtime( telemetry=TelemetryConfig( @@ -136,7 +153,7 @@ async def main(): ) ) - logger.custom_info(f'Starting Temporal Client at {host}...', metadata) + logger.custom_info(f'Starting Temporal Client at {host}...', metadata_runtime) temporal_client = await client.Client.connect( target_host=host, @@ -144,7 +161,7 @@ async def main(): runtime=new_runtime, ) - logger.custom_info('Starting Workers...', metadata) + logger.custom_info(f'Starting Workers (runtime={runtime})...', metadata_runtime) workers = [ prepare_worker( @@ -159,6 +176,7 @@ async def main(): activities.export_data_to_postgres, ], logger=logger, + runtime=runtime, ), prepare_worker( temporal_client=temporal_client, @@ -170,6 +188,7 @@ async def main(): activities.export_data_to_postgres, ], logger=logger, + runtime=runtime, ), prepare_worker( temporal_client=temporal_client, @@ -182,6 +201,7 @@ async def main(): activities.export_data_to_postgres, ], logger=logger, + runtime=runtime, ), prepare_worker( temporal_client=temporal_client, @@ -211,6 +231,7 @@ async def main(): activities.write_pi_web_api_data, ], logger=logger, + runtime=runtime, ), ] @@ -218,7 +239,7 @@ async def main(): for w in workers: handlers.append(w.run()) - logger.custom_info('Workers started successfully', metadata) + logger.custom_info('Workers started successfully', metadata_runtime) exit_code = 0 try: diff --git a/tests/laborious/worker/test_runtime_task_queues.py b/tests/laborious/worker/test_runtime_task_queues.py new file mode 100644 index 0000000..8b40d4a --- /dev/null +++ b/tests/laborious/worker/test_runtime_task_queues.py @@ -0,0 +1,14 @@ +from sientia_do.temporal.worker.prepare_worker import build_queue_name + +from laborious.workflows.drift import Drift +from laborious.workflows.minimal_retrain import MinimalRetrain +from laborious.workflows.predictions_batch import PredictionsBatch +from laborious.workflows.simple_metrics import SimpleMetrics + + +def test_runtime_scoped_queue_names(): + runtime = 'prod-a' + assert build_queue_name(PredictionsBatch.__name__, runtime) == 'predictions_batch-prod-a-queue' + assert build_queue_name(MinimalRetrain.__name__, runtime) == 'minimal_retrain-prod-a-queue' + assert build_queue_name(Drift.__name__, runtime) == 'drift-prod-a-queue' + assert build_queue_name(SimpleMetrics.__name__, runtime) == 'simple_metrics-prod-a-queue' diff --git a/values.yaml b/values.yaml index b07d60b..25d3418 100644 --- a/values.yaml +++ b/values.yaml @@ -209,6 +209,9 @@ env: value: "temporal-frontend.temporal.svc.cluster.local:7233" - name: TEMPORAL_NAMESPACE value: "laborious" + # Suffix for all Temporal task queues: {workflow}-{RUNTIME}-queue + - name: RUNTIME + value: "legacy" - name: MONGODB_USERNAME value: "root" From 90f8bdda6152c95574bed0c722a2230dd200facb Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 20 May 2026 09:26:35 -0300 Subject: [PATCH 10/64] SIENTIAPDE-1646 Update requirements.txt to align with recent dependency changes and ensure compatibility across the project. --- requirements-light.txt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 requirements-light.txt diff --git a/requirements-light.txt b/requirements-light.txt new file mode 100644 index 0000000..3624685 --- /dev/null +++ b/requirements-light.txt @@ -0,0 +1,17 @@ +temporalio +psycopg2-binary +sqlalchemy +asyncua==1.0.6 +redis +sientia_do>=1.12.1 +prometheus-client +botocore +boto3 +s3fs +pyarrow +kaleido +hyperopt +shap +pycurl +scipy<1.14.0 +scikit-learn==1.5.2 From ddb16182095d6b00340ccf774990944af9d583cf Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 20 May 2026 09:30:05 -0300 Subject: [PATCH 11/64] SIENTIAPDE-1646 Update quality-gate workflow to use python-quality-gate template --- .github/workflows/quality-gate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index bcb39de..79249d6 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -8,7 +8,7 @@ on: jobs: quality-gate: - uses: Aignosi/github_workflow_templates/.github/workflows/dataops-module-quality-gate.yml@main + uses: Aignosi/github_workflow_templates/.github/workflows/python-quality-gate.yml@main permissions: write-all with: project_name: 'laborious' From 4ea0754f0cf83f04025bdd5e5fc44a1d1532dd7e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 20 May 2026 09:53:53 -0300 Subject: [PATCH 12/64] SIENTIAPDE-1646 Enhance MLFlow run ID resolution with error handling for missing and invalid source URIs - Added checks in `get_model_run_id` method to raise exceptions for models with missing or invalid source URIs. - Introduced new test cases to validate error handling for these scenarios. - Updated `requirements-light.txt` to include `mlflow` as a dependency. --- .../utils/repository/model_repository.py | 15 ++++++-- requirements-light.txt | 1 + .../utils/repository/test_model_repository.py | 35 +++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index b4fdad5..6a25b8c 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -170,8 +170,19 @@ class MLFlowRepository(SientiaMonitoring): # Sort by version number to get the latest latest_version = max(stage_versions, key=lambda v: int(v.version)) - run_id = latest_version.source.split('/') - return run_id[2] + source = latest_version.source + if source is None: + raise mlflow.exceptions.MlflowException( + f"Model '{model_name}' version '{latest_version.version}' in stage '{stage}' " + 'has no source URI to resolve run ID.' + ) + parts = source.split('/') + if len(parts) <= 2 or not parts[2]: + raise mlflow.exceptions.MlflowException( + f"Model '{model_name}' version '{latest_version.version}' in stage '{stage}' " + f"has invalid source URI '{source}' for run ID resolution." + ) + return parts[2] def get_next_run_name(self, model_name: str) -> str: """ diff --git a/requirements-light.txt b/requirements-light.txt index 3624685..302fb79 100644 --- a/requirements-light.txt +++ b/requirements-light.txt @@ -4,6 +4,7 @@ sqlalchemy asyncua==1.0.6 redis sientia_do>=1.12.1 +mlflow prometheus-client botocore boto3 diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 15f8c6b..54a75bb 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -142,6 +142,41 @@ def test_get_model_run_id_success(mlflow_repository): assert output == '1' +def test_get_model_run_id_missing_source(mlflow_repository): + mlflow_repository.client.search_registered_models.return_value = [MagicMock(name='test')] + + mlflow_repository.client.search_model_versions.return_value = [ + MagicMock(current_stage='Production', version='1', source='runs/test/0'), + MagicMock(current_stage='Production', version='2', source=None), + ] + + with pytest.raises(mlflow_lib.exceptions.MlflowException) as exc_info: + mlflow_repository.get_model_run_id('test') + + assert ( + str(exc_info.value) + == "Model 'test' version '2' in stage 'Production' has no source URI to resolve run ID." + ) + + +def test_get_model_run_id_invalid_source(mlflow_repository): + mlflow_repository.client.search_registered_models.return_value = [MagicMock(name='test')] + + mlflow_repository.client.search_model_versions.return_value = [ + MagicMock(current_stage='Production', version='1', source='runs/test/0'), + MagicMock(current_stage='Production', version='2', source='runs/test'), + ] + + with pytest.raises(mlflow_lib.exceptions.MlflowException) as exc_info: + mlflow_repository.get_model_run_id('test') + + assert ( + str(exc_info.value) + == "Model 'test' version '2' in stage 'Production' has invalid source URI " + "'runs/test' for run ID resolution." + ) + + def test_get_next_run_name(mlflow, mlflow_repository): mlflow.search_runs.return_value = [1, 2, 3] output = mlflow_repository.get_next_run_name('run') From c07457bfbed76850454224569a3aa4b2270f06b8 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Wed, 1 Jul 2026 20:59:29 -0300 Subject: [PATCH 13/64] chore(sonar): update project key --- sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index a2e5f65..37d1521 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,4 +1,4 @@ -sonar.projectKey=Aignosi_sientia-dataops-laborious_temporal_beaec423-6c42-4f26-8134-b676287b499d +sonar.projectKey=Aignosi_sientia-dataops-laborious_temporal_ca1a7039-6db9-49e5-be78-54d29bc93e4f sonar.projectName=sientia-dataops-laborious_temporal sonar.sources=laborious sonar.tests=tests From 2142143ab97c140f00247424a07a56a1128d533d Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Wed, 8 Jul 2026 22:12:50 -0300 Subject: [PATCH 14/64] SIENTIAPDE-1945: Delete values.yaml configuration file for sientia-module Helm chart. --- values.yaml | 318 ---------------------------------------------------- 1 file changed, 318 deletions(-) delete mode 100644 values.yaml diff --git a/values.yaml b/values.yaml deleted file mode 100644 index 25d3418..0000000 --- a/values.yaml +++ /dev/null @@ -1,318 +0,0 @@ -# Default values for sientia-module. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 1 - -# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ -image: - repository: aignosi.azurecr.io/sientia-module - # This sets the pull policy for images. - pullPolicy: Always - # Overrides the image tag whose default is the chart appVersion. - tag: "1.1.2" - -# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ -imagePullSecrets: -- name: docker-hub-secret -# This is to override the chart name. -nameOverride: "sientia-laborious-legacy-worker" -fullnameOverride: "sientia-laborious-legacy-worker" -namespace: sientia - -# This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ -serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "sientia-laborious-legacy-worker" - -# This is for setting Kubernetes Annotations to a Pod. -# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ -podAnnotations: {} -# This is for setting Kubernetes Labels to a Pod. -# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ -podLabels: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - -resources: - # Resource limits and requests are important for ResourceBasedTuner to work correctly. - # The tuner monitors system CPU and memory usage, so proper resource limits must be set. - limits: - cpu: 2000m # 2 CPU cores - memory: 20Gi # 20 GB memory - requests: - cpu: 1000m # 1 CPU core - memory: 2Gi # 2 GB memory - -# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ -# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ -livenessProbe: - exec: - command: - - sh - - -c - - | - curl -sf http://localhost:9090/metrics | grep -q '^app_up{.*} 1' - initialDelaySeconds: 1260 - periodSeconds: 15 - timeoutSeconds: 5 - failureThreshold: 3 - -readinessProbe: - exec: - command: - - sh - - -c - - | - curl -sf http://localhost:9090/metrics | grep -q '^app_up{.*} 1' - initialDelaySeconds: 1200 - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 2 - - - -# This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -# Additional volumes on the output Deployment definition. -volumes: [] -# - name: foo -# secret: -# secretName: mysecret -# optional: false - -# Additional volumeMounts on the output Deployment definition. -volumeMounts: [] -# - name: foo -# mountPath: "/etc/foo" -# readOnly: true - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -services: - sdk-metrics: - enabled: true - type: ClusterIP - port: 9091 - targetPort: 9091 - name: sdk-metrics - - metrics: - enabled: true - type: ClusterIP - port: 9090 - targetPort: 9090 - name: metrics - -# Configuração do ServiceMonitor para o Prometheus Operator -# ref: https://github.com/prometheus-operator/prometheus-operator -serviceMonitor: - # Se true, um recurso ServiceMonitor será criado. - enabled: true - # O intervalo no qual as métricas devem ser coletadas (ex: 30s, 1m). - endpoints: - - port: metrics - path: /metrics - interval: 30s - relabelings: [] - - port: sdk-metrics - path: /metrics - interval: 30s - relabelings: [] - - additionalLabels: - release: kube-prometheus-stack - - -env: - # Entrypoint variables - - name: GITHUB_REPO_URL - value: "git@github.com:Aignosi/sientia-dataops-laborious_temporal.git" - - name: GITHUB_BRANCH - value: "fix/SIENTIAPDE-1811" - - name: PYTHON_APP - value: "laborious.worker.worker" - - # Application variables - - name: POSTGRES_HOST - value: "paradedb-rw.paradedb.svc.cluster.local" - - name: POSTGRES_PORT - value: "5432" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "nFqc81y6kwmr2zuAIx43DhiOosFCVPpeEfTtTWZflkNjB2j1KtEeIANkhFR9mAX3" - - name: POSTGRES_DBNAME - value: "sientia" - - name: POSTGRES_MIN_CONNECTIONS - value: "20" - # max_connections = number_of_workers * max_concurrent_activities * safety_factor - # Example: 4 workers * 50 activities * 0.5 = 100 connections - - name: POSTGRES_MAX_CONNECTIONS - value: "100" - - - name: MLFLOW_HOST - value: "http://sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local" - - name: MLFLOW_PORT - value: "80" - - name: MLFLOW_USERNAME - value: "aignosi" - - name: MLFLOW_PASSWORD - value: "1L0FP50j3ncp123" - - - name: OPC_ID - value: "1" - - name: OPC_SERVER_NAME - value: "default_server" - - name: OPC_URL - value: "opc.tcp://sientia-opc-simulator-opc.sientia.svc.cluster.local:4840" - - - - name: LOG_LEVEL - value: "DEBUG" - - name: HTTP_METRICS_PORT - value: "9090" - - name: HTTP_SDK_METRICS_PORT - value: "9091" - - name: PROJECT_NAME - value: "sientia-laborious" - - - name: TEMPORAL_HOST - value: "temporal-frontend.temporal.svc.cluster.local:7233" - - name: TEMPORAL_NAMESPACE - value: "laborious" - # Suffix for all Temporal task queues: {workflow}-{RUNTIME}-queue - - name: RUNTIME - value: "legacy" - - - name: MONGODB_USERNAME - value: "root" - - name: MONGODB_PASSWORD - value: "wKZDbMNU1c" - - name: MONGODB_URL - value: "my-release-mongodb.mongodb.svc.cluster.local:27017" - - name: MONGODB_DATABASE - value: "sientia" - - name: MONGODB_TTL_INDEX_HOURS - value: "1" - - - name: MINIO_ENDPOINT_URL - value: "minio.minio.svc.cluster.local:9000" - - name: MINIO_ACCESS_KEY - value: "admin" - - name: MINIO_SECRET_KEY - value: "LiArt4eNmJ" - - name: MINIO_DEFAULT_BUCKET - value: "sientia" - - name: MINIO_RETENTION_HOURS - value: "24" - - name: SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES - value: "0.5" - - # Temporal worker tuning for PredictionsBatch. - # IMPORTANT: prefix must be PREDICTIONSBATCH_ (from class name PredictionsBatch). - # Keep workflow-task concurrency moderate to reduce task completion races under load. - - name: PREDICTIONSBATCH_MAX_CONCURRENT_WORKFLOW_TASKS - value: "20" - # Allow higher activity parallelism because most activities are I/O-bound, but keep headroom. - - name: PREDICTIONSBATCH_MAX_CONCURRENT_ACTIVITIES - value: "60" - # Keep local activities controlled so they do not monopolize the event loop. - - name: PREDICTIONSBATCH_MAX_CONCURRENT_LOCAL_ACTIVITIES - value: "20" - # Cache enough workflows for reuse without excessive memory growth. - - name: PREDICTIONSBATCH_MAX_CACHED_WORKFLOWS - value: "200" - # Start with one workflow poller to avoid burst contention at startup. - - name: PREDICTIONSBATCH_WORKFLOW_POLLER_BEHAVIOUR_MINIMUM - value: "3" - # Small initial poller count warms up gradually instead of spiking task fetches. - - name: PREDICTIONSBATCH_WORKFLOW_POLLER_BEHAVIOUR_INITIAL - value: "5" - # Cap workflow pollers to limit scheduling pressure and avoid over-polling. - - name: PREDICTIONSBATCH_WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM - value: "15" - # Keep at least two activity pollers so activity queues do not starve during spikes. - - name: PREDICTIONSBATCH_ACTIVITY_POLLER_BEHAVIOUR_MINIMUM - value: "3" - # Moderate initial activity pollers for faster ramp-up with controlled pressure. - - name: PREDICTIONSBATCH_ACTIVITY_POLLER_BEHAVIOUR_INITIAL - value: "10" - # Limit max activity pollers to preserve CPU for workflow-task completion. - - name: PREDICTIONSBATCH_ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM - value: "30" - - - name: MINIMALRETRAIN_MAX_CONCURRENT_ACTIVITIES - value: "1" - - name: MINIMALRETRAIN_MAX_CONCURRENT_LOCAL_ACTIVITIES - value: "1" - - name: MINIMALRETRAIN_MAX_CACHED_WORKFLOWS - value: "1" - - name: MINIMALRETRAIN_WORKFLOW_POLLER_BEHAVIOUR_MINIMUM - value: "1" - - name: MINIMALRETRAIN_WORKFLOW_POLLER_BEHAVIOUR_INITIAL - value: "1" - - name: MINIMALRETRAIN_WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM - value: "1" - - name: MINIMALRETRAIN_ACTIVITY_POLLER_BEHAVIOUR_MINIMUM - value: "1" - - name: MINIMALRETRAIN_ACTIVITY_POLLER_BEHAVIOUR_INITIAL - value: "1" - - name: MINIMALRETRAIN_ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM - value: "1" - - - name: PI_WEB_API_BASE_URL - value: "https://pivision.votorantimcimentos.com/piwebapi" - - name: PI_WEB_API_AUTH_TYPE - value: "basic" - - name: PI_WEB_API_AUTH_TOKEN - valueFrom: - secretKeyRef: - name: pi-web-api-auth-token - key: token - - - name: PYPI_SERVER - value: "http://library-distribution-server.library.svc.cluster.local:5000" - -ssh: - enabled: true - secretName: git-ssh-key-sientia-laborious-worker - sshPath: /mnt/.ssh - knownHostsPath: /mnt/known_hosts - -# kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp - -# helm upgrade --install sientia-laborious-legacy-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.6.0 - -# kubectl create secret generic git-ssh-key-sientia-laborious-worker \ -# --namespace sientia \ -# --from-file=ssh-privatekey=git_key \ -# --type=kubernetes.io/ssh-auth From 310ceea0d814ddfc3584d52f9a6b662c3f925976 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Tue, 4 Aug 2026 11:41:23 -0300 Subject: [PATCH 15/64] ci(quality-gate): Configure push triggers, concurrency, and granular permissions --- .github/workflows/quality-gate.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 79249d6..f50c9c5 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -1,15 +1,29 @@ name: Quality gate on: + push: + branches: + - main + - 'release/**' + - 'feature/**' pull_request: branches: - main + - 'release/**' + - 'feature/**' types: [ opened, synchronize, reopened ] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: quality-gate: uses: Aignosi/github_workflow_templates/.github/workflows/python-quality-gate.yml@main - permissions: write-all + permissions: + contents: read + pull-requests: write + issues: write with: project_name: 'laborious' repositories: 'sientia-dataops-library, sientia-mlops-library' From acd925cf2a5812f7d2f5662e9a597f37003ac91a Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Tue, 4 Aug 2026 12:01:06 -0300 Subject: [PATCH 16/64] refactor(opc): Rename async close method to aclose Renamed the OPC.close asynchronous method to OPC.aclose to align with common Python conventions for asynchronous context managers and methods, improving clarity. All call sites and tests have been updated accordingly. --- laborious/activities/activities.py | 2 +- laborious/activities/opc.py | 2 +- tests/laborious/activities/test_opc.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/laborious/activities/activities.py b/laborious/activities/activities.py index 9a03f54..df78a08 100644 --- a/laborious/activities/activities.py +++ b/laborious/activities/activities.py @@ -164,6 +164,6 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API): Storage.close(self) MLFlow.close(self) Gates.close(self) - await OPC.close(self) + await OPC.aclose(self) ModelMetrics.close(self) API.close(self) diff --git a/laborious/activities/opc.py b/laborious/activities/opc.py index be5a941..04ed18e 100644 --- a/laborious/activities/opc.py +++ b/laborious/activities/opc.py @@ -491,7 +491,7 @@ class OPC(SientiaMonitoring): return data.to_dict() - async def close(self): + async def aclose(self): """ Gracefully shutdown all OPC server connections and cleanup resources. diff --git a/tests/laborious/activities/test_opc.py b/tests/laborious/activities/test_opc.py index c2d37dd..6a36f8a 100644 --- a/tests/laborious/activities/test_opc.py +++ b/tests/laborious/activities/test_opc.py @@ -734,5 +734,5 @@ async def test_validate_server(opc): @mark.asyncio async def test_close(opc): opc.opc_repository['server1'].disconnect = AsyncMock(return_value=True) - await opc.close() + await opc.aclose() opc.opc_repository['server1'].disconnect.assert_called_once() From 35efb67c89381d93d107a577f8f0b6bff3b397b9 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Tue, 4 Aug 2026 14:43:15 -0300 Subject: [PATCH 17/64] fix(test_activities): Use aclose for async mock shutdown --- tests/laborious/activities/test_activities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/laborious/activities/test_activities.py b/tests/laborious/activities/test_activities.py index 8e6cd78..54fdd89 100644 --- a/tests/laborious/activities/test_activities.py +++ b/tests/laborious/activities/test_activities.py @@ -173,7 +173,7 @@ async def test_shutdown( mock_mlflow_init, mock_storage_init, ): - mock_opc_init.close = AsyncMock() + mock_opc_init.aclose = AsyncMock() postgres_config = { 'host': 'localhost', 'port': 5432, @@ -221,7 +221,7 @@ async def test_shutdown( ) await activities.shutdown() - mock_opc_init.close.assert_called_once() + mock_opc_init.aclose.assert_called_once() mock_storage_init.close.assert_called_once() mock_mlflow_init.close.assert_called_once() mock_gates_init.close.assert_called_once() From 773c980fc3f168b5b08a1b2ec2597094894ce3ce Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Tue, 4 Aug 2026 14:57:52 -0300 Subject: [PATCH 18/64] fix(storage): Prevent AttributeError when closing uninitialized Postgres engine --- laborious/activities/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/laborious/activities/storage.py b/laborious/activities/storage.py index d64cc52..7c67466 100644 --- a/laborious/activities/storage.py +++ b/laborious/activities/storage.py @@ -202,7 +202,8 @@ class Storage(Postgres, MinioManager): def close(self) -> None: """Close Storage resources (MinIO client and Postgres engine).""" - Postgres.close(self) + if hasattr(self, 'engine'): + Postgres.close(self) MinioManager.close(self) def __del__(self): From 6a8c41b328d9488f35d54157f458f5ac24ce6983 Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Tue, 4 Aug 2026 15:21:14 -0300 Subject: [PATCH 19/64] fix(pytest): Workaround unraisableexception plugin crash Disables the unraisableexception plugin in pytest due to a known bug in pytest>=9.1 where it crashes with tracemalloc errors when multiple unraisable exceptions occur close together. --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a878882..db08645 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,12 @@ python_functions = ["test_*"] addopts = [ "-v", "--strict-markers", + # pytest>=9.1 has a known bug where its unraisableexception plugin crashes + # (tracemalloc partially-initialized AttributeError) when 2+ unraisable + # exceptions land close together — e.g. "coroutine was never awaited" from + # AsyncMock-mocked sync methods (metrics_controller, minio_repository) being + # GC'd. Harmless mock artifacts turned into a hard ERROR by the plugin itself. + "-p", "no:unraisableexception", ] markers = [ "asyncio: marks tests as async", From c12f2996c2aaa8e998b3be2b0d425b5c77696da0 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 11 Aug 2026 11:36:22 -0300 Subject: [PATCH 20/64] QTZPOC-13 QTZPOC-13: Introduce joblib fallback for MLflow model loading Enhances model loading resilience by attempting to deserialize models directly with joblib if MLflow's native flavor loaders fail. This mechanism handles cases where .pkl files are present in the artifact. Adds joblib as a core dependency and includes new end-to-end and unit tests to validate fallback behavior, covering success, no-fallback, and double-failure scenarios. Also updates .gitignore for .claude/ files and incorporates minor refactorings and linting adjustments across E2E tests and utility functions. --- .gitignore | 3 +- e2e/conftest.py | 242 +++++++---- e2e/helpers.py | 19 +- e2e/opc_test_server.py | 7 +- e2e/test_child_workflows_e2e.py | 4 +- e2e/test_minio_offload.py | 24 +- e2e/test_opc_real_server.py | 29 +- e2e/test_predictions_batch_format_export.py | 98 +++-- e2e/test_predictions_batch_joblib_fallback.py | 186 +++++++++ e2e/test_predictions_batch_main_workflow.py | 3 +- ...st_predictions_batch_prediction_process.py | 14 +- .../utils/repository/model_repository.py | 203 ++++++++- pyproject.toml | 5 + requirements-light.txt | 1 + requirements-local.txt | 4 +- requirements.txt | 1 + .../utils/repository/test_model_repository.py | 395 +++++++++++++++++- 17 files changed, 1063 insertions(+), 175 deletions(-) create mode 100644 e2e/test_predictions_batch_joblib_fallback.py diff --git a/.gitignore b/.gitignore index c849e0a..3bd4a6b 100644 --- a/.gitignore +++ b/.gitignore @@ -55,4 +55,5 @@ mlruns/ relatorio* openspec/ -.cursor/ \ No newline at end of file +.cursor/ +.claude/ \ No newline at end of file diff --git a/e2e/conftest.py b/e2e/conftest.py index cc49d9b..c9c4504 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -12,28 +12,35 @@ sys.modules.setdefault('sientia', MagicMock()) sys.modules.setdefault('sientia.ModelAnalysis', _model_analysis_module) from io import BytesIO +import joblib import pandas as pd import pytest import pytest_asyncio +from sientia_do.notifications.handlers import CoreNotificationHandler +from sientia_do.observability.metrics_controller import MetricsController +from sklearn.compose import ColumnTransformer +from sklearn.linear_model import LinearRegression +from sklearn.pipeline import Pipeline from sqlalchemy import create_engine, text -from testcontainers.minio import MinioContainer -from testcontainers.postgres import PostgresContainer from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker +from testcontainers.minio import MinioContainer +from testcontainers.postgres import PostgresContainer from e2e.opc_test_server import OpcE2ETestServer from laborious.activities.activities import Activities from laborious.workflows.predictions_batch import PredictionsBatch -from laborious.workflows.sub_workflows.prediction_process import PredictionProcess from laborious.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction -from sientia_do.notifications.handlers import CoreNotificationHandler -from sientia_do.observability.logger import Logger -from sientia_do.observability.metrics_controller import MetricsController +from laborious.workflows.sub_workflows.prediction_process import PredictionProcess # Test constants TEST_MONGODB_CONNECTION_STRING = 'mongodb://localhost:27017' TEST_DATABASE_NAME = 'test_db' +# Dedicated model whose MLflow flavor loaders fail, so the joblib fallback is exercised. +JOBLIB_FALLBACK_MODEL_NAME = 'joblib_fallback_model' +JOBLIB_FALLBACK_FLAVOR_ERROR = 'flavor loader cannot deserialize this artifact' + @pytest_asyncio.fixture(scope='session') def minio_container(): @@ -50,7 +57,7 @@ def minio_container(): def postgres_container(): """ Create a PostgreSQL container using testcontainers. - + This fixture creates a real PostgreSQL database in a Docker container that will be used for all tests in the session. """ @@ -64,21 +71,21 @@ def postgres_container(): def postgres_engine(postgres_container): """ Create SQLAlchemy engine for PostgreSQL test database. - + This fixture creates a connection to the PostgreSQL container created by the postgres_container fixture. """ engine = create_engine(postgres_container.get_connection_url()) - + yield engine - + engine.dispose() def _create_schema_and_tables(engine): """ Helper function to create schema and tables in the given engine. - + Creates predictions_schema with: - laborious_data: Input data table for queries - predictions: Output predictions table @@ -87,8 +94,8 @@ def _create_schema_and_tables(engine): # Use begin() to ensure transaction is properly committed with engine.begin() as conn: # Create predictions_schema - conn.execute(text("CREATE SCHEMA IF NOT EXISTS predictions_schema")) - + conn.execute(text('CREATE SCHEMA IF NOT EXISTS predictions_schema')) + # Create laborious_data table (input data from sensors) create_laborious_data_sql = """ CREATE TABLE IF NOT EXISTS predictions_schema.laborious_data ( @@ -102,7 +109,7 @@ def _create_schema_and_tables(engine): ); """ conn.execute(text(create_laborious_data_sql)) - + # Create predictions table create_predictions_sql = """ CREATE TABLE if not exists predictions_schema.predictions ( @@ -119,7 +126,7 @@ def _create_schema_and_tables(engine): ); """ conn.execute(text(create_predictions_sql)) - + # Create transformed_data table create_transformed_sql = """ CREATE TABLE IF NOT EXISTS predictions_schema.transformed_data ( @@ -139,7 +146,7 @@ def _create_schema_and_tables(engine): def setup_postgres_schema_and_tables(postgres_engine): """ Automatically create necessary schema and tables before each test. - + This fixture runs automatically (autouse=True) and ensures that the predictions_schema and tables exist with the correct structure. """ @@ -150,35 +157,22 @@ def setup_postgres_schema_and_tables(postgres_engine): @pytest_asyncio.fixture def mock_logger(): """Mock logger for testing.""" + def message(message): - print(f"[LOG] {message}") - def custom_message(message, _metadata={}): - print(f"[LOG] {message}") + print(f'[LOG] {message}') + + def custom_message(message, _metadata=None): + print(f'[LOG] {message}') + logger = MagicMock() - logger.info = MagicMock( - side_effect=message - ) - logger.debug = MagicMock( - side_effect=message - ) - logger.error = MagicMock( - side_effect=message - ) - logger.warning = MagicMock( - side_effect=message - ) - logger.custom_info = MagicMock( - side_effect=custom_message - ) - logger.custom_debug = MagicMock( - side_effect=custom_message - ) - logger.custom_error = MagicMock( - side_effect=custom_message - ) - logger.custom_warning = MagicMock( - side_effect=custom_message - ) + logger.info = MagicMock(side_effect=message) + logger.debug = MagicMock(side_effect=message) + logger.error = MagicMock(side_effect=message) + logger.warning = MagicMock(side_effect=message) + logger.custom_info = MagicMock(side_effect=custom_message) + logger.custom_debug = MagicMock(side_effect=custom_message) + logger.custom_error = MagicMock(side_effect=custom_message) + logger.custom_warning = MagicMock(side_effect=custom_message) return logger @@ -186,24 +180,24 @@ def mock_logger(): def mock_mongo_client(): """ Mock MongoDB client to avoid real connections. - + This fixture mocks the pymongo.MongoClient used by CoreNotificationHandler, allowing us to use a real NotificationHandler instance without connecting to MongoDB. """ mock_client = MagicMock() mock_db = MagicMock() mock_collection = MagicMock() - + # Configure the mock chain: client[database] -> db[collection] -> collection mock_client.__getitem__.return_value = mock_db mock_db.__getitem__.return_value = mock_collection - + # Mock server_info() to avoid connection attempts mock_client.server_info = MagicMock() - + # Mock insert_one for notifications mock_collection.insert_one = MagicMock() - + return mock_client @@ -225,7 +219,7 @@ def notification_inserts(mock_mongo_client): def notification_handler(mock_logger, mock_mongo_client): """ Create a real NotificationHandler instance with mocked MongoDB client. - + This fixture creates a real CoreNotificationHandler instance but mocks the underlying MongoDB connection to avoid real database connections. """ @@ -258,7 +252,7 @@ def mock_minio_repository(): parquet_buffer = BytesIO() parquet_df.to_parquet(parquet_buffer, engine='pyarrow', index=True) parquet_bytes = parquet_buffer.getvalue() - + # sientia_do MinioRepository API mock_repo.bucket = 'test-bucket' mock_repo.upload_file = AsyncMock( @@ -273,7 +267,7 @@ def mock_minio_repository(): mock_repo.list_objects = AsyncMock(return_value=[]) mock_repo.delete_file = AsyncMock() mock_repo.close = MagicMock() - + return mock_repo @@ -295,13 +289,12 @@ def mock_pi_web_api_repository(): mock_repo.close = MagicMock() return mock_repo + @pytest_asyncio.fixture def mock_opc_repository(): """Mock OPC repository for OPC operations.""" mock_repo = MagicMock() - mock_repo.write_data = AsyncMock( - return_value=(True, {'response_time': 0.1}) - ) + mock_repo.write_data = AsyncMock(return_value=(True, {'response_time': 0.1})) mock_repo.disconnect = AsyncMock() return mock_repo @@ -318,10 +311,13 @@ async def opc_e2e_server(): finally: await server.stop() + @pytest_asyncio.fixture def patch_create_engine(postgres_engine): """Patch create_engine to return test postgres_engine.""" - with patch('sientia_do.temporal.activities.postgres.create_engine', return_value=postgres_engine): + with patch( + 'sientia_do.temporal.activities.postgres.create_engine', return_value=postgres_engine + ): yield @@ -329,44 +325,54 @@ def patch_create_engine(postgres_engine): def patch_minio_repository(mock_minio_repository): """Patch MinioRepository to return mock.""" # Patch where Activities resolves the symbol (import binds the original class). - with patch('laborious.activities.activities.MinioRepository', return_value=mock_minio_repository): + with patch( + 'laborious.activities.activities.MinioRepository', return_value=mock_minio_repository + ): yield + @pytest_asyncio.fixture def patch_pi_web_api_repository(mock_pi_web_api_repository): """Patch MLflowRepository to return mock.""" with patch('laborious.activities.api.PIWebAPIClient', return_value=mock_pi_web_api_repository): yield + @pytest_asyncio.fixture def mock_mlflow_models(): """Create mock models for MLflow load_model methods.""" # Mock transform model - returns DataFrame with same index as input mock_transform_model = MagicMock() + def mock_transform_predict(data): num_rows = max(len(data), 1) if hasattr(data, '__len__') else 1 print(data.to_csv()) print(data.index) - result = pd.DataFrame({ - 'feature_1': [0.234] * num_rows, - 'feature_2': [0.783] * num_rows, - }) + result = pd.DataFrame( + { + 'feature_1': [0.234] * num_rows, + 'feature_2': [0.783] * num_rows, + } + ) result.index = data.index return result + mock_transform_model.predict = MagicMock(side_effect=mock_transform_predict) - + # Mock predict model - returns array/list of predictions mock_predict_model = MagicMock() + def mock_predict_predict(data): num_rows = max(len(data), 1) if hasattr(data, '__len__') else 1 return [0.5] * num_rows + mock_predict_model.predict = MagicMock(side_effect=mock_predict_predict) - + # Mock PyFuncModel for compressed models mock_pyfunc_model = MagicMock() mock_pyfunc_model._model_impl = MagicMock() mock_pyfunc_model._model_impl.python_model = mock_transform_model - + return { 'transform_model': mock_transform_model, 'predict_model': mock_predict_model, @@ -374,50 +380,130 @@ def mock_mlflow_models(): } +def _build_joblib_fallback_estimator(): + """ + Build a real sklearn estimator for the joblib fallback artifact. + + `request_predict` appends a string `timestamp` column to the transformed + data, so the estimator selects the transform output columns explicitly and + drops the rest. + """ + features = ['feature_1', 'feature_2'] + estimator = Pipeline( + steps=[ + ( + 'select', + ColumnTransformer([('features', 'passthrough', features)], remainder='drop'), + ), + ('regressor', LinearRegression()), + ] + ) + train = pd.DataFrame( + { + 'feature_1': [0.0, 1.0, 2.0, 3.0], + 'feature_2': [0.0, 1.0, 0.0, 1.0], + 'timestamp': ['2024-01-01 12:00:00'] * 4, + } + ) + estimator.fit(train, train['feature_1'] * 2 + train['feature_2'] * 3) + return estimator + + +@pytest.fixture +def joblib_fallback_artifact(tmp_path): + """ + Real artifact directory holding a joblib-compressed sklearn estimator. + + Mirrors what the joblib fallback finds after downloading `prediction_model`: + a top-level `model.pkl` written by `joblib.dump(..., compress=3)` plus an + `MLmodel` placeholder. + + Return: + dict with 'path' (the artifact directory) and 'estimator' (the object + that was serialized, so tests can compute the expected prediction). + """ + artifact_dir = tmp_path / 'prediction_model' + artifact_dir.mkdir() + + estimator = _build_joblib_fallback_estimator() + joblib.dump(estimator, artifact_dir / 'model.pkl', compress=3) + (artifact_dir / 'MLmodel').write_text('flavors:\n sklearn: {}\n') + + return {'path': str(artifact_dir), 'estimator': estimator} + + +@pytest.fixture +def joblib_fallback_artifact_without_pickle(tmp_path): + """Artifact directory with no top-level `.pkl`, so the joblib fallback also fails.""" + artifact_dir = tmp_path / 'prediction_model_without_pickle' + artifact_dir.mkdir() + (artifact_dir / 'MLmodel').write_text('flavors:\n sklearn: {}\n') + return str(artifact_dir) + + @pytest_asyncio.fixture -def patch_mlflow(mock_mlflow_models): +def patch_mlflow(mock_mlflow_models, joblib_fallback_artifact): """Patch mlflow module in repository with load_model mocks.""" mock_mlflow = MagicMock() - + # Mock sklearn.load_model def mock_sklearn_load_model(model_uri): + if JOBLIB_FALLBACK_MODEL_NAME in model_uri: + raise RuntimeError(JOBLIB_FALLBACK_FLAVOR_ERROR) if 'data_model' in model_uri or 'transform' in model_uri.lower(): return mock_mlflow_models['transform_model'] return mock_mlflow_models['predict_model'] + mock_mlflow.sklearn = MagicMock() mock_mlflow.sklearn.load_model = MagicMock(side_effect=mock_sklearn_load_model) - + # Mock pyfunc.load_model def mock_pyfunc_load_model(model_uri): + if JOBLIB_FALLBACK_MODEL_NAME in model_uri: + raise RuntimeError(JOBLIB_FALLBACK_FLAVOR_ERROR) if 'artifacts' in model_uri or 'tmp' in model_uri: return mock_mlflow_models['pyfunc_model'] if 'data_model' in model_uri or 'transform' in model_uri.lower(): return mock_mlflow_models['transform_model'] return mock_mlflow_models['predict_model'] + mock_mlflow.pyfunc = MagicMock() mock_mlflow.pyfunc.load_model = MagicMock(side_effect=mock_pyfunc_load_model) - + # Mock pytorch.load_model mock_mlflow.pytorch = MagicMock() mock_mlflow.pytorch.load_model = MagicMock(return_value=mock_mlflow_models['predict_model']) - + # Mock other mlflow methods that might be called mock_mlflow.set_tracking_uri = MagicMock() - mock_mlflow.get_run = MagicMock(return_value=MagicMock(info=MagicMock(artifact_uri='mlflow-artifacts:/test_run_id'))) + mock_mlflow.get_run = MagicMock( + return_value=MagicMock(info=MagicMock(artifact_uri='mlflow-artifacts:/test_run_id')) + ) mock_mlflow.tracking = MagicMock() - mock_mlflow.tracking.MlflowClient = MagicMock(return_value=MagicMock( + mock_mlflow_client = MagicMock( search_registered_models=MagicMock(return_value=[MagicMock(name='test_model')]), - search_model_versions=MagicMock(return_value=[MagicMock( - current_stage='Production', - version='1', - source='runs:/artifacts/test_run_id' - )]) - )) - + search_model_versions=MagicMock( + return_value=[ + MagicMock( + current_stage='Production', version='1', source='runs:/artifacts/test_run_id' + ) + ] + ), + ) + # Only the joblib fallback calls download_artifacts directly on the client. + mock_mlflow_client.download_artifacts = MagicMock(return_value=joblib_fallback_artifact['path']) + mock_mlflow.tracking.MlflowClient = MagicMock(return_value=mock_mlflow_client) + with patch('laborious.utils.repository.model_repository.mlflow', new=mock_mlflow): yield mock_mlflow +@pytest.fixture +def mlflow_client(patch_mlflow): + """The MlflowClient mock the repository holds as `self.client`.""" + return patch_mlflow.tracking.MlflowClient.return_value + + @pytest_asyncio.fixture(scope='function') async def test_activities( postgres_engine, @@ -430,7 +516,7 @@ async def test_activities( patch_minio_repository, patch_mlflow, patch_pi_web_api_repository, - mock_opc_repository + mock_opc_repository, ): """ Create Activities instance with test dependencies. diff --git a/e2e/helpers.py b/e2e/helpers.py index bb404ca..2919ce9 100644 --- a/e2e/helpers.py +++ b/e2e/helpers.py @@ -11,7 +11,9 @@ from sqlalchemy import text from sqlalchemy.engine import Engine -async def start_and_await_workflow(client, workflow_run, input_data: dict, workflow_id: str, timeout: float = 60.0): +async def start_and_await_workflow( + client, workflow_run, input_data: dict, workflow_id: str, timeout: float = 60.0 +): """ Start a workflow and wait for its result. @@ -44,7 +46,9 @@ def insert_sample_data(postgres_engine: Engine, model_id: int, values: list[Any] values: Per-sensor values; use string 'NULL' for SQL NULL. """ with postgres_engine.begin() as conn: - conn.execute(text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}')) + conn.execute( + text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}') + ) values_sql = [] for i, value in enumerate(values): values_sql.append(f""" @@ -79,7 +83,6 @@ def assert_prediction( comments: Expected exact comments string (optional). comments_contains: Substring expected in comments when queued (optional). """ - import pytest with postgres_engine.connect() as conn: result_query = conn.execute( @@ -90,7 +93,9 @@ def assert_prediction( ) ) prediction_rows = result_query.fetchall() - assert len(prediction_rows) == 1, f'Expected one prediction record, got {len(prediction_rows)}' + assert len(prediction_rows) == 1, ( + f'Expected one prediction record, got {len(prediction_rows)}' + ) row = prediction_rows[0] assert row[0] == model_id, f'Expected model_id={model_id}, got {row[0]}' assert row[1] == prediction or Decimal(str(row[1])) == Decimal(str(prediction)), ( @@ -99,7 +104,9 @@ def assert_prediction( assert row[2] == prediction_confidence or Decimal(str(row[2])) == Decimal( str(prediction_confidence) ), f'Expected prediction_confidence={prediction_confidence}, got {row[2]}' - assert row[3] == prediction_status, f"Expected prediction_status='{prediction_status}', got {row[3]}" + assert row[3] == prediction_status, ( + f"Expected prediction_status='{prediction_status}', got {row[3]}" + ) if comments is not None: assert row[4] == comments, f"Expected comments='{comments}', got {row[4]}" if comments_contains is not None: @@ -135,7 +142,6 @@ def assert_continue( def assert_stop(postgres_engine: Engine, model_id: int) -> None: """Assert no prediction rows for model_id.""" - import pytest with postgres_engine.connect() as conn: result_query = conn.execute( @@ -156,7 +162,6 @@ def assert_repeat(postgres_engine: Engine, model_id: int, last_prediction: tuple model_id: Model id. last_prediction: Tuple (model_id, prediction, confidence, status) to match both rows. """ - import pytest with postgres_engine.connect() as conn: result_query = conn.execute( diff --git a/e2e/opc_test_server.py b/e2e/opc_test_server.py index 69b02c8..8f94eea 100644 --- a/e2e/opc_test_server.py +++ b/e2e/opc_test_server.py @@ -147,8 +147,11 @@ class OpcE2ETestServer: value = await self._confidence_node.read_value() return float(value) - async def _pre_write_callback(self, _event, _service) -> None: - if self._session_bad_on_write: + async def _pre_write_callback(self, event, _service) -> None: + # Only client writes are faulted. asyncua's own clock task writes CurrentTime + # through this same PreWrite dispatch, and raising there kills its time_task; + # the stored exception then resurfaces as a teardown failure in server.stop(). + if self._session_bad_on_write and getattr(event, 'is_external', False): raise ServiceError(ua.StatusCodes.BadSessionIdInvalid) diff --git a/e2e/test_child_workflows_e2e.py b/e2e/test_child_workflows_e2e.py index 52a4b3f..34d97fd 100644 --- a/e2e/test_child_workflows_e2e.py +++ b/e2e/test_child_workflows_e2e.py @@ -26,7 +26,9 @@ async def test_format_and_export_prediction_default_path_e2e( client = temporal_test_env.client model_id = 401 with postgres_engine.begin() as conn: - conn.execute(text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}')) + conn.execute( + text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}') + ) metadata = { 'metadata': { diff --git a/e2e/test_minio_offload.py b/e2e/test_minio_offload.py index f8c0122..dd303f1 100644 --- a/e2e/test_minio_offload.py +++ b/e2e/test_minio_offload.py @@ -29,7 +29,9 @@ async def test_load_query_with_minio_offload_writes_object_to_bucket( """ model_id = 501 with postgres_engine.begin() as conn: - conn.execute(text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}')) + conn.execute( + text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}') + ) insert_sample_data(postgres_engine, model_id, [1.0, 2.0]) metadata = { @@ -53,7 +55,9 @@ async def test_load_query_with_minio_offload_writes_object_to_bucket( } ) assert payload.object_key, 'offloaded payload must reference a MinIO object' - assert payload.data is None or payload.data == {}, 'large payloads should not inline tabular dict' + assert payload.data is None or payload.data == {}, ( + 'large payloads should not inline tabular dict' + ) df = await payload.retrieve(test_activities_real_minio.minio_repository, metadata['metadata']) assert len(df) >= 1 @@ -61,7 +65,9 @@ async def test_load_query_with_minio_offload_writes_object_to_bucket( client = minio_container.get_client() listed = list(client.list_objects('test-bucket', recursive=True)) names = [getattr(o, 'object_name', None) or getattr(o, '_object_name', '') for o in listed] - assert any(n and 'prediction_datasets' in n for n in names), f'unexpected object listing: {names!r}' + assert any(n and 'prediction_datasets' in n for n in names), ( + f'unexpected object listing: {names!r}' + ) @pytest.mark.asyncio @@ -77,9 +83,15 @@ async def test_predictions_batch_with_minio_offload_path( """ model_id = 502 with postgres_engine.begin() as conn: - conn.execute(text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}')) - conn.execute(text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}')) - conn.execute(text(f'DELETE FROM predictions_schema.transformed_data WHERE model_id = {model_id}')) + conn.execute( + text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}') + ) + conn.execute( + text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}') + ) + conn.execute( + text(f'DELETE FROM predictions_schema.transformed_data WHERE model_id = {model_id}') + ) insert_sample_data(postgres_engine, model_id, [10.0, 20.0, 30.0]) input_data = { diff --git a/e2e/test_opc_real_server.py b/e2e/test_opc_real_server.py index 200a8a4..7445656 100644 --- a/e2e/test_opc_real_server.py +++ b/e2e/test_opc_real_server.py @@ -11,7 +11,12 @@ import pytest from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker -from e2e.helpers import assert_prediction, insert_sample_data, make_workflow_id, start_and_await_workflow +from e2e.helpers import ( + assert_prediction, + insert_sample_data, + make_workflow_id, + start_and_await_workflow, +) from e2e.opc_test_server import UNKNOWN_NODE_ID, OpcE2ETestServer, build_opc_output_config from e2e.test_predictions_batch_format_export import get_base_input_data from laborious.activities.activities import Activities @@ -20,16 +25,24 @@ from laborious.utils.repository.opc_repository import OpcRepository from laborious.workflows.predictions_batch import PredictionsBatch -async def _slow_reconnect_under_lock(repo: OpcRepository, hold_seconds: float = 0.75) -> None: +async def _reconnect_under_held_lock( + repo: OpcRepository, acquired: asyncio.Event, release: asyncio.Event +) -> None: """ - Hold the connection lock briefly so concurrent writes see reconnect_in_progress. + Hold the connection lock until released, then reconnect. + + The lock must stay held for the whole workflow, and how long the workflow takes to + reach the OPC write is not bounded — a fixed sleep here lets `_reconnect_locked` + finish first, which sets `_session_ready` again and the write then succeeds. Args: repo (OpcRepository): Connected repository. - hold_seconds (float): Time to keep the lock before reconnecting. + acquired (asyncio.Event): Set once the lock is held, before any write can race. + release (asyncio.Event): Awaited while holding the lock; set by the caller. """ async with repo._connection_lock: - await asyncio.sleep(hold_seconds) + acquired.set() + await release.wait() await repo._reconnect_locked() @@ -170,7 +183,10 @@ async def test_scenario_3_2_5_opc_write_blocked_during_reconnect_real_server( repo = test_activities_real_opc.opc_repository['1'] repo._session_ready.clear() - reconnect_task = asyncio.create_task(_slow_reconnect_under_lock(repo)) + acquired = asyncio.Event() + release = asyncio.Event() + reconnect_task = asyncio.create_task(_reconnect_under_held_lock(repo, acquired, release)) + await acquired.wait() input_data = get_base_input_data(model_id) input_data['opc_output_config'] = build_opc_output_config(opc_e2e_server.node_ids) @@ -184,6 +200,7 @@ async def test_scenario_3_2_5_opc_write_blocked_during_reconnect_real_server( make_workflow_id('test-opc-real-reconnect-block'), ) finally: + release.set() await reconnect_task assert_prediction( diff --git a/e2e/test_predictions_batch_format_export.py b/e2e/test_predictions_batch_format_export.py index 5582d0c..cc338c4 100644 --- a/e2e/test_predictions_batch_format_export.py +++ b/e2e/test_predictions_batch_format_export.py @@ -4,7 +4,7 @@ End-to-end tests for PredictionsBatch workflow - Format and Export scenarios. from decimal import Decimal from typing import Any, cast -from unittest.mock import ANY, AsyncMock, call +from unittest.mock import AsyncMock, call import pytest from sientia_do.notifications.models import NotificationLevel @@ -12,7 +12,12 @@ from sqlalchemy import text from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker -from e2e.helpers import assert_prediction, insert_sample_data, make_workflow_id, start_and_await_workflow +from e2e.helpers import ( + assert_prediction, + insert_sample_data, + make_workflow_id, + start_and_await_workflow, +) from laborious.activities.activities import Activities from laborious.workflows.predictions_batch import PredictionsBatch @@ -46,7 +51,8 @@ base_input_data = { 'datetime_columns': ['timestamp', 'created_at'], } -base_query = "SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = {model_id}" +base_query = 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = {model_id}' + def get_base_input_data(model_id): return { @@ -79,8 +85,12 @@ async def test_scenario_3_1_1_default_prediction_export( model_id = 311 with postgres_engine.begin() as conn: - conn.execute(text(f"DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}")) - conn.execute(text(f"DELETE FROM predictions_schema.transformed_data WHERE model_id = {model_id}")) + conn.execute( + text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}') + ) + conn.execute( + text(f'DELETE FROM predictions_schema.transformed_data WHERE model_id = {model_id}') + ) insert_sample_data(postgres_engine, model_id, ['NULL', 78.2]) input_data = get_base_input_data(model_id) @@ -176,7 +186,9 @@ async def test_scenario_3_1_1_default_prediction_export( with postgres_engine.connect() as conn: tf_count = conn.execute( - text(f"SELECT COUNT(*) FROM predictions_schema.transformed_data WHERE model_id = {model_id}") + text( + f'SELECT COUNT(*) FROM predictions_schema.transformed_data WHERE model_id = {model_id}' + ) ).scalar() assert tf_count == 0, 'transform export must be skipped when path_flag is set' @@ -190,9 +202,6 @@ async def test_scenario_3_1_1_default_prediction_export( ) - - - @pytest.mark.asyncio @pytest.mark.integration async def test_scenario_3_1_2_export_with_opc_only( @@ -203,17 +212,17 @@ async def test_scenario_3_1_2_export_with_opc_only( ): """ Scenario 3.1.2: Export with OPC only - + Description: Export to PostgreSQL and OPC server only (no PI Web API). - + Expected Behavior: - Normal formatting - PostgreSQL export executed - OPC export executed - PI Web API activity skipped - Metrics written with OPC metrics - + Assertions: - PI Web API activity NOT called - OPC activity called @@ -280,7 +289,6 @@ async def test_scenario_3_1_2_export_with_opc_only( assert_prediction(postgres_engine, model_id) - @pytest.mark.asyncio @pytest.mark.integration async def test_scenario_3_1_3_export_with_pi_web_api_only( @@ -291,17 +299,17 @@ async def test_scenario_3_1_3_export_with_pi_web_api_only( ): """ Scenario 3.1.3: Export with PI Web API only - + Description: Export to PostgreSQL and PI Web API only (no OPC). - + Expected Behavior: - Normal formatting - PostgreSQL export executed - PI Web API export executed - OPC activity skipped - Metrics written without OPC metrics - + Assertions: - OPC activity NOT called - PI Web API activity called @@ -364,7 +372,6 @@ async def test_scenario_3_1_3_export_with_pi_web_api_only( assert_prediction(postgres_engine, model_id) - @pytest.mark.asyncio @pytest.mark.integration async def test_scenario_3_1_4_export_without_optional_outputs( @@ -375,16 +382,16 @@ async def test_scenario_3_1_4_export_without_optional_outputs( ): """ Scenario 3.1.4: Export Without Optional Outputs - + Description: Export only to PostgreSQL (no OPC or PI Web API). - + Expected Behavior: - Normal formatting - Only PostgreSQL export executed - OPC and PI Web API activities skipped - Metrics written without OPC metrics - + Assertions: - PI Web API activity NOT called - OPC activity NOT called @@ -412,7 +419,6 @@ async def test_scenario_3_1_4_export_without_optional_outputs( assert_prediction(postgres_engine, model_id) - @pytest.mark.asyncio @pytest.mark.integration async def test_scenario_3_1_5_export_without_transformed_data( @@ -423,15 +429,15 @@ async def test_scenario_3_1_5_export_without_transformed_data( ): """ Scenario 3.1.5: Export Without Transformed Data - + Description: Only prediction exported, no transform table. - + Expected Behavior: - Only prediction formatted and exported - Transform export skipped - Single PostgreSQL write - + Assertions: - format_transformed_data NOT called - One PostgreSQL export @@ -443,7 +449,9 @@ async def test_scenario_3_1_5_export_without_transformed_data( insert_sample_data(postgres_engine, model_id, [23.5, 78.2]) with postgres_engine.begin() as conn: - conn.execute(text(f"DELETE FROM predictions_schema.transformed_data WHERE model_id = {model_id}")) + conn.execute( + text(f'DELETE FROM predictions_schema.transformed_data WHERE model_id = {model_id}') + ) input_data = get_base_input_data(model_id) input_data['save_transform'] = False # Don't save transformed data @@ -533,15 +541,16 @@ async def test_scenario_3_1_5_export_without_transformed_data( with postgres_engine.connect() as conn: result_query = conn.execute( - text(f"SELECT COUNT(*) FROM predictions_schema.transformed_data WHERE model_id = {model_id}") + text( + f'SELECT COUNT(*) FROM predictions_schema.transformed_data WHERE model_id = {model_id}' + ) ) count = result_query.scalar() - assert count == 0, f"Expected transform table to be empty, but found {count} records" + assert count == 0, f'Expected transform table to be empty, but found {count} records' assert_prediction(postgres_engine, model_id) - @pytest.mark.asyncio @pytest.mark.integration async def test_scenario_3_2_1_pi_web_api_write_error( @@ -565,7 +574,8 @@ async def test_scenario_3_2_1_pi_web_api_write_error( insert_sample_data(postgres_engine, model_id, [23.5, 78.2]) test_activities.pi_web_api_client.write_value.side_effect = Exception( - "PI Web API service unavailable") + 'PI Web API service unavailable' + ) input_data = get_base_input_data(model_id) input_data['pi_web_api_output_config'] = { @@ -593,14 +603,14 @@ async def test_scenario_3_2_1_pi_web_api_write_error( ) assert_prediction( - postgres_engine, model_id, + postgres_engine, + model_id, prediction_confidence=13, comments='PI Web API service unavailable', ) assert notification_inserts.call_count >= 1 - @pytest.mark.asyncio @pytest.mark.integration async def test_scenario_3_2_2_opc_write_error( @@ -622,14 +632,17 @@ async def test_scenario_3_2_2_opc_write_error( insert_sample_data(postgres_engine, model_id, [23.5, 78.2]) opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data) - opc_write_data.return_value = (False, { - 'notification_id': 'OPC_WRITE_DATA_ERROR_1', - 'message': 'OPC server unavailable', - 'block': 'opc_repository', - 'level': NotificationLevel.ERROR, - 'attachment_content': 'OPC server unavailable', - }) - + opc_write_data.return_value = ( + False, + { + 'notification_id': 'OPC_WRITE_DATA_ERROR_1', + 'message': 'OPC server unavailable', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + 'attachment_content': 'OPC server unavailable', + }, + ) + input_data = get_base_input_data(model_id) input_data['opc_output_config'] = { '1': { @@ -656,7 +669,8 @@ async def test_scenario_3_2_2_opc_write_error( ) assert_prediction( - postgres_engine, model_id, + postgres_engine, + model_id, prediction_confidence=12, comments='Some data could not be written to OPC servers', ) @@ -779,8 +793,8 @@ async def test_scenario_3_2_3_pi_web_api_partial_write_error( ) assert_prediction( - postgres_engine, model_id, + postgres_engine, + model_id, prediction_confidence=13, comments="The number of written tags does not match the number of tag names: Expected ['tag_1', 'tag_3'] tags, but ['tag_1'] tags were written.", ) - diff --git a/e2e/test_predictions_batch_joblib_fallback.py b/e2e/test_predictions_batch_joblib_fallback.py new file mode 100644 index 0000000..a807c3e --- /dev/null +++ b/e2e/test_predictions_batch_joblib_fallback.py @@ -0,0 +1,186 @@ +""" +End-to-end tests for the joblib fallback on the MLflow model load paths. + +`joblib.load` is NOT mocked here: the artifact fixture holds a real sklearn +estimator serialized with `joblib.dump(..., compress=3)`, so a successful run +proves the fallback really deserializes a compressed pickle and serves it. +""" + +import pandas as pd +import pytest +from sqlalchemy import text +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Worker + +from e2e.conftest import JOBLIB_FALLBACK_FLAVOR_ERROR, JOBLIB_FALLBACK_MODEL_NAME +from e2e.helpers import ( + assert_prediction, + assert_stop, + insert_sample_data, + make_workflow_id, + start_and_await_workflow, +) +from laborious.activities.activities import Activities +from laborious.workflows.predictions_batch import PredictionsBatch + +base_query = 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = {model_id}' + + +def build_input_data(model_id: int, model_name: str) -> dict: + """Workflow input for a plain transform + predict run of a given model.""" + return { + 'schedule_name': 'test-schedule', + 'model_name': model_name, + 'model_id': model_id, + 'query': base_query.format(model_id=model_id), + 'schema': 'predictions_schema', + 'table_name': 'predictions', + 'transform_table_name': 'transformed_data', + 'input_filters': { + 'SPECIFIC_VARIABLES_NULL_VALUES': { + 'POLICY': 'CONTINUE', + 'CONFIG': {'variables': ['sensor_1']}, + }, + }, + 'mlflow_transform_filters': { + 'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}, + }, + 'mlflow_predict_filters': { + 'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}, + }, + 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], + 'opc_output_config': {}, + 'pi_web_api_output_config': {}, + 'save_transform': True, + 'prediction_store_policy': 'lts:1', + 'model_config': { + 'retention_minutes': 0, + 'transform_flavor': 'sklearn', + 'predict_flavor': 'sklearn', + }, + 'datetime_columns': ['timestamp', 'created_at'], + } + + +def expected_fallback_prediction(estimator) -> float: + """Prediction the fixture estimator produces for the mocked transform output.""" + features = pd.DataFrame( + { + 'feature_1': [0.234], + 'feature_2': [0.783], + 'timestamp': ['2024-01-01 12:00:00'], + } + ) + return float(estimator.predict(features)[0]) + + +def assert_single_prediction_close(postgres_engine, model_id, expected, tolerance=1e-9): + """Assert exactly one Good prediction row whose value matches `expected`.""" + with postgres_engine.connect() as conn: + rows = conn.execute( + text( + 'SELECT prediction, prediction_status ' + 'FROM predictions_schema.predictions ' + f'WHERE model_id = {model_id}' + ) + ).fetchall() + + assert len(rows) == 1, f'Expected one prediction record, got {len(rows)}' + assert abs(float(rows[0][0]) - expected) < tolerance, ( + f'Expected prediction close to {expected}, got {rows[0][0]}' + ) + assert rows[0][1] == 'Good', f"Expected prediction_status='Good', got {rows[0][1]}" + + +def logged_messages(mock_logger, level: str) -> str: + """Concatenate every message logged at a given custom level.""" + calls = getattr(mock_logger, f'custom_{level}').call_args_list + return ' '.join(str(logged.args[0]) for logged in calls) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_prediction_workflow_recovered_by_joblib_fallback( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + test_activities: Activities, + postgres_engine, + joblib_fallback_artifact, + mlflow_client, + mock_logger, +): + """Flavor loader fails, joblib fallback loads the real compressed pickle, prediction lands.""" + client = temporal_test_env.client + model_id = 271 + insert_sample_data(postgres_engine, model_id, [60.0, 78.2]) + input_data = build_input_data(model_id, JOBLIB_FALLBACK_MODEL_NAME) + + await start_and_await_workflow( + client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-fallback') + ) + + assert_single_prediction_close( + postgres_engine, + model_id, + expected_fallback_prediction(joblib_fallback_artifact['estimator']), + ) + + assert mlflow_client.download_artifacts.call_count == 1 + assert mlflow_client.download_artifacts.call_args.args[1] == 'prediction_model' + + assert JOBLIB_FALLBACK_FLAVOR_ERROR in logged_messages(mock_logger, 'warning') + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_prediction_workflow_without_fallback_is_unchanged( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + test_activities: Activities, + postgres_engine, + mlflow_client, + mock_mlflow_models, +): + """A model whose flavor loader works keeps the current result and downloads nothing.""" + client = temporal_test_env.client + model_id = 272 + insert_sample_data(postgres_engine, model_id, [60.0, 78.2]) + input_data = build_input_data(model_id, 'test_model') + + await start_and_await_workflow( + client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-no-fallback') + ) + + assert_prediction(postgres_engine, model_id, prediction=0.5) + mock_mlflow_models['predict_model'].predict.assert_called() + mlflow_client.download_artifacts.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_prediction_workflow_reports_both_causes_when_fallback_fails( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + test_activities: Activities, + postgres_engine, + joblib_fallback_artifact_without_pickle, + mlflow_client, + mock_logger, +): + """Both attempts fail: workflow stops and the logged traceback carries both causes.""" + mlflow_client.download_artifacts.return_value = joblib_fallback_artifact_without_pickle + + client = temporal_test_env.client + model_id = 273 + insert_sample_data(postgres_engine, model_id, [60.0, 78.2]) + input_data = build_input_data(model_id, JOBLIB_FALLBACK_MODEL_NAME) + + await start_and_await_workflow( + client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-fallback-failed') + ) + + assert_stop(postgres_engine, model_id) + + logged = logged_messages(mock_logger, 'debug') + assert JOBLIB_FALLBACK_FLAVOR_ERROR in logged + assert 'No .pkl file found in prediction_model artifact' in logged diff --git a/e2e/test_predictions_batch_main_workflow.py b/e2e/test_predictions_batch_main_workflow.py index 3f8d37f..9719a9f 100644 --- a/e2e/test_predictions_batch_main_workflow.py +++ b/e2e/test_predictions_batch_main_workflow.py @@ -4,12 +4,11 @@ End-to-end tests for PredictionsBatch workflow - Main workflow scenarios. import asyncio +import pytest from sqlalchemy import text from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker -import pytest - from e2e.helpers import make_workflow_id, start_and_await_workflow from laborious.activities.activities import Activities from laborious.workflows.predictions_batch import PredictionsBatch diff --git a/e2e/test_predictions_batch_prediction_process.py b/e2e/test_predictions_batch_prediction_process.py index af097cd..6623c9e 100644 --- a/e2e/test_predictions_batch_prediction_process.py +++ b/e2e/test_predictions_batch_prediction_process.py @@ -3,7 +3,7 @@ End-to-end tests for PredictionsBatch workflow - Prediction Process scenarios. """ from decimal import Decimal -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import numpy as np import pandas as pd @@ -69,7 +69,9 @@ def get_base_input_data(model_id): def insert_sample_prediction(postgres_engine, model_id): with postgres_engine.begin() as conn: - conn.execute(text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}')) + conn.execute( + text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}') + ) insert_sql = f""" INSERT INTO predictions_schema.predictions (model_id, timestamp, prediction, prediction_confidence, prediction_status, comments, response_time) VALUES @@ -180,7 +182,9 @@ async def test_scenario_2_1_4_input_gate_repeat_without_prior_prediction( model_id = 214 insert_sample_data(postgres_engine, model_id, ['NULL', 78.2]) with postgres_engine.begin() as conn: - conn.execute(text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}')) + conn.execute( + text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}') + ) input_data = get_base_input_data(model_id) input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['POLICY'] = 'REPEAT' await start_and_await_workflow( @@ -370,7 +374,9 @@ async def test_scenario_2_4_1_input_empty_data_stop( client = temporal_test_env.client model_id = 241 with postgres_engine.begin() as conn: - conn.execute(text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}')) + conn.execute( + text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}') + ) input_data = get_base_input_data(model_id) input_data['input_filters'] = {'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}}} await start_and_await_workflow( diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 6a25b8c..d6dc2bc 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -21,10 +21,11 @@ import time import traceback from datetime import datetime, timedelta from io import StringIO -from os import environ, makedirs, path +from os import environ, listdir, makedirs, path from shutil import rmtree from typing import Any, Literal, overload +import joblib import mlflow import pandas as pd from mlflow.entities import Experiment @@ -351,6 +352,117 @@ class MLFlowRepository(SientiaMonitoring): self.info(f'Loaded dataframe from {model_name}:{artifact_path}', metadata) return dataframe + def _find_model_pickle( + self, local_dir: str, artifact_name: str, flavor_error: Exception + ) -> str: + """ + Locate the pickle file of a downloaded model artifact. + + Fallback path only: this is reached after the MLflow flavor loader has + already failed. The target artifact is always a single `.pkl` at the top + level of the artifact directory, so the listing does not recurse — + nested wrapper artifacts must never be deserialized in place of the + model. No other extension is considered and `MLmodel` is not parsed. + + Args: + local_dir (str): Local directory holding the downloaded artifact. + artifact_name (str): Artifact name, e.g. 'prediction_model'. + flavor_error (Exception): Error raised by the MLflow flavor loader, + reported in the message so the original cause is not lost. + + Returns: + str: Path to the first top-level `.pkl` file, in name order. + + Raises: + ValueError: If the top level of `local_dir` has no `.pkl` file. + """ + entries = sorted(listdir(local_dir)) + pkl_files = [ + entry + for entry in entries + if entry.endswith('.pkl') and path.isfile(path.join(local_dir, entry)) + ] + + if not pkl_files: + raise ValueError( + f'No .pkl file found in {artifact_name} artifact. ' + f'Original error: {flavor_error}. ' + f'Top-level entries in {local_dir}: {entries}' + ) + + return path.join(local_dir, pkl_files[0]) + + def _load_model_pickle( + self, local_dir: str, artifact_name: str, flavor_error: Exception + ) -> Any: + """ + Deserialize a downloaded model artifact with `joblib`. + + Fallback path only. `joblib.load` reads the compressed `.pkl` produced + by `joblib.dump`, which is what the MLflow flavor loader fails to read. + The deserialized object is returned as is: no `._model_impl.python_model` + unwrap is applied, because the pickled object already is the model. + + Args: + local_dir (str): Local directory holding the downloaded artifact. + artifact_name (str): Artifact name, e.g. 'prediction_model'. + flavor_error (Exception): Error raised by the MLflow flavor loader. + + Returns: + Any: The object deserialized from the artifact's pickle file. + + Raises: + ValueError: If the artifact directory has no top-level `.pkl` file. + Exception: Any error raised by `joblib.load` itself. + """ + pkl_path = self._find_model_pickle(local_dir, artifact_name, flavor_error) + + self.debug(f'joblib fallback: loading {artifact_name} from {pkl_path}') + + return joblib.load(pkl_path) + + async def _download_and_load_pickle( + self, + model_name: str, + artifact_name: str, + flavor_error: Exception, + metadata: dict[str, Any], + ) -> Any: + """ + Download a model artifact and deserialize it with `joblib`. + + Fallback path only. The download runs without `dst_path`, so MLflow + writes to a per-call temporary directory: the shared `ARTIFACTS_PATH` + tree is never written to and `dowload_artifacts` — which removes its + destination before downloading — is deliberately not reused. The target + artifact is always a `.pkl` and the deserialized object is returned as + is. + + Args: + model_name (str): Registered model name. + artifact_name (str): Artifact to download, e.g. 'prediction_model'. + flavor_error (Exception): Error raised by the MLflow flavor loader. + metadata (dict[str, Any]): Metadata used for structured logging. + + Returns: + Any: The object deserialized from the artifact's pickle file. + + Raises: + ValueError: If the downloaded artifact has no top-level `.pkl` file. + Exception: Any error raised while resolving the Production run, + downloading the artifact, or deserializing the pickle. + """ + run_id = self.get_model_run_id(model_name=model_name, stage='Production') + + self.info( + f'joblib fallback: downloading {artifact_name} of {model_name} from run {run_id}', + metadata, + ) + + local_dir = self.client.download_artifacts(run_id, artifact_name) + + return self._load_model_pickle(local_dir, artifact_name, flavor_error) + async def load_predict_model( self, model_name: str, metadata: dict[str, Any], flavor: str = 'sklearn' ) -> Any: @@ -370,23 +482,43 @@ class MLFlowRepository(SientiaMonitoring): - The model is fetched from the "Production" stage of the MLflow Model Registry. - Warnings during the model loading process are suppressed. """ + if flavor not in ('pyfunc', 'sklearn', 'pytorch'): + raise ValueError(INVALID_FLAVOR_MESSAGE) + model_uri = f'models:/{model_name}/production' self.info(f'Loading prediction model {model_name} from {model_uri}') core_labels = self.get_core_labels(metadata, operation_type='load_predict_model') + fallback_labels = self.get_core_labels( + metadata, operation_type='load_predict_model_joblib_fallback' + ) start_time = time.time() try: if flavor == 'pyfunc': model = mlflow.pyfunc.load_model(model_uri) elif flavor == 'sklearn': model = mlflow.sklearn.load_model(model_uri) - elif flavor == 'pytorch': - model = mlflow.pytorch.load_model(model_uri) else: - raise ValueError(INVALID_FLAVOR_MESSAGE) - except Exception as e: - await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels) - raise e + model = mlflow.pytorch.load_model(model_uri) + except Exception as flavor_error: + self.warning( + f'Flavor loader {flavor} failed for prediction model {model_name}, ' + f'trying joblib fallback. Original error: {flavor_error}', + metadata, + ) + try: + model = await self._download_and_load_pickle( + model_name, 'prediction_model', flavor_error, metadata + ) + except Exception as fallback_error: + await self.emit_metric( + metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=fallback_labels + ) + raise fallback_error from flavor_error + + await self.observe_lag(start_time, metrics.MODEL_READ_LAG, fallback_labels) + await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=fallback_labels) + return model await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels) await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels) @@ -415,25 +547,45 @@ class MLFlowRepository(SientiaMonitoring): model cannot be loaded. """ + if flavor not in ('sklearn', 'pyfunc', 'pytorch'): + raise ValueError(INVALID_FLAVOR_MESSAGE) + latest_production_id = self.get_model_run_id(model_name=model_name, stage='Production') model_uri = self.get_model_uri(latest_production_id, prediction=False) self.info(f'Loading data model {model_name} from {model_uri}') core_labels = self.get_core_labels(metadata, operation_type='load_transform_model') + fallback_labels = self.get_core_labels( + metadata, operation_type='load_transform_model_joblib_fallback' + ) start_time = time.time() try: if flavor == 'sklearn': model = mlflow.sklearn.load_model(model_uri) elif flavor == 'pyfunc': model = mlflow.pyfunc.load_model(model_uri) - elif flavor == 'pytorch': - model = mlflow.pytorch.load_model(model_uri) else: - raise ValueError(INVALID_FLAVOR_MESSAGE) - except Exception as e: - await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels) - raise e + model = mlflow.pytorch.load_model(model_uri) + except Exception as flavor_error: + self.warning( + f'Flavor loader {flavor} failed for data model {model_name}, ' + f'trying joblib fallback. Original error: {flavor_error}', + metadata, + ) + try: + model = await self._download_and_load_pickle( + model_name, 'data_model', flavor_error, metadata + ) + except Exception as fallback_error: + await self.emit_metric( + metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=fallback_labels + ) + raise fallback_error from flavor_error + + await self.observe_lag(start_time, metrics.MODEL_READ_LAG, fallback_labels) + await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=fallback_labels) + return model await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels) await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels) @@ -482,8 +634,29 @@ class MLFlowRepository(SientiaMonitoring): f'Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}' ) - raw_model = mlflow.pyfunc.load_model(artifact_path) - model = raw_model._model_impl.python_model + fallback_labels = self.get_core_labels( + metadata, operation_type='download_model_wrapper_joblib_fallback' + ) + start_time = time.time() + try: + raw_model = mlflow.pyfunc.load_model(artifact_path) + model = raw_model._model_impl.python_model + except Exception as flavor_error: + self.warning( + f'Wrapper loader failed for {model_type} model {model_name}, ' + f'trying joblib fallback. Original error: {flavor_error}', + metadata, + ) + try: + model = self._load_model_pickle(artifact_path, target, flavor_error) + except Exception as fallback_error: + await self.emit_metric( + metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=fallback_labels + ) + raise fallback_error from flavor_error + + await self.observe_lag(start_time, metrics.MODEL_READ_LAG, fallback_labels) + await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=fallback_labels) self.debug( f'Model wrapper loaded: {model.__class__.__name__}:{model.__dict__}', metadata diff --git a/pyproject.toml b/pyproject.toml index db08645..4a90296 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,11 @@ ignore = [ "S105", # hardcoded passwords ok in tests "S106", # hardcoded passwords ok in tests ] +"e2e/conftest.py" = [ + # Imports below the sientia/ModelAnalysis stub are load-order critical: the stub + # must be in sys.modules before Activities is imported, so they cannot move up. + "E402", +] [tool.ruff.lint.mccabe] max-complexity = 15 diff --git a/requirements-light.txt b/requirements-light.txt index 302fb79..d5128de 100644 --- a/requirements-light.txt +++ b/requirements-light.txt @@ -16,3 +16,4 @@ shap pycurl scipy<1.14.0 scikit-learn==1.5.2 +joblib diff --git a/requirements-local.txt b/requirements-local.txt index 73a9a47..852328a 100644 --- a/requirements-local.txt +++ b/requirements-local.txt @@ -15,4 +15,6 @@ hyperopt shap pycurl scipy<1.14.0 -scikit-learn==1.5.2 \ No newline at end of file +scikit-learn==1.4.2 +pycaret==3.3.2 +joblib \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4ad005d..8f994e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ shap pycurl scipy<1.14.0 scikit-learn==1.5.2 +joblib diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 54a75bb..64c5e5f 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -1,13 +1,20 @@ from datetime import UTC, datetime from unittest.mock import ANY, AsyncMock, MagicMock, call, patch +import joblib as joblib_lib import mlflow as mlflow_lib import numpy as np import pytest from pandas import DataFrame, Timestamp +from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ +from sklearn.linear_model import LinearRegression from laborious import metrics -from laborious.utils.repository.model_repository import MLFlowRepository, force_memory_release +from laborious.utils.repository.model_repository import ( + INVALID_FLAVOR_MESSAGE, + MLFlowRepository, + force_memory_release, +) @patch('laborious.utils.repository.model_repository.ctypes') @@ -608,32 +615,46 @@ def test_detect_and_parse_datetime_index_empty(mlflow_repository): assert response.equals(input_data) +# Each case is (index, fragments expected in the error message). +# +# The index is passed explicitly instead of through a dict-of-dict: building a +# DataFrame from nested dicts makes pandas sort the keys to infer the index, which a +# mixed-type index cannot do. +# +# Fragments, not whole messages: the parse branch appends pandas' own multi-line +# error, and the type branch appends the fully qualified class name, so neither is +# stable enough to compare literally. invalid_cases = [ ( - {'value': {'2024-01-01 12:00:00': 1, 2024: 2}}, - "Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format %Y-%m-%d %H:%M:%S. Elements are , .", + ['2024-01-01 12:00:00', 2024], + ["Elements are ,"], ), ( - {'value': {'2024-01-01': 1, '2024-01-02': 2}}, - 'Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format %Y-%m-%d %H:%M:%S. Unable to parse given date format', + ['2024-01-01', '2024-01-02'], + ['Unable to parse given date format'], ), ( - {'value': {Any(): 1, Any(): 2}}, - 'Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format %Y-%m-%d %H:%M:%S. Got .', + [Any(), Any()], + ['Got ."], ), ] @pytest.mark.parametrize('data', invalid_cases) def test_detect_and_parse_datetime_index_error_cases(mlflow_repository, data): - input_data = DataFrame(data[0]) + input_data = DataFrame({'value': [1, 2]}, index=data[0]) - message = data[1] + expected_fragments = data[1] with pytest.raises(ValueError) as e: mlflow_repository.detect_and_parse_datetime_index(input_data, metadata['metadata']) - assert str(e) == message + message = str(e.value) + + assert 'Index must be all timestamp like column' in message + assert f'string in format {DATETIME_FORMAT_WITH_TZ}' in message + for fragment in expected_fragments: + assert fragment in message, f'Expected {fragment!r} in {message!r}' valid_cases = [ @@ -1713,3 +1734,357 @@ async def test_predict_success_ndarray(mlflow_repository): assert 'prediction' in content.columns assert 'response_time' in content.columns assert content.index.tolist() == data.index.tolist() + + +""" +Tests for the joblib fallback on the model load paths +""" + +JOBLIB_PATH = 'laborious.utils.repository.model_repository.joblib' + + +def build_artifact_dir(tmp_path, *filenames): + """Create a fake downloaded artifact directory with the given top-level files.""" + for filename in filenames: + (tmp_path / filename).write_bytes(b'pickle-bytes') + return str(tmp_path) + + +def emitted_tags(emit_metric_mock, metric_object): + """Collect the `tags` kwarg of every emit_metric call for a given metric.""" + return [ + emitted.kwargs['tags'] + for emitted in emit_metric_mock.call_args_list + if emitted.kwargs['metric_object'] is metric_object + ] + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_predict_model_flavor_success_skips_fallback( + joblib_mock, mlflow, mlflow_repository +): + result = await mlflow_repository.load_predict_model('test_model', {}, 'sklearn') + + assert result == mlflow.sklearn.load_model.return_value + joblib_mock.load.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_transform_model_flavor_success_skips_fallback( + joblib_mock, mlflow, mlflow_repository +): + mlflow_repository.get_model_run_id = MagicMock() + mlflow_repository.get_model_uri = MagicMock() + + result = await mlflow_repository.load_transform_model('test_model', {}, 'sklearn') + + assert result == mlflow.sklearn.load_model.return_value + joblib_mock.load.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_download_model_wrapper_success_skips_fallback( + joblib_mock, mlflow, mlflow_repository +): + mlflow_repository.dowload_artifacts = AsyncMock() + + result = await mlflow_repository.download_model('test_model', {}, 'predict', 'pyfunc', True) + + assert result == ( + mlflow.pyfunc.load_model.return_value._model_impl.python_model, + mlflow_repository.dowload_artifacts.return_value, + ) + joblib_mock.load.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_predict_model_invalid_flavor_skips_fallback(joblib_mock, mlflow_repository): + with pytest.raises(ValueError) as exc_info: + await mlflow_repository.load_predict_model('test_model', {}, 'invalid') + + assert str(exc_info.value) == INVALID_FLAVOR_MESSAGE + joblib_mock.load.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + mlflow_repository.emit_metric.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_transform_model_invalid_flavor_skips_fallback(joblib_mock, mlflow_repository): + with pytest.raises(ValueError) as exc_info: + await mlflow_repository.load_transform_model('test_model', {}, 'invalid') + + assert str(exc_info.value) == INVALID_FLAVOR_MESSAGE + joblib_mock.load.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + mlflow_repository.emit_metric.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_predict_model_fallback_recovers_load( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + mlflow.sklearn.load_model.side_effect = Exception('flavor boom') + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( + tmp_path, 'MLmodel', 'model.pkl' + ) + + result = await mlflow_repository.load_predict_model('test_model', {}, 'sklearn') + + assert result is joblib_mock.load.return_value + mlflow_repository.get_model_run_id.assert_called_once_with( + model_name='test_model', stage='Production' + ) + mlflow_repository.client.download_artifacts.assert_called_once_with( + 'run-123', 'prediction_model' + ) + joblib_mock.load.assert_called_once_with(str(tmp_path / 'model.pkl')) + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_transform_model_fallback_recovers_load( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + mlflow.sklearn.load_model.side_effect = Exception('flavor boom') + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.get_model_uri = MagicMock() + mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( + tmp_path, 'MLmodel', 'model.pkl' + ) + + result = await mlflow_repository.load_transform_model('test_model', {}, 'sklearn') + + assert result is joblib_mock.load.return_value + mlflow_repository.get_model_run_id.assert_called_with( + model_name='test_model', stage='Production' + ) + mlflow_repository.client.download_artifacts.assert_called_once_with('run-123', 'data_model') + joblib_mock.load.assert_called_once_with(str(tmp_path / 'model.pkl')) + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_download_model_wrapper_fallback_reuses_downloaded_dir( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + artifact_dir = build_artifact_dir(tmp_path, 'MLmodel', 'python_model.pkl') + mlflow_repository.dowload_artifacts = AsyncMock(return_value=artifact_dir) + mlflow.pyfunc.load_model.side_effect = Exception('flavor boom') + + model, returned_path = await mlflow_repository.download_model( + 'test_model', {}, 'predict', 'pyfunc', True + ) + + mlflow_repository.dowload_artifacts.assert_called_once_with( + 'test_model', {}, 'prediction_model' + ) + mlflow_repository.client.download_artifacts.assert_not_called() + joblib_mock.load.assert_called_once_with(str(tmp_path / 'python_model.pkl')) + + assert model is joblib_mock.load.return_value + assert returned_path == artifact_dir + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_predict_model_both_attempts_fail_chains_causes( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + flavor_error = Exception('flavor boom') + mlflow.sklearn.load_model.side_effect = flavor_error + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( + tmp_path, 'model.pkl' + ) + joblib_mock.load.side_effect = ValueError('joblib boom') + + with pytest.raises(ValueError) as exc_info: + await mlflow_repository.load_predict_model('test_model', {}, 'sklearn') + + assert str(exc_info.value) == 'joblib boom' + assert exc_info.value.__cause__ is flavor_error + + error_tags = emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_ERROR_COUNT) + assert len(error_tags) == 1 + assert error_tags[0]['operation_type'] == 'load_predict_model_joblib_fallback' + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_transform_model_both_attempts_fail_emits_error_metric( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + mlflow.sklearn.load_model.side_effect = Exception('flavor boom') + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.get_model_uri = MagicMock() + mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( + tmp_path, 'model.pkl' + ) + joblib_mock.load.side_effect = ValueError('joblib boom') + + with pytest.raises(ValueError): + await mlflow_repository.load_transform_model('test_model', {}, 'sklearn') + + error_tags = emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_ERROR_COUNT) + assert len(error_tags) == 1 + assert error_tags[0]['operation_type'] == 'load_transform_model_joblib_fallback' + assert emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_COUNT) == [] + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_predict_model_fallback_success_metrics( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + mlflow.sklearn.load_model.side_effect = Exception('flavor boom') + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( + tmp_path, 'model.pkl' + ) + mlflow_repository.warning = MagicMock() + + await mlflow_repository.load_predict_model('test_model', {}, 'sklearn') + + read_tags = emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_COUNT) + assert len(read_tags) == 1 + assert read_tags[0]['operation_type'] == 'load_predict_model_joblib_fallback' + assert emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_ERROR_COUNT) == [] + + mlflow_repository.observe_lag.assert_called_once() + lag_call = mlflow_repository.observe_lag.call_args + assert lag_call.args[1] is metrics.MODEL_READ_LAG + assert lag_call.args[2]['operation_type'] == 'load_predict_model_joblib_fallback' + + mlflow_repository.warning.assert_called_once() + assert 'flavor boom' in mlflow_repository.warning.call_args.args[0] + + +@pytest.mark.parametrize( + ('filenames', 'expected'), + [ + (('MLmodel', 'model.pkl'), 'model.pkl'), + (('MLmodel', 'python_model.pkl'), 'python_model.pkl'), + (('b_model.pkl', 'a_model.pkl'), 'a_model.pkl'), + ], +) +def test_find_model_pickle_selection(mlflow_repository, tmp_path, filenames, expected): + local_dir = build_artifact_dir(tmp_path, *filenames) + + result = mlflow_repository._find_model_pickle( + local_dir, 'prediction_model', Exception('flavor boom') + ) + + assert result == str(tmp_path / expected) + + +def test_find_model_pickle_ignores_nested_pickles(mlflow_repository, tmp_path): + nested = tmp_path / 'artifacts' + nested.mkdir() + (nested / 'stacking_model.pkl').write_bytes(b'pickle-bytes') + + with pytest.raises(ValueError) as exc_info: + mlflow_repository._find_model_pickle( + str(tmp_path), 'prediction_model', Exception('flavor boom') + ) + + assert 'No .pkl file found' in str(exc_info.value) + + +def test_find_model_pickle_empty_directory(mlflow_repository, tmp_path): + with pytest.raises(ValueError) as exc_info: + mlflow_repository._find_model_pickle(str(tmp_path), 'data_model', Exception('flavor boom')) + + assert 'data_model' in str(exc_info.value) + + +def test_find_model_pickle_error_message_carries_context(mlflow_repository, tmp_path): + local_dir = build_artifact_dir(tmp_path, 'MLmodel') + + with pytest.raises(ValueError) as exc_info: + mlflow_repository._find_model_pickle( + local_dir, 'prediction_model', Exception('flavor boom') + ) + + message = str(exc_info.value) + assert 'prediction_model' in message + assert 'flavor boom' in message + assert 'MLmodel' in message + + +def test_find_model_pickle_ignores_directory_named_pkl(mlflow_repository, tmp_path): + (tmp_path / 'algo.pkl').mkdir() + + with pytest.raises(ValueError) as exc_info: + mlflow_repository._find_model_pickle( + str(tmp_path), 'prediction_model', Exception('flavor boom') + ) + + assert 'algo.pkl' in str(exc_info.value) + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_get_model_caches_model_loaded_by_fallback( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + mlflow.sklearn.load_model.side_effect = Exception('flavor boom') + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( + tmp_path, 'model.pkl' + ) + + first = await mlflow_repository.get_model('test_model', {}, 5, 'predict', 'sklearn') + second = await mlflow_repository.get_model('test_model', {}, 5, 'predict', 'sklearn') + + assert first is joblib_mock.load.return_value + assert second is first + assert mlflow.sklearn.load_model.call_count == 1 + assert joblib_mock.load.call_count == 1 + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_predict_reports_both_causes_when_fallback_fails( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + mlflow.sklearn.load_model.side_effect = Exception('flavor boom') + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( + tmp_path, 'model.pkl' + ) + joblib_mock.load.side_effect = ValueError('joblib boom') + + data = DataFrame({'feat_1': {'index_1': 2, 'index_2': 3}}) + + output = await mlflow_repository.predict( + 'test_model', + data, + {'retention_minutes': 0, 'predict_flavor': 'sklearn'}, + metadata['metadata'], + ) + + assert output['success'] is False + assert 'joblib boom' in output['content']['traceback'] + assert 'flavor boom' in output['content']['traceback'] + + +def test_load_model_pickle_real_joblib_round_trip(mlflow_repository, tmp_path): + estimator = LinearRegression().fit(np.array([[0.0], [1.0], [2.0]]), np.array([0.0, 1.0, 2.0])) + joblib_lib.dump(estimator, tmp_path / 'model.pkl', compress=3) + + loaded = mlflow_repository._load_model_pickle( + str(tmp_path), 'prediction_model', Exception('flavor boom') + ) + + assert loaded is not estimator + assert loaded.predict(np.array([[3.0]]))[0] == pytest.approx(3.0) From 695e4c07a68261cfa30fecb613f8becbd62c70d5 Mon Sep 17 00:00:00 2001 From: Eduardo Rios Date: Mon, 17 Aug 2026 16:42:31 -0300 Subject: [PATCH 21/64] SIENTIAPDE-2072: bump sientia_do pin to 1.12.2 Picks up the notification timestamp -> native datetime fix. --- requirements-light.txt | 2 +- requirements-local.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-light.txt b/requirements-light.txt index 302fb79..5daca26 100644 --- a/requirements-light.txt +++ b/requirements-light.txt @@ -3,7 +3,7 @@ psycopg2-binary sqlalchemy asyncua==1.0.6 redis -sientia_do>=1.12.1 +sientia_do>=1.12.2 mlflow prometheus-client botocore diff --git a/requirements-local.txt b/requirements-local.txt index 73a9a47..06a69d7 100644 --- a/requirements-local.txt +++ b/requirements-local.txt @@ -3,7 +3,7 @@ psycopg2-binary sqlalchemy asyncua==1.0.6 redis -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.1 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.2 git+ssh://git@github.com/Aignosi/sientia-model-library.git@0.10.0 prometheus-client botocore diff --git a/requirements.txt b/requirements.txt index 4ad005d..c44b8a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ psycopg2-binary sqlalchemy asyncua==1.0.6 redis -sientia_do>=1.12.1 +sientia_do>=1.12.2 sientia>0.40.0 prometheus-client botocore From d7bf36605d40d8cd9aae6d137461f2bb2f0a79cc Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 19 Aug 2026 11:01:52 -0300 Subject: [PATCH 22/64] QTZPOC-13 QTZPOC-13: Dynamically resolve MLflow pyfunc model code paths to support varied layouts during re-logging. --- .../utils/repository/model_repository.py | 39 ++++++++++++++++- .../utils/repository/test_model_repository.py | 43 ++++++++++++++++--- 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index d6dc2bc..ab007d4 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -1094,6 +1094,43 @@ class MLFlowRepository(SientiaMonitoring): } return retrain_data + def resolve_code_paths(self, artifact_path: str | None, metadata: dict) -> list[str]: + """Resolve the code dependencies to forward when re-logging a pyfunc model. + + MLflow flattens whatever was given as `code_paths` into `/code/`, + so the layout differs per model: some ship a `utils` package, others a + single self-contained module (`_model.py`). Re-logging must forward + the entries that are actually on disk — assuming one fixed layout makes + `_validate_and_copy_code_paths` raise `FileNotFoundError` for the other. + + Args: + artifact_path (str | None): Local directory of the downloaded model + artifact, e.g. `./tmp/artifacts//prediction_model`. + metadata (dict): Metadata for structured logging. + + Returns: + list[str]: Existing paths inside `/code`, empty when + the model carries no code directory. + """ + if not artifact_path: + self.warning('No artifact path available, logging model without code paths', metadata) + return [] + + code_dir = path.join(artifact_path, 'code') + + if not path.isdir(code_dir): + self.warning( + f'No code directory found at {code_dir}, logging model without code paths', + metadata, + ) + return [] + + return [ + path.join(code_dir, entry) + for entry in sorted(listdir(code_dir)) + if entry != '__pycache__' + ] + async def log_model(self, model_data: dict, flavor: str, model_type: str, metadata: dict): """Log a model into the active MLflow run. @@ -1114,7 +1151,7 @@ class MLFlowRepository(SientiaMonitoring): if flavor == 'sklearn': mlflow.sklearn.log_model(model, model_type) elif flavor == 'pyfunc': - code_path = [path.join(model_data['artifact_path'], 'code', 'utils')] + code_path = self.resolve_code_paths(model_data.get('artifact_path'), metadata) self.debug(f'Code path: {code_path}', metadata) diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 64c5e5f..457e29f 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -1049,21 +1049,52 @@ async def test_log_model_sklearn(mlflow, mlflow_repository): ) -@patch('laborious.utils.repository.model_repository.path') +def test_resolve_code_paths_package_layout(tmp_path, mlflow_repository): + code_dir = tmp_path / 'code' + (code_dir / 'utils').mkdir(parents=True) + (code_dir / '__pycache__').mkdir() + + assert mlflow_repository.resolve_code_paths(str(tmp_path), metadata['metadata']) == [ + str(code_dir / 'utils') + ] + + +def test_resolve_code_paths_single_module_layout(tmp_path, mlflow_repository): + code_dir = tmp_path / 'code' + code_dir.mkdir() + (code_dir / 'loop_valve_classifier_model.py').touch() + + assert mlflow_repository.resolve_code_paths(str(tmp_path), metadata['metadata']) == [ + str(code_dir / 'loop_valve_classifier_model.py') + ] + + +def test_resolve_code_paths_without_code_dir(tmp_path, mlflow_repository): + assert mlflow_repository.resolve_code_paths(str(tmp_path), metadata['metadata']) == [] + + +def test_resolve_code_paths_without_artifact_path(mlflow_repository): + assert mlflow_repository.resolve_code_paths(None, metadata['metadata']) == [] + + @pytest.mark.asyncio -async def test_log_model_pyfunc(path, mlflow, mlflow_repository): +async def test_log_model_pyfunc(tmp_path, mlflow, mlflow_repository): + code_dir = tmp_path / 'code' + code_dir.mkdir() + (code_dir / 'loop_valve_classifier_model.py').touch() + model_mock = MagicMock() - model_data = {'model': model_mock, 'artifact_path': 'artifact_path'} + model_data = {'model': model_mock, 'artifact_path': str(tmp_path)} await mlflow_repository.log_model( model_data, 'pyfunc', 'prediction_model', metadata['metadata'] ) mlflow.pyfunc.log_model.assert_not_called() - path.join.assert_called_once_with('artifact_path', 'code', 'utils') - model_mock.store_model.assert_called_once_with( - artifact_path='prediction_model', code_path=[path.join.return_value], to_disk=False + artifact_path='prediction_model', + code_path=[str(code_dir / 'loop_valve_classifier_model.py')], + to_disk=False, ) mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_WRITE_LAG, ANY) From 03ac4b31e521ec880fdc774ad7fbefc400104917 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 20 Aug 2026 08:28:45 -0300 Subject: [PATCH 23/64] QTZPOC-13 fix coverage --- .../utils/repository/test_model_repository.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 457e29f..5ea4ce7 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -1972,6 +1972,30 @@ async def test_load_transform_model_both_attempts_fail_emits_error_metric( assert emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_COUNT) == [] +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_download_model_wrapper_both_attempts_fail_chains_causes( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + flavor_error = Exception('flavor boom') + artifact_dir = build_artifact_dir(tmp_path, 'MLmodel', 'python_model.pkl') + mlflow_repository.dowload_artifacts = AsyncMock(return_value=artifact_dir) + mlflow.pyfunc.load_model.side_effect = flavor_error + joblib_mock.load.side_effect = ValueError('joblib boom') + + with pytest.raises(ValueError) as exc_info: + await mlflow_repository.download_model('test_model', {}, 'predict', 'pyfunc', True) + + assert str(exc_info.value) == 'joblib boom' + assert exc_info.value.__cause__ is flavor_error + + error_tags = emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_ERROR_COUNT) + assert len(error_tags) == 1 + assert error_tags[0]['operation_type'] == 'download_model_wrapper_joblib_fallback' + assert emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_COUNT) == [] + mlflow_repository.observe_lag.assert_not_called() + + @patch(JOBLIB_PATH) @pytest.mark.asyncio async def test_load_predict_model_fallback_success_metrics( From 291d16efa53a4f05667b9dc17234048015bd4b67 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 20 Aug 2026 10:23:24 -0300 Subject: [PATCH 24/64] QTZPOC-13 QTZPOC-13: Centralize MLflow joblib fallback logic, ensuring consistent model version resolution by reusing the initial run ID, improving logging, and updating dependencies. --- .../utils/repository/model_repository.py | 235 ++++++++++++------ requirements-light.txt | 3 +- requirements.txt | 3 +- .../utils/repository/test_model_repository.py | 22 ++ 4 files changed, 179 insertions(+), 84 deletions(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index ab007d4..e89b685 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -16,9 +16,11 @@ Capabilities: import ctypes import gc +import inspect import threading import time import traceback +from collections.abc import Callable from datetime import datetime, timedelta from io import StringIO from os import environ, listdir, makedirs, path @@ -393,7 +395,11 @@ class MLFlowRepository(SientiaMonitoring): return path.join(local_dir, pkl_files[0]) def _load_model_pickle( - self, local_dir: str, artifact_name: str, flavor_error: Exception + self, + local_dir: str, + artifact_name: str, + flavor_error: Exception, + metadata: dict[str, Any] | None = None, ) -> Any: """ Deserialize a downloaded model artifact with `joblib`. @@ -407,6 +413,9 @@ class MLFlowRepository(SientiaMonitoring): local_dir (str): Local directory holding the downloaded artifact. artifact_name (str): Artifact name, e.g. 'prediction_model'. flavor_error (Exception): Error raised by the MLflow flavor loader. + metadata (dict[str, Any] | None): Metadata used for structured + logging, so the chosen pickle path correlates with the other + fallback log lines. Returns: Any: The object deserialized from the artifact's pickle file. @@ -417,7 +426,7 @@ class MLFlowRepository(SientiaMonitoring): """ pkl_path = self._find_model_pickle(local_dir, artifact_name, flavor_error) - self.debug(f'joblib fallback: loading {artifact_name} from {pkl_path}') + self.debug(f'joblib fallback: loading {artifact_name} from {pkl_path}', metadata) return joblib.load(pkl_path) @@ -427,6 +436,7 @@ class MLFlowRepository(SientiaMonitoring): artifact_name: str, flavor_error: Exception, metadata: dict[str, Any], + run_id: str | None = None, ) -> Any: """ Download a model artifact and deserialize it with `joblib`. @@ -438,11 +448,20 @@ class MLFlowRepository(SientiaMonitoring): artifact is always a `.pkl` and the deserialized object is returned as is. + Callers that already resolved a run to build the flavor URI must pass it + as `run_id`: resolving the Production run again here would open a window + where a promotion between the two lookups makes the fallback serve a + different version than the one the flavor attempt tried and logged. Only + callers whose flavor attempt used a symbolic URI — `models://production`, + resolved by MLflow itself — leave it as `None`. + Args: model_name (str): Registered model name. artifact_name (str): Artifact to download, e.g. 'prediction_model'. flavor_error (Exception): Error raised by the MLflow flavor loader. metadata (dict[str, Any]): Metadata used for structured logging. + run_id (str | None): Run already resolved by the caller. When `None`, + the Production run is resolved here. Returns: Any: The object deserialized from the artifact's pickle file. @@ -452,7 +471,8 @@ class MLFlowRepository(SientiaMonitoring): Exception: Any error raised while resolving the Production run, downloading the artifact, or deserializing the pickle. """ - run_id = self.get_model_run_id(model_name=model_name, stage='Production') + if run_id is None: + run_id = self.get_model_run_id(model_name=model_name, stage='Production') self.info( f'joblib fallback: downloading {artifact_name} of {model_name} from run {run_id}', @@ -461,7 +481,80 @@ class MLFlowRepository(SientiaMonitoring): local_dir = self.client.download_artifacts(run_id, artifact_name) - return self._load_model_pickle(local_dir, artifact_name, flavor_error) + return self._load_model_pickle(local_dir, artifact_name, flavor_error, metadata) + + async def _load_with_joblib_fallback( + self, + primary_loader: Callable[[], Any], + fallback_loader: Callable[[Exception], Any], + failure_description: str, + metadata: dict[str, Any], + fallback_labels: dict[str, Any], + core_labels: dict[str, Any] | None = None, + ) -> Any: + """ + Run a model load with the joblib fallback and its observability. + + Single owner of the fallback flow shared by every load path: warn on the + flavor failure without emitting an error metric, try the fallback, and + emit lag/count under the fallback labels — or the error count chained to + the flavor error when the fallback also fails. Fixes to the flow belong + here, not in the call sites. + + Both loaders are called with no observability concern of their own: + `fallback_loader` receives the flavor error and may be sync or async, + which is what lets the download-based paths and the already-downloaded + wrapper path share this method. + + Args: + primary_loader (Callable[[], Any]): Flavor load attempt. Any + exception it raises triggers the fallback. + fallback_loader (Callable[[Exception], Any]): joblib fallback, + receiving the flavor error. Its result is awaited when + awaitable. + failure_description (str): What failed, e.g. 'Flavor loader sklearn + failed for prediction model x'. The fallback and original error + are appended to it. + metadata (dict[str, Any]): Metadata used for structured logging. + fallback_labels (dict[str, Any]): Metric labels for the fallback + path, already carrying its `operation_type`. + core_labels (dict[str, Any] | None): Metric labels for a successful + flavor load. `None` emits no metric on that path. + + Returns: + Any: The model loaded by the flavor loader or by the fallback. + + Raises: + Exception: The fallback error, chained from the flavor error, when + both attempts fail. + """ + start_time = time.time() + try: + model = primary_loader() + except Exception as flavor_error: + self.warning( + f'{failure_description}, trying joblib fallback. Original error: {flavor_error}', + metadata, + ) + try: + model = fallback_loader(flavor_error) + if inspect.isawaitable(model): + model = await model + except Exception as fallback_error: + await self.emit_metric( + metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=fallback_labels + ) + raise fallback_error from flavor_error + + await self.observe_lag(start_time, metrics.MODEL_READ_LAG, fallback_labels) + await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=fallback_labels) + return model + + if core_labels is not None: + await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels) + await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels) + + return model async def load_predict_model( self, model_name: str, metadata: dict[str, Any], flavor: str = 'sklearn' @@ -492,37 +585,26 @@ class MLFlowRepository(SientiaMonitoring): fallback_labels = self.get_core_labels( metadata, operation_type='load_predict_model_joblib_fallback' ) - start_time = time.time() - try: + + def load_flavor() -> Any: if flavor == 'pyfunc': - model = mlflow.pyfunc.load_model(model_uri) - elif flavor == 'sklearn': - model = mlflow.sklearn.load_model(model_uri) - else: - model = mlflow.pytorch.load_model(model_uri) - except Exception as flavor_error: - self.warning( - f'Flavor loader {flavor} failed for prediction model {model_name}, ' - f'trying joblib fallback. Original error: {flavor_error}', - metadata, - ) - try: - model = await self._download_and_load_pickle( - model_name, 'prediction_model', flavor_error, metadata - ) - except Exception as fallback_error: - await self.emit_metric( - metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=fallback_labels - ) - raise fallback_error from flavor_error + return mlflow.pyfunc.load_model(model_uri) + if flavor == 'sklearn': + return mlflow.sklearn.load_model(model_uri) + return mlflow.pytorch.load_model(model_uri) - await self.observe_lag(start_time, metrics.MODEL_READ_LAG, fallback_labels) - await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=fallback_labels) - return model - - await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels) - await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels) - return model + return await self._load_with_joblib_fallback( + primary_loader=load_flavor, + fallback_loader=lambda flavor_error: self._download_and_load_pickle( + model_name, 'prediction_model', flavor_error, metadata + ), + failure_description=( + f'Flavor loader {flavor} failed for prediction model {model_name}' + ), + metadata=metadata, + fallback_labels=fallback_labels, + core_labels=core_labels, + ) async def load_transform_model( self, model_name: str, metadata: dict[str, Any], flavor: str = 'sklearn' @@ -545,6 +627,11 @@ class MLFlowRepository(SientiaMonitoring): Raises: Exception: If the model run ID or URI cannot be retrieved, or if the model cannot be loaded. + + Notes: + - The Production run is resolved once and reused by the joblib + fallback, so a promotion between both steps cannot make the + fallback serve a different version than the flavor attempt. """ if flavor not in ('sklearn', 'pyfunc', 'pytorch'): @@ -559,38 +646,28 @@ class MLFlowRepository(SientiaMonitoring): fallback_labels = self.get_core_labels( metadata, operation_type='load_transform_model_joblib_fallback' ) - start_time = time.time() - try: + + def load_flavor() -> Any: if flavor == 'sklearn': - model = mlflow.sklearn.load_model(model_uri) - elif flavor == 'pyfunc': - model = mlflow.pyfunc.load_model(model_uri) - else: - model = mlflow.pytorch.load_model(model_uri) - except Exception as flavor_error: - self.warning( - f'Flavor loader {flavor} failed for data model {model_name}, ' - f'trying joblib fallback. Original error: {flavor_error}', + return mlflow.sklearn.load_model(model_uri) + if flavor == 'pyfunc': + return mlflow.pyfunc.load_model(model_uri) + return mlflow.pytorch.load_model(model_uri) + + return await self._load_with_joblib_fallback( + primary_loader=load_flavor, + fallback_loader=lambda flavor_error: self._download_and_load_pickle( + model_name, + 'data_model', + flavor_error, metadata, - ) - try: - model = await self._download_and_load_pickle( - model_name, 'data_model', flavor_error, metadata - ) - except Exception as fallback_error: - await self.emit_metric( - metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=fallback_labels - ) - raise fallback_error from flavor_error - - await self.observe_lag(start_time, metrics.MODEL_READ_LAG, fallback_labels) - await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=fallback_labels) - return model - - await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels) - await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels) - - return model + run_id=latest_production_id, + ), + failure_description=f'Flavor loader {flavor} failed for data model {model_name}', + metadata=metadata, + fallback_labels=fallback_labels, + core_labels=core_labels, + ) async def download_model( self, @@ -637,26 +714,20 @@ class MLFlowRepository(SientiaMonitoring): fallback_labels = self.get_core_labels( metadata, operation_type='download_model_wrapper_joblib_fallback' ) - start_time = time.time() - try: - raw_model = mlflow.pyfunc.load_model(artifact_path) - model = raw_model._model_impl.python_model - except Exception as flavor_error: - self.warning( - f'Wrapper loader failed for {model_type} model {model_name}, ' - f'trying joblib fallback. Original error: {flavor_error}', - metadata, - ) - try: - model = self._load_model_pickle(artifact_path, target, flavor_error) - except Exception as fallback_error: - await self.emit_metric( - metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=fallback_labels - ) - raise fallback_error from flavor_error - await self.observe_lag(start_time, metrics.MODEL_READ_LAG, fallback_labels) - await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=fallback_labels) + def load_wrapper_flavor() -> Any: + raw_model = mlflow.pyfunc.load_model(artifact_path) + return raw_model._model_impl.python_model + + model = await self._load_with_joblib_fallback( + primary_loader=load_wrapper_flavor, + fallback_loader=lambda flavor_error: self._load_model_pickle( + artifact_path, target, flavor_error, metadata + ), + failure_description=f'Wrapper loader failed for {model_type} model {model_name}', + metadata=metadata, + fallback_labels=fallback_labels, + ) self.debug( f'Model wrapper loaded: {model.__class__.__name__}:{model.__dict__}', metadata diff --git a/requirements-light.txt b/requirements-light.txt index d5128de..48cb400 100644 --- a/requirements-light.txt +++ b/requirements-light.txt @@ -15,5 +15,6 @@ hyperopt shap pycurl scipy<1.14.0 -scikit-learn==1.5.2 +scikit-learn==1.4.2 +pycaret==3.3.2 joblib diff --git a/requirements.txt b/requirements.txt index 8f994e9..4b2a2a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,5 +15,6 @@ hyperopt shap pycurl scipy<1.14.0 -scikit-learn==1.5.2 +scikit-learn==1.4.2 +pycaret==3.3.2 joblib diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 5ea4ce7..4a89d69 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -1903,6 +1903,28 @@ async def test_load_transform_model_fallback_recovers_load( joblib_mock.load.assert_called_once_with(str(tmp_path / 'model.pkl')) +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_transform_model_fallback_reuses_resolved_run( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + """A promotion between both steps must not change the run served.""" + mlflow.sklearn.load_model.side_effect = Exception('flavor boom') + mlflow_repository.get_model_run_id = MagicMock(side_effect=['run-A', 'run-B']) + mlflow_repository.get_model_uri = MagicMock() + mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( + tmp_path, 'MLmodel', 'model.pkl' + ) + + result = await mlflow_repository.load_transform_model('test_model', {}, 'sklearn') + + assert result is joblib_mock.load.return_value + mlflow_repository.get_model_run_id.assert_called_once_with( + model_name='test_model', stage='Production' + ) + mlflow_repository.client.download_artifacts.assert_called_once_with('run-A', 'data_model') + + @patch(JOBLIB_PATH) @pytest.mark.asyncio async def test_download_model_wrapper_fallback_reuses_downloaded_dir( From 0e445de386942728e5d6556b3b5ca32d981ef38b Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 25 Aug 2026 10:40:46 -0300 Subject: [PATCH 25/64] QTZPOC-20 QTZPOC-20: Implement .sientia model bundle import workflow Introduces a new Temporal workflow and activities to import .sientia model bundles. This enables: - Secure decryption and validation of bundles through a seven-gate process. - Provisioning imported models in MLflow and updating the MongoDB model listing. - Comprehensive status tracking in the public.experiment_run table with new IMPORT_SUCCESS/IMPORT_ERROR states. - Dedicated metrics, robust error handling, and configurable settings. - Includes updated e2e database fixtures and extensive unit/integration tests for bundle format, reader, and overall workflow. --- e2e/fixtures/experiment_run.sql | 67 ++ laborious/activities/activities.py | 45 +- laborious/activities/model_import.py | 1044 +++++++++++++++++ laborious/activities/model_import_errors.py | 348 ++++++ laborious/metrics.py | 39 + laborious/utils/bundle/__init__.py | 27 + laborious/utils/bundle/format.py | 216 ++++ laborious/utils/bundle/reader.py | 473 ++++++++ laborious/utils/bundle/steps.py | 55 + laborious/utils/connectors_config.py | 72 +- laborious/utils/import_password.py | 98 ++ laborious/worker/worker.py | 47 + laborious/workflows/import_model.py | 416 +++++++ requirements-local.txt | 14 +- requirements.txt | 21 +- tests/fixtures/bundle/README.md | 86 ++ .../bundle/golden_model_v3.metadata.json | 24 + tests/fixtures/bundle/golden_model_v3.sientia | Bin 0 -> 110767 bytes tests/fixtures/bundle/regenerate.py | 185 +++ tests/helpers/__init__.py | 0 tests/helpers/bundle_factory.py | 272 +++++ tests/laborious/utils/bundle/__init__.py | 0 tests/laborious/utils/bundle/conftest.py | 34 + tests/laborious/utils/bundle/test_format.py | 227 ++++ tests/laborious/utils/bundle/test_reader.py | 460 ++++++++ .../laborious/utils/test_connectors_config.py | 113 +- tests/laborious/utils/test_import_password.py | 99 ++ 27 files changed, 4471 insertions(+), 11 deletions(-) create mode 100644 e2e/fixtures/experiment_run.sql create mode 100644 laborious/activities/model_import.py create mode 100644 laborious/activities/model_import_errors.py create mode 100644 laborious/utils/bundle/__init__.py create mode 100644 laborious/utils/bundle/format.py create mode 100644 laborious/utils/bundle/reader.py create mode 100644 laborious/utils/bundle/steps.py create mode 100644 laborious/utils/import_password.py create mode 100644 laborious/workflows/import_model.py create mode 100644 tests/fixtures/bundle/README.md create mode 100644 tests/fixtures/bundle/golden_model_v3.metadata.json create mode 100644 tests/fixtures/bundle/golden_model_v3.sientia create mode 100644 tests/fixtures/bundle/regenerate.py create mode 100644 tests/helpers/__init__.py create mode 100644 tests/helpers/bundle_factory.py create mode 100644 tests/laborious/utils/bundle/__init__.py create mode 100644 tests/laborious/utils/bundle/conftest.py create mode 100644 tests/laborious/utils/bundle/test_format.py create mode 100644 tests/laborious/utils/bundle/test_reader.py create mode 100644 tests/laborious/utils/test_import_password.py diff --git a/e2e/fixtures/experiment_run.sql b/e2e/fixtures/experiment_run.sql new file mode 100644 index 0000000..d43cfd1 --- /dev/null +++ b/e2e/fixtures/experiment_run.sql @@ -0,0 +1,67 @@ +-- public.experiment_run as migrations V1 … V12 leave it, for the e2e Postgres container. +-- +-- Source of truth: sientia-dataops-database-migrations, +-- postgres/sientia-core-mlops-bff/migrations/ — this file mirrors the *final state* of: +-- +-- V1 create_table_experiment_run (13 columns, one index) +-- V2 update_check_constraint_on_status (named the status CHECK) +-- V3 update_check_constraints (named every other CHECK) +-- V4 update_status_column_size_and_add_orchestrator_validation_error (status -> VARCHAR(50)) +-- V5 migrate_mage_statuses_to_orchestrator (data only) +-- V6 rename_mage_response_data_column (-> orchestrator_response_data) +-- V7 migrate_mlflow_statuses_to_tracking (data only) +-- V8 remove_orchestrator_request_sent_status (status vocabulary) +-- V9 remove_file_upload_statuses (status vocabulary) +-- V10 remove_unused_statuses (status vocabulary) +-- V11 add_run_type_to_experiment_run (run_type + reindexed) +-- V12 add_import_statuses_to_experiment_run (IMPORT_SUCCESS / IMPORT_ERROR + pairing) +-- +-- The same drift trade already accepted for the bundle fixtures: this is a copy, and the tests +-- that read it are what make the copy auditable. If a migration lands there, update this file and +-- the version list above. Nothing in this repository owns the DDL — no import ever executes it +-- outside the test suite. + +CREATE TABLE IF NOT EXISTS public.experiment_run ( + id BIGSERIAL CONSTRAINT pk_experiment_run PRIMARY KEY, + experiment_name VARCHAR(50) NOT NULL, + run_name VARCHAR(50), + username VARCHAR(320) NOT NULL, + status VARCHAR(50) NOT NULL, + error_message VARCHAR(1024), + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, + updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, + bucket_name VARCHAR(63), + file_name VARCHAR(1024), + request_data JSONB NOT NULL, + orchestrator_response_data JSONB, + run_type VARCHAR(20) NOT NULL DEFAULT 'TRAIN', + + CONSTRAINT chk_experiment_run_experiment_name_min_length CHECK (LENGTH(experiment_name) >= 3), + CONSTRAINT chk_experiment_run_run_name_min_length CHECK (LENGTH(run_name) >= 3), + CONSTRAINT chk_experiment_run_username_min_length CHECK (LENGTH(username) >= 3), + CONSTRAINT chk_experiment_run_username_pattern + CHECK (username ~* '^[a-zA-Z0-9_!#$%&''*+/=?`{|}~^.-]+@[a-zA-Z0-9.-]+$'), + CONSTRAINT chk_experiment_run_error_message_min_length CHECK (LENGTH(error_message) >= 3), + CONSTRAINT chk_experiment_run_bucket_name_min_length CHECK (LENGTH(bucket_name) >= 3), + CONSTRAINT chk_experiment_run_bucket_name_pattern CHECK (bucket_name ~* '^[a-z0-9-]+$'), + CONSTRAINT chk_experiment_run_file_name_min_length CHECK (LENGTH(file_name) >= 3), + CONSTRAINT chk_experiment_run_file_name_pattern + CHECK (file_name ~* '^[a-zA-Z0-9 /._-]+$'), + + -- V12: the two shared stage values plus the training and import terminals. + CONSTRAINT chk_experiment_run_status CHECK (status IN ('PENDING', 'ORCHESTRATOR_REQUEST_ERROR', + 'ORCHESTRATOR_WAITING_PROC', + 'ORCHESTRATOR_VALIDATION_ERROR', + 'TRAINING_SUCCESS', 'TRAINING_ERROR', + 'IMPORT_SUCCESS', 'IMPORT_ERROR')), + CONSTRAINT chk_experiment_run_run_type CHECK (run_type IN ('TRAIN', 'IMPORT')), + + -- V12: each pair of terminal values belongs to its own run_type. The shared values stay out. + CONSTRAINT chk_experiment_run_terminal_status_run_type CHECK ( + (run_type = 'IMPORT' OR status NOT IN ('IMPORT_SUCCESS', 'IMPORT_ERROR')) + AND (run_type = 'TRAIN' OR status NOT IN ('TRAINING_SUCCESS', 'TRAINING_ERROR')) + ) +); + +CREATE INDEX IF NOT EXISTS idx_experiment_run_user_exp_status_date_type + ON public.experiment_run (username, experiment_name, status, created_at, run_type); diff --git a/laborious/activities/activities.py b/laborious/activities/activities.py index df78a08..f4a9a2c 100644 --- a/laborious/activities/activities.py +++ b/laborious/activities/activities.py @@ -7,16 +7,22 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import Logger from sientia_do.repository.minio_repository import MinioRepository + from sientia_do.repository.mongodb_repository import MongoDBRepository from laborious.activities.api import API from laborious.activities.gates import Gates from laborious.activities.mlflow import MLFlow + from laborious.activities.model_import import ModelImport from laborious.activities.model_metrics import ModelMetrics from laborious.activities.opc import OPC from laborious.activities.storage import Storage + from laborious.utils.connectors_config import ( + build_import_config, + build_import_status_config, + ) -class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API): +class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, ModelImport): """ Main activities orchestrator for the Laborious system. @@ -50,6 +56,9 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API): pi_web_api_config: dict[str, Any], logger: Logger, notification_handler: NotificationHandler, + mongo_config: dict[str, Any] | None = None, + import_config: dict[str, Any] | None = None, + import_status_config: dict[str, Any] | None = None, ): """ Initialize the Activities orchestrator with all required configurations. @@ -68,6 +77,13 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API): Required keys: base_url, auth_type, auth_token logger: Logger instance for observability and debugging notification_handler: Notification handler for alerts and monitoring + mongo_config: MongoDB configuration for the model listing the import writes. When + absent, the import activities that need it refuse rather than guess — the + notification handler keeps its own client either way. + import_config: `build_import_config()`; read from the environment when absent + import_status_config: `build_import_status_config()`; read from the environment when + absent. This is the BFF database holding the import log, a second Postgres + connection that is never the one Storage holds. Raises: Exception: If any parent class initialization fails @@ -147,6 +163,32 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API): metrics_controller=metrics_controller, ) + mongo_repository = ( + MongoDBRepository( + connection_string=mongo_config['connection_string'], + database_name=mongo_config['database_name'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + if mongo_config + else None + ) + + ModelImport.__init__( + self, + status_config=import_status_config or build_import_status_config(), + import_config=import_config or build_import_config(), + minio_repository=minio_repository, + # The MLflow repository is shared rather than rebuilt: setting the tracking URI is a + # process-wide side effect, so two instances would be two chances to disagree. + mlflow_repository=self.model_monitoring_repository, + mongo_repository=mongo_repository, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + async def shutdown(self): """ Gracefully shutdown all activities and clean up resources. @@ -167,3 +209,4 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API): await OPC.aclose(self) ModelMetrics.close(self) API.close(self) + ModelImport.close(self) diff --git a/laborious/activities/model_import.py b/laborious/activities/model_import.py new file mode 100644 index 0000000..125ee97 --- /dev/null +++ b/laborious/activities/model_import.py @@ -0,0 +1,1044 @@ +"""Model import activities: the import log row, the bundle, MLflow, the model document, cleanup. + +The import log row is **not created here**. It is `public.experiment_run` in the Postgres database +`sientia-core-mlops-bff`; the frontend inserts it through the BFF with `run_type='IMPORT'` and +`status='PENDING'`, and passes its `id` in the workflow input. Every write in this module is an +`UPDATE ... WHERE id = :import_run_id`, on an engine of its own against that database — never the +`sientia` database Laborious itself uses, and never in a transaction shared with a provisioning +write. + +Three classes of status write, and only the first one is strict: + +- **A, the claim** — `WHERE id AND run_type='IMPORT' AND (PENDING OR my own workflow id)`. This is + the ownership decision, and a claim matching nothing fails the import explicitly. +- **B, progress** — `WHERE id`, nothing else. A hint. Its failure never fails the import. +- **C, terminal** — `WHERE id`, nothing else, in the workflow's `finally`, after cleanup. The + authority: it rewrites status, code, reason and step from one classification of the outcome. +""" + +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + import json + import shutil + from datetime import timedelta + from enum import StrEnum + from pathlib import Path + from typing import Any, Final + + from minio.commonconfig import ENABLED, Filter + from minio.lifecycleconfig import Expiration, LifecycleConfig, Rule + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.observability.logger import Logger + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.repository.minio_repository import MinioRepository + from sientia_do.repository.mongodb_repository import MongoDBRepository + from sientia_do.temporal.constants import now + from sqlalchemy import create_engine, text + from sqlalchemy.engine import URL + from sqlalchemy.exc import DataError, IntegrityError, ProgrammingError + from temporalio.common import RetryPolicy + + from laborious import metrics + from laborious.activities.model_import_errors import ( + ImportErrorCode, + ImportGateRejectedError, + ImportLogRowNotClaimableError, + ImportStatusRejectedError, + classify_import_failure, + report_import_failure, + ) + from laborious.utils.bundle import format as fmt + from laborious.utils.bundle.reader import BundleGateError, BundleLimits, BundleReader + from laborious.utils.bundle.steps import ImportStep + from laborious.utils.import_password import ( + EnvelopeError, + decrypt_password_envelope, + load_envelope_key, + ) + from laborious.utils.repository.model_repository import MLFlowRepository + +# The table is a constant of the code, never an input: there is exactly one import log on this +# platform, and letting a caller name it would be a SQL-injection surface bought for nothing. The +# statements below write the name out literally rather than interpolating this constant — an +# interpolation is a shape a reader (and bandit) has to reason about, and there is nothing to +# parameterise. The constant is what the tests assert every statement against. +IMPORT_LOG_TABLE: Final[str] = 'public.experiment_run' + +RUN_NAME_MAX_LENGTH: Final[int] = 50 +RUN_NAME_PREFIX: Final[str] = 'import-' + +_MISSING_MINIO = 'Minio repository not initialized' +_MISSING_MLFLOW = 'MLflow repository not initialized' +_MISSING_MONGO = 'MongoDB repository not initialized' + +# Where a digest may sit inside the BFF-owned `request_data`. Checked in this order; absence is not +# a failure, only a disagreement is. +REQUEST_DIGEST_KEYS: Final[tuple[str, ...]] = ( + 'expected_digest', + 'digest', + 'sha256', + 'file_digest', +) + + +class ImportState(StrEnum): + """The workflow's internal vocabulary, mapped onto the shared status column by D17.""" + + RUNNING = 'RUNNING' + COMPLETED = 'COMPLETED' + ERROR = 'ERROR' + + +class ExperimentRunStatus(StrEnum): + """The values this workflow writes into `experiment_run.status`. + + Two are **reused** from the training flow because they describe a stage, not a flow — the row's + flow is told by `run_type`, not by the status. Two are **new in V12**, because + `TRAINING_SUCCESS` / `TRAINING_ERROR` would lie about where the row came from. Nobody should + "fix" the reuse into an `IMPORT_RUNNING` twin: it would carry no information `run_type` does not + already carry, and it would widen the pairing CHECK and the frontend's switch for nothing. + """ + + ORCHESTRATOR_WAITING_PROC = 'ORCHESTRATOR_WAITING_PROC' # reused + ORCHESTRATOR_VALIDATION_ERROR = 'ORCHESTRATOR_VALIDATION_ERROR' # reused + IMPORT_SUCCESS = 'IMPORT_SUCCESS' # new in V12 + IMPORT_ERROR = 'IMPORT_ERROR' # new in V12 + + +# A failure at these steps is the user's to act on — re-export, re-type the password, re-submit — +# which is the one thing the import screen can branch on without reading the error code. +USER_ACTIONABLE_STEPS: Final[frozenset[ImportStep]] = frozenset( + { + ImportStep.RECEIVED, + ImportStep.DOWNLOAD, + ImportStep.DECRYPTION, + ImportStep.ARCHIVE_INSPECTION, + ImportStep.EXTRACTION, + ImportStep.STRUCTURE_VALIDATION, + ImportStep.CONTENT_POLICY, + } +) + +# Only the status writes retry, and only because they are idempotent: `UPDATE ... SET +# WHERE id` leaves the row the same however many times it runs. A provisioning call has no such +# shape, which is why it keeps `maximum_attempts=1`. Defined here rather than in `sientia_do`, which +# other repositories share. +IMPORT_STATUS_RETRY_POLICY: Final[RetryPolicy] = RetryPolicy( + maximum_attempts=10, + initial_interval=timedelta(seconds=1), + backoff_coefficient=2.0, + maximum_interval=timedelta(seconds=30), + non_retryable_error_types=['ImportLogRowNotClaimableError', 'ImportStatusRejectedError'], +) + +# Progress writes are hints nobody reads — the terminal write rewrites them — so a 151-second pause +# before each of a dozen stage updates would stall a working import for twenty minutes because of a +# database the import itself does not need. +IMPORT_STATUS_HINT_RETRY_POLICY: Final[RetryPolicy] = RetryPolicy( + maximum_attempts=3, + initial_interval=timedelta(seconds=1), + backoff_coefficient=2.0, + maximum_interval=timedelta(seconds=2), + non_retryable_error_types=['ImportLogRowNotClaimableError', 'ImportStatusRejectedError'], +) + +_CLAIM_SQL = text( + """ + UPDATE public.experiment_run + SET status = :status, + updated_at = :updated_at, + orchestrator_response_data = CAST(:detail AS JSONB) + WHERE id = :import_run_id + AND run_type = 'IMPORT' + AND (status = 'PENDING' + OR (status = 'ORCHESTRATOR_WAITING_PROC' + AND orchestrator_response_data -> 'import' ->> 'workflow_id' = :workflow_id)) + """ +) + +_READ_ROW_SQL = text( + """ + SELECT id, + run_type, + status, + bucket_name, + file_name, + request_data, + orchestrator_response_data -> 'import' ->> 'workflow_id' AS claimed_by + FROM public.experiment_run + WHERE id = :import_run_id + """ +) + +# Class B and C key on the id alone: ownership was decided once, by the claim. Re-checking it on +# every later write buys nothing and costs the thing that matters — a predicate miss turning a +# pipeline that already imported the model into a reported failure. +_PROGRESS_SQL = text( + """ + UPDATE public.experiment_run + SET status = :status, + updated_at = :updated_at, + orchestrator_response_data = CAST(:detail AS JSONB) + WHERE id = :import_run_id + """ +) + +_NAMES_SQL = text( + """ + UPDATE public.experiment_run + SET experiment_name = :experiment_name, + run_name = :run_name, + updated_at = :updated_at, + orchestrator_response_data = CAST(:detail AS JSONB) + WHERE id = :import_run_id + """ +) + +_TERMINAL_SQL = text( + """ + UPDATE public.experiment_run + SET status = :status, + error_message = :error_message, + updated_at = :updated_at, + orchestrator_response_data = CAST(:detail AS JSONB) + WHERE id = :import_run_id + """ +) + + +def map_state_to_status( + state: ImportState, step: ImportStep | str | None = None +) -> ExperimentRunStatus: + """Map the workflow's internal state onto the shared status column (design D17). + + `RUNNING` is the orchestrator holding the row. `COMPLETED` is `IMPORT_SUCCESS`. An `ERROR` is + split in two by *who can act on it*: the user, for everything up to the content policy, or only + support, for everything after it — the same split the table already draws between + `ORCHESTRATOR_VALIDATION_ERROR` and `TRAINING_ERROR`. + """ + if state is ImportState.RUNNING: + return ExperimentRunStatus.ORCHESTRATOR_WAITING_PROC + if state is ImportState.COMPLETED: + return ExperimentRunStatus.IMPORT_SUCCESS + if step is not None and ImportStep(step) in USER_ACTIONABLE_STEPS: + return ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR + return ExperimentRunStatus.IMPORT_ERROR + + +def derive_run_name(model_name: str, model_version: str) -> str: + """`import-{model_name}-v{model_version}`, truncated inside the model-name segment. + + Deterministic, because the recorded name and the created run's name must not diverge: the run is + created with exactly this value, passed in as an argument rather than recomputed. The `import-` + prefix guarantees the column's length-3 floor; the truncation is what keeps it under 50. + """ + suffix = f'-v{model_version}' + room = RUN_NAME_MAX_LENGTH - len(RUN_NAME_PREFIX) - len(suffix) + if room < 1: + # A version string long enough to leave no room for a name: keep the prefix and as much of + # the version as fits, so the value is still a legible, deterministic 50 characters. + return f'{RUN_NAME_PREFIX}{suffix.lstrip("-")}'[:RUN_NAME_MAX_LENGTH] + return f'{RUN_NAME_PREFIX}{model_name[:room]}{suffix}' + + +def build_import_detail( + *, + workflow_id: str, + step: ImportStep | str | None = None, + code: ImportErrorCode | str | None = None, + gate: int | None = None, + source: dict[str, Any] | None = None, + cleanup_failed: bool = False, +) -> str: + """Build `orchestrator_response_data` as JSON text. + + The whole `import` object is rewritten on every write, which is what keeps the workflow + identifier the claim wrote from ever being dropped. There is no column for any of this (design + D21): the code the frontend switches on and the identifier support quotes live at fixed keys in + the JSON column the workflow already owns. + """ + detail: dict[str, Any] = {'workflow_id': workflow_id, 'gate': gate} + if step is not None: + detail['step'] = str(step) + if code is not None: + detail['code'] = str(code) + if source: + detail['source'] = source + if cleanup_failed: + detail['cleanup_failed'] = True + return json.dumps({'import': detail}) + + +class ModelImport(SientiaMonitoring): + """Activities turning a `.sientia` object into a registered model, or into a durable ERROR.""" + + def __init__( + self, + status_config: dict[str, Any], + import_config: dict[str, Any], + minio_repository: MinioRepository | None = None, + mlflow_repository: MLFlowRepository | None = None, + mongo_repository: MongoDBRepository | None = None, + logger: Logger | None = None, + notification_handler: NotificationHandler | None = None, + metrics_controller: MetricsController | None = None, + ): + """ + Args: + status_config: `build_import_status_config()` — the BFF database holding the import log. + This is a **second** Postgres connection, independent of the one Laborious already + holds, which is what makes the transactional independence of design D7 literal. + import_config: `build_import_config()` — bucket, prefix, ceilings, work directory, + envelope key and the models collection name. + minio_repository: the shared MinIO repository, for the uploaded bundle. + mlflow_repository: the shared MLflow repository — the same instance the rest of the + worker uses, because setting the tracking URI is a process-wide side effect. + mongo_repository: MongoDB, where the model listing lives. + logger: logger instance. + notification_handler: the notification path the rest of the repository uses. + metrics_controller: metrics controller. + """ + SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) + self.import_config = import_config + self.status_config = status_config + self.import_minio_repository = minio_repository + self.import_mlflow_repository = mlflow_repository + self.mongo_repository = mongo_repository + self.status_engine = create_engine( + URL.create( + 'postgresql', + username=status_config['user'], + password=status_config['password'], + host=status_config['host'], + port=status_config['port'], + database=status_config['dbname'], + ), + pool_size=2, + max_overflow=3, + pool_pre_ping=True, + ) + + def close(self) -> None: + """Dispose the status engine. The import holds no other connection.""" + engine = getattr(self, 'status_engine', None) + if engine is not None: + engine.dispose() + + # ------------------------------------------------------------------ helpers + + @staticmethod + def _timestamp() -> Any: + """`updated_at` is `TIMESTAMP WITHOUT TIME ZONE`, so the value handed over is naive.""" + moment = now() + return moment.replace(tzinfo=None) if moment.tzinfo else moment + + @staticmethod + def _digest_from_request_data(request_data: Any) -> str | None: + """Find the uploader's digest inside the BFF-owned request payload, if it recorded one.""" + if not isinstance(request_data, dict): + return None + for key in REQUEST_DIGEST_KEYS: + value = request_data.get(key) + if isinstance(value, str) and value: + return value.lower() + return None + + def _execute_status_write(self, statement: Any, parameters: dict[str, Any]) -> int: + """Run one status statement in its own transaction and return the rows it affected. + + Raises: + ImportStatusRejectedError: the database accepted the connection and refused the + statement — a CHECK violation, an undefined column, a value too long. Non-retryable, + so a missing `V12` fails on the first attempt instead of after the full envelope. + Exception: anything else (connection refused, reset, timeout, a pod being replaced) is + left to propagate, because repeating it is exactly what fixes it. + """ + try: + with self.status_engine.begin() as connection: + return connection.execute(statement, parameters).rowcount + except (IntegrityError, ProgrammingError, DataError) as error: + raise ImportStatusRejectedError( + 'the import log database refused the status statement' + ) from error + + def _read_row(self, import_run_id: Any) -> dict[str, Any] | None: + """Read the record. A read, not a write: nothing of somebody else's is overwritten.""" + with self.status_engine.connect() as connection: + row = connection.execute(_READ_ROW_SQL, {'import_run_id': import_run_id}).mappings() + found = row.first() + return dict(found) if found is not None else None + + @staticmethod + def _describe_unclaimable(row: dict[str, Any] | None, workflow_id: str) -> str: + """Name the cause of a zero-row claim: missing row, training row, or someone else's.""" + if row is None: + return 'no such import log row' + if row.get('run_type') != 'IMPORT': + return f'row is a {row.get("run_type")} run, not an import' + claimed_by = row.get('claimed_by') + if claimed_by and claimed_by != workflow_id: + return f'row is already claimed by workflow {claimed_by}' + return f'row is in status {row.get("status")} and cannot be claimed' + + # -------------------------------------------------------------- class A: claim + + @activity.defn(name='claim_import_status') + async def claim_import_status(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Claim the import log row the frontend inserted. The workflow's first write. + + One conditional `UPDATE`, whose second disjunct makes a re-execution of the same workflow + idempotent instead of a conflict — a lost response on attempt 1 must not become a false + "not mine" on attempt 2. Zero rows affected covers the id typo, the duplicated start and the + wrong `run_type` in one condition, with no read-then-write race, and it is what stops a + duplicated workflow start from provisioning twice. + + Args (input_data): + metadata (dict): workflow metadata for logging and metrics. + import_run_id (int): the record's primary key, from the workflow input. + workflow_id (str): this workflow's identifier, recorded in the JSON detail. + source (dict, optional): the bundle's declared origin, when already known. + + Returns: + dict: `claimed`, plus the row values the input guards are checked against — + `bucket_name`, `file_name` and `request_digest`. + + Raises: + ImportLogRowNotClaimableError: the row is not this workflow's. Non-retryable, reported + through the channel, and **written to no row** — the loudness has to go somewhere + that is not somebody else's audit record. + """ + metadata = input_data.get('metadata', {}) + import_run_id = input_data['import_run_id'] + workflow_id = input_data['workflow_id'] + + affected = self._execute_status_write( + _CLAIM_SQL, + { + 'status': ExperimentRunStatus.ORCHESTRATOR_WAITING_PROC.value, + 'updated_at': self._timestamp(), + 'detail': build_import_detail( + workflow_id=workflow_id, + step=ImportStep.RECEIVED, + source=input_data.get('source'), + ), + 'import_run_id': import_run_id, + 'workflow_id': workflow_id, + }, + ) + + row = self._read_row(import_run_id) + if affected == 0: + observed = self._describe_unclaimable(row, workflow_id) + error = ImportLogRowNotClaimableError( + f'import log row {import_run_id} could not be claimed: {observed}', + import_run_id=import_run_id, + observed=observed, + ) + await report_import_failure( + self, + None, + error, + import_run_id=import_run_id, + workflow_id=workflow_id, + metadata=metadata, + code=ImportErrorCode.LOG_ROW_NOT_CLAIMABLE, + marker='not_claimable', + ) + raise error + + await self.emit_metric( + metric_object=metrics.MODEL_IMPORT_STARTED_COUNT, + tags=self.get_core_labels(metadata, operation_type=ImportStep.RECEIVED.value), + ) + self.info(f'import log row {import_run_id} claimed by {workflow_id}', metadata) + row = row or {} + return { + 'claimed': True, + 'bucket_name': row.get('bucket_name'), + 'file_name': row.get('file_name'), + 'request_digest': self._digest_from_request_data(row.get('request_data')), + } + + # ----------------------------------------------------------- class B: progress + + @activity.defn(name='record_import_status') + async def record_import_status(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Record the step the workflow is about to attempt. A hint, keyed on the id alone. + + Its failure is **not** an import failure: the workflow reports it through the channel and + carries on, because losing a hint costs a record pointing one step behind and nothing else — + the terminal write rewrites status, code, reason and step from one classification. + + Args (input_data): + metadata (dict): workflow metadata. + import_run_id (int): the record's primary key. + workflow_id (str): recorded in the detail, so it is never dropped. + step (str): the step about to be attempted. + source (dict, optional): the bundle's declared origin. + """ + affected = self._execute_status_write( + _PROGRESS_SQL, + { + 'status': ExperimentRunStatus.ORCHESTRATOR_WAITING_PROC.value, + 'updated_at': self._timestamp(), + 'detail': build_import_detail( + workflow_id=input_data['workflow_id'], + step=input_data['step'], + source=input_data.get('source'), + ), + 'import_run_id': input_data['import_run_id'], + }, + ) + return {'recorded': affected > 0, 'rows': affected} + + @activity.defn(name='record_import_names') + async def record_import_names(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Write the experiment and run names, before anything exists in MLflow. + + Both columns exist so a person watching the import list sees *which* model is being + imported. The earliest moment both values are known is when the bundle is open, so that is + when they are written — before the experiment exists, before the run exists. + + Args (input_data): + metadata (dict): workflow metadata. + import_run_id (int): the record's primary key. + workflow_id (str): recorded in the detail. + experiment_name (str): the bundle's own experiment name, verbatim. + run_name (str): the derived run name, which the run is then created with. + source (dict, optional): the bundle's declared origin. + """ + affected = self._execute_status_write( + _NAMES_SQL, + { + 'experiment_name': input_data['experiment_name'], + 'run_name': input_data['run_name'], + 'updated_at': self._timestamp(), + 'detail': build_import_detail( + workflow_id=input_data['workflow_id'], + step=input_data.get('step', ImportStep.EXPERIMENT_CREATION), + source=input_data.get('source'), + ), + 'import_run_id': input_data['import_run_id'], + }, + ) + return {'recorded': affected > 0, 'rows': affected} + + # ----------------------------------------------------------- class C: terminal + + @activity.defn(name='record_import_terminal_status') + async def record_import_terminal_status(self, input_data: dict[str, Any]) -> dict[str, Any]: + """The single authoritative write, run from the workflow's `finally` after cleanup. + + Every failure is classified **once**, here, from the step that failed. On success the reason + column is written as `NULL` — never `''`, which its length-3 CHECK would refuse — and the + detail carries no code. A cleanup failure is noted in the detail without changing a + successful status: the user's model was imported, and reporting ERROR would be false. + + Args (input_data): + metadata (dict): workflow metadata. + import_run_id (int): the record's primary key. + workflow_id (str): recorded in the detail. + succeeded (bool): whether every step succeeded. + step (str, optional): the failing step, or the last step reached on success. + gate (int, optional): the gate that rejected the bundle, when one did. + source (dict, optional): the bundle's declared origin. + cleanup_failed (bool, optional): whether the temporary-file cleanup failed. + + Returns: + dict: the verdict that was written — status, code and reason — so a caller that has to + report a failure to record it still holds the verdict it could not write. + """ + metadata = input_data.get('metadata', {}) + succeeded = bool(input_data.get('succeeded')) + step = input_data.get('step') + cleanup_failed = bool(input_data.get('cleanup_failed')) + + if succeeded: + status = map_state_to_status(ImportState.COMPLETED) + code: ImportErrorCode | None = None + reason: str | None = None + else: + failure = classify_import_failure(ImportStep(step) if step else None) + status = map_state_to_status(ImportState.ERROR, step) + code, reason = failure.code, failure.reason + + verdict = { + 'status': status.value, + 'code': code.value if code else None, + 'reason': reason, + 'step': str(step) if step else None, + } + + self._execute_status_write( + _TERMINAL_SQL, + { + 'status': status.value, + 'error_message': reason, + 'updated_at': self._timestamp(), + 'detail': build_import_detail( + workflow_id=input_data['workflow_id'], + step=step, + code=code, + gate=input_data.get('gate'), + source=input_data.get('source'), + cleanup_failed=cleanup_failed, + ), + 'import_run_id': input_data['import_run_id'], + }, + ) + + if succeeded: + await self.emit_metric( + metric_object=metrics.MODEL_IMPORT_COMPLETED_COUNT, + tags=self.get_core_labels(metadata, operation_type=status.value), + ) + self.info(f'import {input_data["import_run_id"]} recorded as {status.value}', metadata) + return verdict + + @activity.defn(name='report_import_status_write_failure') + async def report_import_status_write_failure( + self, input_data: dict[str, Any] + ) -> dict[str, Any]: + """Report a status write that could not be made — the one failure with no row to land on. + + Called by the workflow when a progress or terminal write exhausted its attempts or was + refused. It carries the verdict it could not write, so the outcome survives in text even + when it cannot survive in the row, and it never turns a completed import into a failed one. + + Args (input_data): + metadata (dict): workflow metadata. + import_run_id (int): the record's primary key. + workflow_id (str): the identifier that correlates this line with the record. + detail (str): the underlying failure, as text — it crossed an activity boundary, so the + exception object itself is gone by now. + verdict (str, optional): the verdict the terminal write could not record. + """ + await report_import_failure( + self, + None, + None, + import_run_id=input_data['import_run_id'], + workflow_id=input_data['workflow_id'], + metadata=input_data.get('metadata', {}), + code=ImportErrorCode.STATUS_NOT_RECORDED, + marker='status_write', + verdict=input_data.get('verdict'), + detail_text=input_data.get('detail'), + ) + return {'reported': True} + + # ------------------------------------------------------------- bundle helpers + + def _bundle_reader(self) -> BundleReader: + """A reader carrying the configured ceilings.""" + return BundleReader( + BundleLimits( + max_object_bytes=self.import_config['max_object_bytes'], + max_entries=self.import_config['max_entries'], + max_uncompressed_bytes=self.import_config['max_uncompressed_bytes'], + max_compression_ratio=self.import_config['max_compression_ratio'], + object_prefix=self.import_config['prefix'], + ) + ) + + def _work_dir(self, workflow_id: str) -> Path: + """This import's own scratch directory. One per workflow, never shared.""" + return Path(self.import_config['work_dir']) / workflow_id + + def _effective_bucket(self, requested: str | None) -> str: + return requested or self.import_config['bucket'] + + # --------------------------------------------------------- download and open + + @activity.defn(name='download_import_bundle') + async def download_import_bundle(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Gate 1, then the download, then the mandatory digest comparison. + + The object's size and key shape are checked from `stat_object`, before its body is fetched. + The SHA-256 of what was downloaded is then compared against `expected_digest` — the value the + uploader computed over the bytes it uploaded — with **no path that skips the comparison**, + because a missing value was already rejected at `received`. A mismatch fails here, before the + header is parsed: it means the object behind that key is not the file the user chose. + + Args (input_data): + metadata (dict): workflow metadata. + workflow_id (str): names this import's scratch directory. + bucket (str, optional): the effective bucket; the configured one when absent. + object_key (str): the uploaded object's key. + expected_digest (str): the uploader's SHA-256, lowercase hex. + + Returns: + dict: `path`, `size` and `digest` of the downloaded file. + + Raises: + ImportGateRejectedError: gate 1 rejected the object, or the digests disagree. + """ + if self.import_minio_repository is None: + raise ValueError(_MISSING_MINIO) + + metadata = input_data.get('metadata', {}) + bucket = self._effective_bucket(input_data.get('bucket')) + object_key = input_data['object_key'] + expected_digest = input_data['expected_digest'] + reader = self._bundle_reader() + + client = self.import_minio_repository.minio_client + try: + stat = client.stat_object(bucket, object_key) + except Exception as error: + raise ImportGateRejectedError( + 'the uploaded object could not be read', step=ImportStep.DOWNLOAD, gate=1 + ) from error + + try: + reader.check_object_limits(size=stat.size or 0, key=object_key) + except BundleGateError as error: + raise ImportGateRejectedError(str(error), step=error.step, gate=error.gate) from error + + work_dir = self._work_dir(input_data['workflow_id']) + work_dir.mkdir(parents=True, exist_ok=True) + target = work_dir / 'bundle.sientia' + try: + client.fget_object(bucket, object_key, str(target)) + except Exception as error: + raise ImportGateRejectedError( + 'the uploaded object could not be downloaded', step=ImportStep.DOWNLOAD, gate=1 + ) from error + + digest = fmt.file_digest(target) + if digest != str(expected_digest).lower(): + target.unlink(missing_ok=True) + raise ImportGateRejectedError( + 'the downloaded object does not match the digest the uploader computed', + step=ImportStep.DOWNLOAD, + gate=1, + ) + + self.info(f'import bundle downloaded from {bucket} ({stat.size} bytes)', metadata) + return {'path': str(target), 'size': stat.size, 'digest': digest} + + @activity.defn(name='open_import_bundle') + async def open_import_bundle(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Open the envelope, then the bundle: gates 2 to 7, in the reader's own order. + + The password envelope is decrypted **here**, inside the activity that uses the password, so + no plaintext password is ever an activity input, an activity result or a workflow variable — + and therefore never lands in the Temporal event history. The recovered password is a local + value and nothing else. + + Args (input_data): + metadata (dict): workflow metadata. + workflow_id (str): names this import's scratch directory. + path (str): the downloaded bundle. + password_envelope (str): base64 `nonce || ciphertext || tag`, AES-256-GCM. + + Returns: + dict: `extracted_dir`, `metadata`, `parameters`, `digest` — nothing secret. + + Raises: + ImportGateRejectedError: with the step and the gate that rejected the bundle. + """ + metadata = input_data.get('metadata', {}) + work_dir = self._work_dir(input_data['workflow_id']) + extract_root = work_dir / 'extract' + + try: + key = load_envelope_key(self.import_config.get('password_key')) + password = decrypt_password_envelope(input_data['password_envelope'], key) + except EnvelopeError as error: + # A distinct failure from a wrong *bundle* password, and never confused with it: the + # envelope is the platform's own transport, and its message says so. Same step, because + # the step is where the import stopped, not who is at fault. + raise ImportGateRejectedError( + 'the import password envelope could not be opened', + step=ImportStep.DECRYPTION, + gate=3, + cause='envelope', + ) from error + + reader = self._bundle_reader() + try: + opened = reader.open(input_data['path'], password, extract_root, work_dir) + except BundleGateError as error: + raise ImportGateRejectedError(str(error), step=error.step, gate=error.gate) from error + finally: + del password + + await self.emit_metric( + metric_object=metrics.MODEL_IMPORT_KDF_DURATION, + tags=self.get_core_labels(metadata, operation_type=ImportStep.DECRYPTION.value), + method='observe', + value=opened.kdf_seconds, + ) + self.info(f'import bundle opened into {opened.extracted_dir}', metadata) + return { + 'extracted_dir': str(opened.extracted_dir), + 'metadata': opened.metadata, + 'parameters': opened.parameters, + 'digest': opened.digest, + } + + # ------------------------------------------------------------ MLflow provisioning + + @activity.defn(name='create_import_experiment') + async def create_import_experiment(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Get or create the experiment named by the bundle, keeping the origin's name. + + Args (input_data): + metadata (dict): workflow metadata. + experiment_name (str): the bundle's `metadata.experiment_name`. + + Returns: + dict: `experiment_id` and `experiment_name`. + """ + if self.import_mlflow_repository is None: + raise ValueError(_MISSING_MLFLOW) + + experiment_name = input_data['experiment_name'] + experiment = self.import_mlflow_repository.get_experiment( + experiment_name, create_if_not_exists=True + ) + self.info( + f'import experiment ready: {experiment_name} ({experiment.experiment_id})', + input_data.get('metadata', {}), + ) + return {'experiment_id': experiment.experiment_id, 'experiment_name': experiment_name} + + @activity.defn(name='upload_import_artifacts') + async def upload_import_artifacts(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Create a new run with the recorded name and log the bundle's tree and parameters. + + Always a new run — the importer never appends to an existing one. The run is created with + exactly the `run_name` already written on the record, passed in rather than recomputed, so + the two cannot diverge. The artifact tree is logged verbatim, keeping `prediction_model` and + `data_model` at the run's artifact root, which is what the legacy loader resolves. The + bundle's `parameters` are logged unchanged, and the origin's own identity is logged + alongside them under `source.*` — records of where the model came from, not links. + + Args (input_data): + metadata (dict): workflow metadata. + experiment_id (str): the experiment to create the run in. + run_name (str): the name already on the record. + extracted_dir (str): the opened bundle's directory. + parameters (dict): the bundle's `parameters` block, verbatim. + source (dict): the bundle's `metadata` block, logged as `source.*`. + + Returns: + dict: `run_id` and `run_name`. + """ + if self.import_mlflow_repository is None: + raise ValueError(_MISSING_MLFLOW) + + client = self.import_mlflow_repository.client + run = client.create_run( + experiment_id=input_data['experiment_id'], run_name=input_data['run_name'] + ) + run_id = run.info.run_id + + artifacts_dir = Path(input_data['extracted_dir']) / 'artifacts' + client.log_artifacts(run_id, str(artifacts_dir)) + + for name, value in (input_data.get('parameters') or {}).items(): + client.log_param(run_id, name, value) + for name, value in (input_data.get('source') or {}).items(): + client.log_param(run_id, f'source.{name}', value) + + client.set_terminated(run_id, 'FINISHED') + self.info(f'import artifacts uploaded to run {run_id}', input_data.get('metadata', {})) + return {'run_id': run_id, 'run_name': input_data['run_name']} + + @activity.defn(name='register_import_model_version') + async def register_import_model_version(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Register a version whose source the legacy loader can parse. No stage change. + + The source is `runs://prediction_model`, because `get_model_run_id` extracts the run + id as `source.split('/')[2]` — a `models:/m-*` source, what the new stack emits, breaks it. + The stage is left untouched: a `COMPLETED` import is a registered model, not a servable one, + and promoting it to `Production` is a manual operator step outside this workflow. + + Args (input_data): + metadata (dict): workflow metadata. + model_name (str): the bundle's `metadata.model_name`, verbatim. + run_id (str): the run this import created. + + Returns: + dict: `model_name`, `version` and `source`. + """ + if self.import_mlflow_repository is None: + raise ValueError(_MISSING_MLFLOW) + + client = self.import_mlflow_repository.client + model_name = input_data['model_name'] + run_id = input_data['run_id'] + + try: + client.get_registered_model(model_name) + except Exception: + # First import of this name: the registered model is created on demand. A second import + # lands as a new version of the same registered model, never a differently named one. + client.create_registered_model(model_name) + + source = f'runs:/{run_id}/prediction_model' + version = client.create_model_version(name=model_name, source=source, run_id=run_id) + self.info( + f'import registered {model_name} version {version.version}', + input_data.get('metadata', {}), + ) + return {'model_name': model_name, 'version': str(version.version), 'source': source} + + # --------------------------------------------------- the MongoDB model document + + @activity.defn(name='write_import_model_document') + async def write_import_model_document(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Write the document that *is* the model listing, inert, last. + + This is what the runtime reads to know how to load a model — `model_config.transform_flavor`, + `model_config.predict_flavor` and `model_config.target` — so without it an imported model is + invisible to every workflow. `id` continues the collection's own numbering: the largest + existing id **as an integer** plus one, never a count and never a lexicographic sort, which + would answer `"9" > "30"`. A name already in the collection is left exactly as it is: + `active`, `target` and the flavors of a model already on the platform are operator-owned. + + Args (input_data): + metadata (dict): workflow metadata. + model_name (str): the bundle's `metadata.model_name`, verbatim. + target (str): the bundle's `parameters.target_variable`. + + Returns: + dict: `written`, `id`, `skipped` and how many unusable ids were ignored. + """ + if self.mongo_repository is None: + raise ValueError(_MISSING_MONGO) + + metadata = input_data.get('metadata', {}) + collection = self.import_config['models_collection'] + model_name = input_data['model_name'] + + existing = await self.mongo_repository.find_one(collection, {'name': model_name}, metadata) + if existing is not None: + self.info( + f'model document for {model_name} already exists; leaving it untouched', metadata + ) + return {'written': False, 'skipped': True, 'id': existing.get('id')} + + next_id, ignored = await self._next_model_document_id(collection, metadata) + document = { + 'id': next_id, + 'name': model_name, + 'active': False, + 'model_config': { + 'transform_flavor': 'sklearn', + 'predict_flavor': 'joblib', + 'retention_minutes': 60, + 'target': input_data['target'], + }, + } + await self.mongo_repository.insert(collection, document, metadata) + self.info(f'model document written for {model_name} with id {next_id}', metadata) + return {'written': True, 'skipped': False, 'id': next_id, 'ignored_ids': ignored} + + async def _next_model_document_id( + self, collection: str, metadata: dict[str, Any] + ) -> tuple[str, int]: + """`max(int(id)) + 1` as a decimal string, tolerating ids that are not numbers.""" + if self.mongo_repository is None: + raise ValueError(_MISSING_MONGO) + raw_ids = await self.mongo_repository.distinct(collection, 'id', {}, metadata) + numeric: list[int] = [] + ignored = 0 + for value in raw_ids: + try: + numeric.append(int(str(value))) + except (TypeError, ValueError): + # A document whose id is not a decimal number cannot take part in the numbering. + # Ignored rather than fatal, and counted so the log says how many there were. + ignored += 1 + if ignored: + self.info(f'ignored {ignored} non-numeric model document ids', metadata) + return str(max(numeric, default=0) + 1), ignored + + # ----------------------------------------------------------- cleanup and retention + + @activity.defn(name='cleanup_import_files') + async def cleanup_import_files(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Remove this import's scratch directory. Never the uploaded object. + + Runs on every path, success or failure, and **never raises**: a cleanup failure is reported + through the channel and noted in the record's detail, but the user's model was imported and + reporting an error would be false. + + Args (input_data): + metadata (dict): workflow metadata. + workflow_id (str): names the directory to remove. + import_run_id (int): for correlation when the removal fails. + + Returns: + dict: `cleaned` and, when it failed, the reason as text. + """ + metadata = input_data.get('metadata', {}) + work_dir = self._work_dir(input_data['workflow_id']) + try: + shutil.rmtree(work_dir, ignore_errors=False) + except FileNotFoundError: + # Nothing was downloaded, or a gate removed its own partial tree. Not a failure. + return {'cleaned': True, 'existed': False} + except OSError as error: + await report_import_failure( + self, + ImportStep.CLEANUP, + error, + import_run_id=input_data.get('import_run_id'), + workflow_id=input_data['workflow_id'], + metadata=metadata, + code=ImportErrorCode.CLEANUP_INCOMPLETE, + marker=ImportStep.CLEANUP.value, + ) + return {'cleaned': False, 'existed': True, 'reason': type(error).__name__} + return {'cleaned': True, 'existed': True} + + @activity.defn(name='ensure_import_bundle_retention') + async def ensure_import_bundle_retention(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Ensure the 7-day expiry rule on the import prefix, idempotently. + + Applied by the importer rather than assumed from a Helm value, so the acceptance criterion + ("lifecycle active") is verifiable from the code that depends on it. The uploaded object is + never deleted by the import itself — it is left to this rule, as the record of an attempt. + + Args (input_data): + metadata (dict): workflow metadata. + bucket (str, optional): the effective bucket; the configured one when absent. + + Returns: + dict: whether the rule was already there and what it expires. + """ + if self.import_minio_repository is None: + raise ValueError(_MISSING_MINIO) + + metadata = input_data.get('metadata', {}) + bucket = self._effective_bucket(input_data.get('bucket')) + prefix = self.import_config['prefix'] + days = self.import_config['retention_days'] + rule_id = 'sientia-import-bundle-expiry' + client = self.import_minio_repository.minio_client + + try: + existing = client.get_bucket_lifecycle(bucket) + except Exception: + # A bucket with no lifecycle configuration at all answers with an error on some + # deployments and with `None` on others; both mean "there is no rule yet". + existing = None + + rules = [rule for rule in (existing.rules if existing else []) if rule.rule_id != rule_id] + rules.append( + Rule( + ENABLED, + rule_filter=Filter(prefix=prefix), + rule_id=rule_id, + expiration=Expiration(days=days), + ) + ) + client.set_bucket_lifecycle(bucket, LifecycleConfig(rules)) + self.info(f'import retention rule ensured on {bucket} ({days} days)', metadata) + return {'bucket': bucket, 'prefix': prefix, 'days': days, 'rules': len(rules)} diff --git a/laborious/activities/model_import_errors.py b/laborious/activities/model_import_errors.py new file mode 100644 index 0000000..657f47f --- /dev/null +++ b/laborious/activities/model_import_errors.py @@ -0,0 +1,348 @@ +"""The import's failure vocabulary: a closed catalog, typed errors and one reporting channel. + +Three things live here and nowhere else: + +- **the catalog** — one stable `error_code` and one plain-language `error_reason` per pipeline step. + The reason is written for a person who does not know this system exists: what happened, what state + the model is in, what to do next. It never carries a path, a table name, an object key, a Python + exception class or a traceback, and it is never built from `str(exception)`. +- **the typed errors** — the two failures that must not be retried, because repeating them cannot + change the answer: a record this workflow does not own, and a statement the database refuses. +- **the channel** — `report_import_failure`, the single funnel every failure passes through. It is + the only place in the import path that logs an error or sends a notification, which a test + asserts, so no failure is reported twice or half-reported. +""" + +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + import traceback + from collections.abc import Mapping + from dataclasses import dataclass + from enum import StrEnum + from types import MappingProxyType + from typing import Any, Final, Protocol + + from temporalio.exceptions import ApplicationError + + from laborious import metrics + from laborious.utils.bundle.steps import ImportStep + + +class ImportErrorCode(StrEnum): + """The machine-readable half of a failure: the frontend's i18n key and support's triage key. + + Written under `import.code` in the record's JSON detail — not in a column of its own, so the + import adds nothing to a table it shares with the training flow. + """ + + REQUEST_INVALID = 'IMPORT_REQUEST_INVALID' + FILE_UNREADABLE = 'IMPORT_FILE_UNREADABLE' + BUNDLE_CANNOT_BE_OPENED = 'IMPORT_BUNDLE_CANNOT_BE_OPENED' + BUNDLE_REJECTED = 'IMPORT_BUNDLE_REJECTED' + BUNDLE_UNPACK_FAILED = 'IMPORT_BUNDLE_UNPACK_FAILED' + BUNDLE_INCOMPLETE = 'IMPORT_BUNDLE_INCOMPLETE' + BUNDLE_UNEXPECTED_CONTENT = 'IMPORT_BUNDLE_UNEXPECTED_CONTENT' + STORAGE_PREPARATION_FAILED = 'IMPORT_STORAGE_PREPARATION_FAILED' + MODEL_FILES_NOT_STORED = 'IMPORT_MODEL_FILES_NOT_STORED' + MODEL_NOT_REGISTERED = 'IMPORT_MODEL_NOT_REGISTERED' + MODEL_SETTINGS_NOT_SAVED = 'IMPORT_MODEL_SETTINGS_NOT_SAVED' + UNEXPECTED_ERROR = 'IMPORT_UNEXPECTED_ERROR' + # Codes that never reach a record's `error_message`, and are therefore absent from the catalog: + # one names the failure to own a row, one the failure to write one, one the temporary files left + # on the worker after an import that otherwise succeeded. All three live on the channel — log, + # metric, notification, Temporal error — and the third is additionally flagged in the record's + # detail as `cleanup_failed`, without a code and without changing the status. + LOG_ROW_NOT_CLAIMABLE = 'IMPORT_LOG_ROW_NOT_CLAIMABLE' + STATUS_NOT_RECORDED = 'IMPORT_STATUS_NOT_RECORDED' + CLEANUP_INCOMPLETE = 'IMPORT_CLEANUP_INCOMPLETE' + + +@dataclass(frozen=True) +class ImportFailure: + """One classification of one failure: the step, its code and the sentence it stores.""" + + step: ImportStep | None + code: ImportErrorCode + reason: str + + +_CONTACT_SUPPORT = "Contact support with this import's identifier." +_RE_EXPORT = 'Export the model again from the origin platform.' + +IMPORT_FAILURE_CATALOG: Final[Mapping[ImportStep, tuple[ImportErrorCode, str]]] = MappingProxyType( + { + ImportStep.RECEIVED: ( + ImportErrorCode.REQUEST_INVALID, + 'The import could not be started because the request was incomplete or ' + 'contradictory. Nothing was created. Submit the import again.', + ), + ImportStep.DOWNLOAD: ( + ImportErrorCode.FILE_UNREADABLE, + 'The uploaded file could not be read, or it is not the file that was sent. ' + 'Upload it again.', + ), + ImportStep.DECRYPTION: ( + ImportErrorCode.BUNDLE_CANNOT_BE_OPENED, + 'This file could not be opened: the password is wrong, or the file is damaged or ' + 'was altered. Confirm the password with whoever exported the model, then try ' + 'again.', + ), + ImportStep.ARCHIVE_INSPECTION: ( + ImportErrorCode.BUNDLE_REJECTED, + "The file's contents did not pass the platform's safety checks, so it was not " + f'opened. {_RE_EXPORT}', + ), + ImportStep.EXTRACTION: ( + ImportErrorCode.BUNDLE_UNPACK_FAILED, + f'The file could not be unpacked safely and nothing from it was kept. {_RE_EXPORT}', + ), + ImportStep.STRUCTURE_VALIDATION: ( + ImportErrorCode.BUNDLE_INCOMPLETE, + 'This file is not a complete model export — part of what the platform needs is ' + f'missing from it. {_RE_EXPORT}', + ), + ImportStep.CONTENT_POLICY: ( + ImportErrorCode.BUNDLE_UNEXPECTED_CONTENT, + 'The file contains something a model export should not contain, so it was ' + f'rejected. {_RE_EXPORT}', + ), + ImportStep.EXPERIMENT_CREATION: ( + ImportErrorCode.STORAGE_PREPARATION_FAILED, + 'The platform could not prepare a place to keep this model. The import stopped ' + f'and no model was created. {_CONTACT_SUPPORT}', + ), + ImportStep.ARTIFACT_UPLOAD: ( + ImportErrorCode.MODEL_FILES_NOT_STORED, + "The model's files could not be stored on this platform. The import stopped and " + f'the model is not available. {_CONTACT_SUPPORT}', + ), + ImportStep.REGISTRATION: ( + ImportErrorCode.MODEL_NOT_REGISTERED, + "The model's files were stored, but the model itself could not be registered, so " + f'it cannot be used. {_CONTACT_SUPPORT}', + ), + ImportStep.MODEL_DOCUMENT: ( + ImportErrorCode.MODEL_SETTINGS_NOT_SAVED, + 'The model was registered, but the settings that tell the platform how to run it ' + f'could not be saved. {_CONTACT_SUPPORT}', + ), + } +) + +UNCLASSIFIED_FAILURE: Final[tuple[ImportErrorCode, str]] = ( + ImportErrorCode.UNEXPECTED_ERROR, + f'The import stopped for an unexpected reason and no model was created. {_CONTACT_SUPPORT}', +) + +# `cleanup` deletes temporary files on the worker. It is never the reported failure: when every +# other step succeeded and only cleanup failed, the import is a success and the leak is a log line. +# So it deliberately has no user-facing sentence, and the catalog-completeness test knows it. +STEPS_WITHOUT_A_USER_FACING_REASON: Final[frozenset[ImportStep]] = frozenset({ImportStep.CLEANUP}) + + +def classify_import_failure( + step: ImportStep | None, exception: BaseException | None = None +) -> ImportFailure: + """Turn a failure into the one code and the one sentence that will be recorded. + + The exception's text is **ignored entirely** — it is what carries paths, table names and type + names, and it belongs in the log. What decides the outcome is the step. + + Args: + step: the step that was being attempted, or `None` when none had been entered. + exception: the failure, when there is one to hand. It is accepted so callers read as + "classify this failure" and never absent because the classification needs it: the + catalog is keyed by step, and an exception cannot cross an activity boundary with its + type intact anyway — the terminal write classifies from the step alone. + + Returns: + ImportFailure: step, code and the reason to store. + """ + del exception # the catalog is keyed by step; the exception's text never reaches the record + if step is None or step not in IMPORT_FAILURE_CATALOG: + code, reason = UNCLASSIFIED_FAILURE + return ImportFailure(step=step, code=code, reason=reason) + code, reason = IMPORT_FAILURE_CATALOG[step] + return ImportFailure(step=step, code=code, reason=reason) + + +class ImportLogRowNotClaimableError(ApplicationError): + """The record named by `import_run_id` is not this workflow's to write. + + Raised when the claim affects zero rows: no such row, a `TRAIN` row, or a row already running + under another workflow. Non-retryable on purpose — a second attempt would read the same rows and + reach the same conclusion — and loud everywhere except the row itself, which belongs to someone + else. + """ + + def __init__(self, message: str, *, import_run_id: Any = None, observed: str = '') -> None: + super().__init__( + message, + type='ImportLogRowNotClaimableError', + non_retryable=True, + ) + self.import_run_id = import_run_id + self.observed = observed + + +class ImportStatusRejectedError(ApplicationError): + """The database accepted the connection and refused the statement. + + A `CHECK` violation, an undefined column, a value too long for its column. `V12` not being + applied yet lands here, at the terminal write, and it must fail on the first attempt instead of + spending the whole retry envelope to say so. + """ + + def __init__(self, message: str) -> None: + super().__init__(message, type='ImportStatusRejectedError', non_retryable=True) + + +class ImportInputError(ApplicationError): + """The request is incomplete or contradictory. Fails at `received`, on the claimed row.""" + + def __init__(self, message: str) -> None: + super().__init__( + message, + {'step': ImportStep.RECEIVED.value, 'gate': None}, + type='ImportInputError', + non_retryable=True, + ) + + +class ImportGateRejectedError(ApplicationError): + """A gate rejected the bundle, and the error says which one. + + Gates 2 to 7 all run inside one activity, so the step the workflow was *attempting* is not + precise enough to record. An exception loses its Python type crossing an activity boundary but + keeps its `details`, so the step and the gate travel there and the terminal write can name them. + """ + + def __init__( + self, + message: str, + *, + step: ImportStep, + gate: int | None = None, + cause: str | None = None, + ) -> None: + super().__init__( + message, + {'step': step.value, 'gate': gate, 'cause': cause}, + type='ImportGateRejectedError', + non_retryable=True, + ) + self.step = step + self.gate = gate + + +def failure_location(exception: BaseException | None) -> tuple[str | None, int | None]: + """Read the step and gate an activity's failure carries, if it carried any. + + Walks the `__cause__` chain because Temporal wraps an activity failure in an `ActivityError`. + Returns `(None, None)` when the failure did not say where it happened, and the caller then falls + back to the step it was attempting. + """ + seen = 0 + candidate: BaseException | None = exception + while candidate is not None and seen < 10: + details = getattr(candidate, 'details', None) + if details and isinstance(details[0], dict) and 'step' in details[0]: + location = details[0] + gate = location.get('gate') + return location.get('step'), int(gate) if isinstance(gate, int) else None + candidate = candidate.__cause__ + seen += 1 + return None, None + + +class FailureSinks(Protocol): + """What the channel needs from its caller: the repository's existing observability surface.""" + + def error(self, message: str, metadata: dict | None = None) -> None: ... + + def get_core_labels(self, metadata: dict | None = None, operation_type: str = '-') -> dict: ... + + async def emit_metric( + self, + metric_object: Any, + tags: dict[str, Any], + method: str = 'inc', + value: float | None = None, + ) -> None: ... + + async def send_notification_async( + self, + metadata: dict, + notification_id: str, + message: str, + block: str, + level: str = 'INFO', + attachment_content: str | None = None, + ) -> None: ... + + +async def report_import_failure( + sinks: FailureSinks, + step: ImportStep | None, + exception: BaseException | None, + *, + import_run_id: Any, + workflow_id: str, + metadata: dict[str, Any] | None = None, + code: ImportErrorCode | None = None, + marker: str | None = None, + verdict: str | None = None, + detail_text: str | None = None, +) -> ImportFailure: + """Report one failure through every sink, exactly once. + + Sinks, in order: a structured log carrying the ids, the step, the code **and** the exception + with its traceback (the only sink that gets the exception text); the import error metric, with + the code in `operation_type`; a notification through the existing handler. The exception itself + reaches Temporal by being raised, and the record is the fifth, optional sink — the terminal + write, when the row can be written at all. + + Args: + sinks: the activity providing logger, metrics and notifications. + step: the pipeline step, or `None` for a failure that is not one (see `marker`). + exception: the failure. + import_run_id: the record's id, for correlation. + workflow_id: the identifier a user quotes to support. + metadata: workflow metadata for labels and the notification. + code: an explicit code, for the two failures that are not pipeline steps. + marker: `not_claimable` or `status_write` — what a step name would have been. + verdict: for a failed terminal write, the verdict it could not record, so the outcome + survives in text even when it cannot survive in the row. + detail_text: the failure as text, for the callers whose exception object is already gone — + anything that crossed an activity boundary. Used in place of a traceback. + + Returns: + ImportFailure: the classification that was reported. + """ + failure = classify_import_failure(step, exception) + if code is not None: + failure = ImportFailure(step=step, code=code, reason=failure.reason) + + where = marker or (step.value if step else 'unknown') + if exception is not None: + detail = ''.join(traceback.format_exception(exception)).strip() + else: + detail = detail_text or 'no exception object available' + verdict_line = f' verdict_not_recorded={verdict}' if verdict else '' + sinks.error( + f'model import failed: step={where} code={failure.code.value} ' + f'import_run_id={import_run_id} workflow_id={workflow_id}{verdict_line}\n{detail}', + metadata or {}, + ) + + labels = sinks.get_core_labels(metadata or {}, operation_type=failure.code.value) + await sinks.emit_metric(metric_object=metrics.MODEL_IMPORT_ERROR_COUNT, tags=labels) + await sinks.send_notification_async( + metadata=metadata or {}, + notification_id=failure.code.value, + message=failure.reason, + block=where, + level='ERROR', + ) + return failure diff --git a/laborious/metrics.py b/laborious/metrics.py index 74a9189..3f4de2c 100644 --- a/laborious/metrics.py +++ b/laborious/metrics.py @@ -183,6 +183,45 @@ MODEL_ANALYZE_ERROR_COUNT = Counter( ) +# ================== Model import metrics ================== +# +# The failing step and the error code travel in `operation_type`, the one label the shared +# `CORE_LABELS` leaves free. Nothing else about a failure is a label: an error reason is a sentence +# and an exception is a log line, neither of which belongs in a metric's cardinality. + +MODEL_IMPORT_STARTED_COUNT = Counter( + 'laborious_model_import_started_count', + 'Number of model imports that claimed their import log row', + CORE_LABELS, +) + +MODEL_IMPORT_COMPLETED_COUNT = Counter( + 'laborious_model_import_completed_count', + 'Number of model imports that reached a registered version and a model document', + CORE_LABELS, +) + +MODEL_IMPORT_ERROR_COUNT = Counter( + 'laborious_model_import_error_count', + 'Number of failed model imports, with the failing step or error code in operation_type', + CORE_LABELS, +) + +MODEL_IMPORT_KDF_DURATION = Histogram( + 'laborious_model_import_kdf_duration_seconds', + 'Time spent deriving the bundle key with Argon2id', + CORE_LABELS, + buckets=[0.5, 1.0, 2.0, 3.0, 5.0, 8.0, 13.0, 21.0], +) + +MODEL_IMPORT_STEP_DURATION = Histogram( + 'laborious_model_import_step_duration_seconds', + 'Duration of each model import step, named in operation_type', + CORE_LABELS, + buckets=[0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0], +) + + # ================== PI Web API metrics ================== PI_WEB_API_LABELS = [*CORE_LABELS, 'tag_name'] diff --git a/laborious/utils/bundle/__init__.py b/laborious/utils/bundle/__init__.py new file mode 100644 index 0000000..77e7945 --- /dev/null +++ b/laborious/utils/bundle/__init__.py @@ -0,0 +1,27 @@ +"""Reading `.sientia` bundles: the wire format, the seven gates and the step vocabulary. + +There is no writer here. The producer is the Streamlit app in `sientia-projects-templates` +(`app/src/operations/model_export/`), and a second production writer for a format with one producer +would be a fork waiting to happen. What stands in for it in the tests is +`tests/helpers/bundle_factory.py` plus the golden fixtures under `tests/fixtures/bundle/`. +""" + +from laborious.utils.bundle.format import BundleFormatError, BundleHeader +from laborious.utils.bundle.reader import ( + BundleGateError, + BundleLimits, + BundleReader, + OpenedBundle, +) +from laborious.utils.bundle.steps import GATE_TO_STEP, ImportStep + +__all__ = [ + 'GATE_TO_STEP', + 'BundleFormatError', + 'BundleGateError', + 'BundleHeader', + 'BundleLimits', + 'BundleReader', + 'ImportStep', + 'OpenedBundle', +] diff --git a/laborious/utils/bundle/format.py b/laborious/utils/bundle/format.py new file mode 100644 index 0000000..f07d2f8 --- /dev/null +++ b/laborious/utils/bundle/format.py @@ -0,0 +1,216 @@ +"""The `.sientia` wire format, as the producer writes it. + +Every constant here is copied from the producer that emits the bundles this reader opens — the +Streamlit app in `sientia-projects-templates`, modules +`app/src/operations/model_export/crypto.py` and `app/src/operations/model_export/bundle.py`. The +producer file each value came from is named next to it. The golden fixtures under +`tests/fixtures/bundle/` are the anti-drift device: if the producer's layout moves, the tests here +fail against the committed bytes instead of failing in production. + +This module deliberately imports nothing from Laborious, Temporal, MLflow or libsodium: it is pure +`struct` and `hashlib` over bytes, so it stays liftable into a shared package if the format ever +gets one (the cancelled QTZPOC-15). `reader.py` is where the crypto lives. +""" + +from __future__ import annotations + +import hashlib +import struct +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +# --- Producer: app/src/operations/model_export/crypto.py --- +MAGIC: Final[bytes] = b'SIENTIA1' +FORMAT_VERSION: Final[int] = 1 +AEAD_ID_XCHACHA20POLY1305_SECRETSTREAM: Final[int] = 1 +KDF_ID_ARGON2ID: Final[int] = 1 +SALT_SIZE: Final[int] = 16 +DIGEST_SIZE: Final[int] = 32 +RESERVED_SIGNATURE_SIZE: Final[int] = 64 +# `KDF_MEMLIMIT_BYTES` / `KDF_OPSLIMIT` on the producer: what it writes is also the ceiling the +# reader refuses to exceed, so header bytes cannot buy an arbitrary allocation on the worker. +MAX_KDF_MEMLIMIT: Final[int] = 256 * 1024 * 1024 # 268435456 +MAX_KDF_OPSLIMIT: Final[int] = 3 +# `KDF_PARALLELISM_RESERVED`: libsodium's `crypto_pwhash` has no lane parameter, so the field is +# reserved and must be exactly 1. +PARALLELISM: Final[int] = 1 +CHUNK_SIZE: Final[int] = 64 * 1024 # 65536, fixed by the format +KEY_SIZE: Final[int] = 32 # crypto_secretstream_xchacha20poly1305_KEYBYTES + +# magic(8s) version(B) aead_id(B) kdf_id(B) salt(16s) memlimit(Q) opslimit(Q) +# parallelism(B) digest(32s) reserved_signature(64s) — producer's `_HEADER_STRUCT`. +HEADER_STRUCT: Final[struct.Struct] = struct.Struct('>8sBBB16sQQB32s64s') +HEADER_SIZE: Final[int] = HEADER_STRUCT.size # 140 + +# --- libsodium constants the framing depends on --- +# Not producer constants, but constants of the primitive the producer uses. Kept as literals so +# this module stays import-free; `tests/laborious/utils/bundle/test_format.py` asserts they equal +# the values `nacl.bindings` reports, which is what makes the literals safe. +STREAM_HEADER_SIZE: Final[int] = 24 # crypto_secretstream_xchacha20poly1305_HEADERBYTES +ABYTES: Final[int] = 17 # crypto_secretstream_xchacha20poly1305_ABYTES + +# The pre-upload check prefix of `08-security-and-encryption.md` § 5.3: enough bytes to hold the +# header, the stream header and one full chunk. Derived from constants, never read from the file. +PREFIX_SIZE: Final[int] = HEADER_SIZE + STREAM_HEADER_SIZE + CHUNK_SIZE + ABYTES # 65717 + +# The digest covers the chunk stream only. The producer computes `sha256(ciphertext)` over the +# joined chunks and *then* writes `header + stream_header + ciphertext`, so on disk the covered +# range starts after both headers. Verified against a bundle produced by the producer's own code: +# `sha256(file[164:])` matches the header field and `sha256(file[140:])` does not. +DIGEST_OFFSET: Final[int] = HEADER_SIZE + STREAM_HEADER_SIZE # 164 + +MIN_FILE_SIZE: Final[int] = HEADER_SIZE + STREAM_HEADER_SIZE + +SUPPORTED_FORMAT_VERSIONS: Final[frozenset[int]] = frozenset({FORMAT_VERSION}) +SUPPORTED_AEAD_IDS: Final[frozenset[int]] = frozenset({AEAD_ID_XCHACHA20POLY1305_SECRETSTREAM}) +SUPPORTED_KDF_IDS: Final[frozenset[int]] = frozenset({KDF_ID_ARGON2ID}) + +_HASH_BLOCK_SIZE: Final[int] = 1024 * 1024 + + +class BundleFormatError(Exception): + """The bytes are not a `.sientia` bundle this reader implements. + + Raised for every header-level rejection: wrong magic, unknown version, unknown primitive ids, + cost parameters above the ceiling, a reserved field carrying a value, a file too short to hold + a header, or a chunk stream whose digest does not match the header's. `reader.py` turns this + into a gate-2 rejection; nothing here knows about gates or import steps. + """ + + +@dataclass(frozen=True) +class BundleHeader: + """The 140-byte plaintext header, parsed and validated.""" + + format_version: int + aead_id: int + kdf_id: int + salt: bytes + kdf_memlimit_bytes: int + kdf_opslimit: int + kdf_parallelism: int + ciphertext_digest: bytes + reserved_signature: bytes + + +def parse_header(raw: bytes) -> BundleHeader: + """Parse and validate the plaintext header. + + Everything the reader trusts about the file's shape is decided here, before any offset past + the header is used and before a key is derived. + + Args: + raw: at least `HEADER_SIZE` bytes read from the start of the object. + + Returns: + BundleHeader: the validated header. + + Raises: + BundleFormatError: on any invalid or unsupported field. + """ + if len(raw) < HEADER_SIZE: + raise BundleFormatError( + f'file is shorter than the {HEADER_SIZE}-byte header ({len(raw)} bytes available)' + ) + + ( + magic, + format_version, + aead_id, + kdf_id, + salt, + kdf_memlimit_bytes, + kdf_opslimit, + kdf_parallelism, + ciphertext_digest, + reserved_signature, + ) = HEADER_STRUCT.unpack(raw[:HEADER_SIZE]) + + if magic != MAGIC: + raise BundleFormatError('file does not start with the expected bundle marker') + if format_version not in SUPPORTED_FORMAT_VERSIONS: + raise BundleFormatError(f'unsupported bundle format version: {format_version}') + if aead_id not in SUPPORTED_AEAD_IDS: + raise BundleFormatError(f'unsupported encryption identifier: {aead_id}') + if kdf_id not in SUPPORTED_KDF_IDS: + raise BundleFormatError(f'unsupported key derivation identifier: {kdf_id}') + if kdf_memlimit_bytes > MAX_KDF_MEMLIMIT: + raise BundleFormatError( + f'declared key derivation memory {kdf_memlimit_bytes} exceeds the ' + f'{MAX_KDF_MEMLIMIT} ceiling' + ) + if kdf_opslimit > MAX_KDF_OPSLIMIT: + raise BundleFormatError( + f'declared key derivation passes {kdf_opslimit} exceeds the {MAX_KDF_OPSLIMIT} ceiling' + ) + if kdf_memlimit_bytes <= 0 or kdf_opslimit <= 0: + raise BundleFormatError('key derivation parameters must be positive') + if kdf_parallelism != PARALLELISM: + raise BundleFormatError( + f'reserved parallelism field must be {PARALLELISM}, found {kdf_parallelism}' + ) + if reserved_signature != bytes(RESERVED_SIGNATURE_SIZE): + raise BundleFormatError('reserved signature field must be empty in this format version') + + return BundleHeader( + format_version=format_version, + aead_id=aead_id, + kdf_id=kdf_id, + salt=salt, + kdf_memlimit_bytes=kdf_memlimit_bytes, + kdf_opslimit=kdf_opslimit, + kdf_parallelism=kdf_parallelism, + ciphertext_digest=ciphertext_digest, + reserved_signature=reserved_signature, + ) + + +def read_header(path: Path | str) -> BundleHeader: + """Read and validate the header of the file at `path`. + + Raises: + BundleFormatError: when the file cannot hold a header plus a stream header, or when the + header itself is invalid. + """ + file_path = Path(path) + size = file_path.stat().st_size + if size < MIN_FILE_SIZE: + raise BundleFormatError( + f'file is {size} bytes, smaller than the {MIN_FILE_SIZE} bytes a bundle needs for its ' + 'header and encrypted stream header' + ) + with file_path.open('rb') as handle: + return parse_header(handle.read(HEADER_SIZE)) + + +def compute_ciphertext_digest(path: Path | str) -> bytes: + """SHA-256 of the chunk stream — the bytes from `DIGEST_OFFSET` to the end of the file.""" + digest = hashlib.sha256() + with Path(path).open('rb') as handle: + handle.seek(DIGEST_OFFSET) + while block := handle.read(_HASH_BLOCK_SIZE): + digest.update(block) + return digest.digest() + + +def verify_ciphertext_digest(path: Path | str, header: BundleHeader) -> None: + """Compare the file's chunk-stream digest against the header's. + + Raises: + BundleFormatError: when they differ — a truncated, overwritten or corrupted transfer. This + runs before any key derivation, so a damaged file never costs an Argon2id pass. + """ + if compute_ciphertext_digest(path) != header.ciphertext_digest: + raise BundleFormatError( + 'the encrypted content does not match the digest recorded in the bundle header' + ) + + +def file_digest(path: Path | str) -> str: + """Hex SHA-256 of the whole file, for the `expected_digest` comparison of gate 1.""" + digest = hashlib.sha256() + with Path(path).open('rb') as handle: + while block := handle.read(_HASH_BLOCK_SIZE): + digest.update(block) + return digest.hexdigest() diff --git a/laborious/utils/bundle/reader.py b/laborious/utils/bundle/reader.py new file mode 100644 index 0000000..61fcd90 --- /dev/null +++ b/laborious/utils/bundle/reader.py @@ -0,0 +1,473 @@ +"""Reading a `.sientia` bundle: the seven gates, in order, in one function. + +`BundleReader.open()` is the only entry point. Gate ordering is a property of that function rather +than of the caller, so it cannot be reordered by accident and can be unit-tested without Temporal: + +1. object-level limits (`check_object_limits`, called by the download activity before the body is + fetched — the only gate that does not need the file) +2. header validation, including the chunk-stream digest, before any key derivation +3. AEAD decryption — the integrity check; a wrong password and a tampered file fail identically +4. archive inspection over the central directory, writing nothing to disk +5. extraction into a fresh, isolated directory +6. structure and schema of the extracted tree +7. content policy over the file set + +Nothing here logs, notifies or emits a metric: rejections are raised as `BundleGateError` and the +activity layer owns the one failure channel. +""" + +from __future__ import annotations + +import json +import os +import shutil +import stat +import time +import unicodedata +import zipfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import nacl.bindings as sodium +import nacl.exceptions + +from laborious.utils.bundle import format as fmt +from laborious.utils.bundle.steps import GATE_TO_STEP, ImportStep + +# Gate 6 — the layout `bundle.build_zip` writes and `model_repository.py` reads back. +METADATA_FILE_NAME = 'metadata.json' +ARTIFACTS_DIR_NAME = 'artifacts' +REQUIRED_TOP_LEVEL_KEYS = frozenset({'parameters', 'metadata'}) +REQUIRED_METADATA_FIELDS = ( + 'model_name', + 'experiment_name', + 'run_id', + 'model_version', + 'model_project', + 'export_timestamp', +) +REQUIRED_PARAMETERS = ('target_variable',) +REQUIRED_MODEL_DIRS = ('prediction_model', 'data_model') +REQUIRED_MODEL_FILES = ('MLmodel', 'model.pkl') + +# The import record's `experiment_name` column is `VARCHAR(50)` with a length-3 floor, and the name +# is written onto the record before provisioning starts — so the column's limits are enforced here, +# not three activities later as a constraint violation. +EXPERIMENT_NAME_MIN_LENGTH = 3 +EXPERIMENT_NAME_MAX_LENGTH = 50 + +# Gate 7 — an MLflow 2.x artifact tree for this platform: the model directories' known files, the +# model card, the CSVs the training template logs and the bundle's own metadata. +ALLOWED_FILE_NAMES = frozenset( + { + 'MLmodel', + 'metadata.json', + 'model.pkl', + 'conda.yaml', + 'python_env.yaml', + 'requirements.txt', + 'model_card.json', + 'model_card.svg', + } +) +ALLOWED_FILE_SUFFIXES = frozenset({'.csv', '.json', '.yaml', '.yml', '.txt', '.pkl', '.svg', '.md'}) + +_DECRYPT_FAILURE_MESSAGE = ( + 'could not open this bundle: wrong password, or the file is damaged or was altered' +) + + +class BundleGateError(Exception): + """A gate rejected the bundle. + + Carries the gate number and the import step that gate maps onto, so the workflow never has to + guess which step to record. + """ + + def __init__(self, message: str, *, gate: int, step: ImportStep | None = None) -> None: + super().__init__(message) + self.gate = gate + self.step = step if step is not None else GATE_TO_STEP[gate] + + +@dataclass(frozen=True) +class BundleLimits: + """Ceilings enforced by gates 1 and 4. Values come from `build_import_config()`.""" + + max_object_bytes: int = 1024 * 1024 * 1024 + max_entries: int = 5000 + max_uncompressed_bytes: int = 4 * 1024 * 1024 * 1024 + max_compression_ratio: float = 200.0 + object_prefix: str = 'imported_models/' + object_suffix: str = '.sientia' + + +@dataclass(frozen=True) +class OpenedBundle: + """What a bundle yields once every gate has passed. Nothing secret is carried.""" + + extracted_dir: Path + metadata: dict[str, Any] = field(default_factory=dict) + parameters: dict[str, Any] = field(default_factory=dict) + digest: str = '' + header: fmt.BundleHeader | None = None + # How long Argon2id took, so the worker's most expensive step is measurable without the + # activity layer having to reach inside the reader. + kdf_seconds: float = 0.0 + + +class BundleReader: + """Opens `.sientia` bundles produced by the exporter in `sientia-projects-templates`.""" + + def __init__(self, limits: BundleLimits | None = None) -> None: + self.limits = limits or BundleLimits() + + # ------------------------------------------------------------------ gate 1 + + def check_object_limits(self, *, size: int, key: str) -> None: + """Gate 1, object level: reject before the object's body is fetched. + + Args: + size: the object's size as reported by `stat_object`. + key: the object key. + + Raises: + BundleGateError: oversized object, key outside the import prefix, or wrong suffix. + """ + if size > self.limits.max_object_bytes: + raise BundleGateError( + f'uploaded object is {size} bytes, above the ' + f'{self.limits.max_object_bytes} byte limit', + gate=1, + ) + if size <= 0: + raise BundleGateError('uploaded object is empty', gate=1) + if not key.startswith(self.limits.object_prefix): + raise BundleGateError('uploaded object is not under the import prefix', gate=1) + if not key.endswith(self.limits.object_suffix): + raise BundleGateError( + f'uploaded object does not end in {self.limits.object_suffix}', gate=1 + ) + + # --------------------------------------------------------------------- open + + def open( + self, + encrypted_path: Path | str, + password: str, + extract_root: Path | str, + work_dir: Path | str | None = None, + ) -> OpenedBundle: + """Run gates 2 to 7 in order and return the opened bundle. + + Args: + encrypted_path: the downloaded `.sientia` file. + password: the plaintext bundle password. Never stored, never logged, never returned. + extract_root: the directory to extract into. It MUST NOT exist yet (gate 5). + work_dir: where the decrypted zip is written. Defaults to `extract_root`'s parent. + + Returns: + OpenedBundle: extracted tree, `metadata`, `parameters`, digest and header. + + Raises: + BundleGateError: with the gate that rejected the bundle and its import step. + """ + encrypted = Path(encrypted_path) + root = Path(extract_root) + staging = Path(work_dir) if work_dir is not None else root.parent + + header = self._gate_2_header(encrypted) + zip_path, kdf_seconds = self._gate_3_decrypt(encrypted, password, header, staging) + try: + self._gate_4_inspect(zip_path) + self._gate_5_extract(zip_path, root) + metadata, parameters = self._gate_6_structure(root) + self._gate_7_content_policy(root) + except BaseException: + # A rejection after decryption leaves nothing extracted behind (gate 5's contract) and + # never leaves a decrypted zip on the worker for the next import to trip over. + shutil.rmtree(root, ignore_errors=True) + zip_path.unlink(missing_ok=True) + raise + zip_path.unlink(missing_ok=True) + + return OpenedBundle( + extracted_dir=root, + metadata=metadata, + parameters=parameters, + digest=header.ciphertext_digest.hex(), + header=header, + kdf_seconds=kdf_seconds, + ) + + # ------------------------------------------------------------------ gate 2 + + def _gate_2_header(self, encrypted: Path) -> fmt.BundleHeader: + """Parse the header and check the chunk-stream digest, before deriving anything.""" + try: + header = fmt.read_header(encrypted) + fmt.verify_ciphertext_digest(encrypted, header) + except fmt.BundleFormatError as error: + raise BundleGateError(str(error), gate=2) from error + except OSError as error: + raise BundleGateError('downloaded bundle could not be read', gate=2) from error + return header + + # ------------------------------------------------------------------ gate 3 + + def _gate_3_decrypt( + self, encrypted: Path, password: str, header: fmt.BundleHeader, staging: Path + ) -> tuple[Path, float]: + """Derive the key with the header's parameters and decrypt the stream to a file. + + The AEAD *is* the integrity check: a wrong password and a tampered byte fail here with the + same message, which never claims to distinguish them. + """ + staging.mkdir(parents=True, exist_ok=True) + zip_path = staging / 'bundle.zip' + try: + started = time.monotonic() + key = self._derive_key(password, header) + kdf_seconds = time.monotonic() - started + self._pull_stream(encrypted, key, zip_path) + except BundleGateError: + zip_path.unlink(missing_ok=True) + raise + except (nacl.exceptions.CryptoError, ValueError, RuntimeError) as error: + zip_path.unlink(missing_ok=True) + raise BundleGateError(_DECRYPT_FAILURE_MESSAGE, gate=3) from error + except OSError as error: + zip_path.unlink(missing_ok=True) + raise BundleGateError('decrypted bundle could not be written', gate=3) from error + return zip_path, kdf_seconds + + @staticmethod + def _derive_key(password: str, header: fmt.BundleHeader) -> bytes: + """Argon2id over the NFC-normalised password, with the header's own cost parameters. + + The password is normalised and **not** trimmed, matching the producer's `_derive_key`: the + same password typed on another OS must derive the same key, while a trailing space is part + of the secret. + """ + normalized = unicodedata.normalize('NFC', password) + return sodium.crypto_pwhash_alg( + fmt.KEY_SIZE, + normalized.encode('utf-8'), + header.salt, + header.kdf_opslimit, + header.kdf_memlimit_bytes, + sodium.crypto_pwhash_ALG_ARGON2ID13, + ) + + @staticmethod + def _pull_stream(encrypted: Path, key: bytes, zip_path: Path) -> None: + """Decrypt the chunk stream into `zip_path`, requiring the FINAL tag.""" + chunk_size = fmt.CHUNK_SIZE + fmt.ABYTES + state = sodium.crypto_secretstream_xchacha20poly1305_state() + with encrypted.open('rb') as source, zip_path.open('wb') as target: + source.seek(fmt.HEADER_SIZE) + stream_header = source.read(fmt.STREAM_HEADER_SIZE) + sodium.crypto_secretstream_xchacha20poly1305_init_pull(state, stream_header, key) + last_tag: int | None = None + while chunk := source.read(chunk_size): + plaintext, last_tag = sodium.crypto_secretstream_xchacha20poly1305_pull( + state, chunk + ) + target.write(plaintext) + if last_tag != sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL: + # A stream that stops without its FINAL tag was cut short. The digest already catches + # truncation of a stored file; this catches a stream that was framed to look complete. + raise BundleGateError(_DECRYPT_FAILURE_MESSAGE, gate=3) + + # ------------------------------------------------------------------ gate 4 + + def _gate_4_inspect(self, zip_path: Path) -> None: + """Inspect the central directory only. Nothing is written to disk by this gate.""" + try: + with zipfile.ZipFile(zip_path) as archive: + entries = archive.infolist() + except zipfile.BadZipFile as error: + raise BundleGateError('bundle content is not a readable archive', gate=4) from error + + if len(entries) > self.limits.max_entries: + raise BundleGateError( + f'archive declares {len(entries)} entries, above the ' + f'{self.limits.max_entries} limit', + gate=4, + ) + if not entries: + raise BundleGateError('archive is empty', gate=4) + + uncompressed = sum(entry.file_size for entry in entries) + compressed = sum(entry.compress_size for entry in entries) + if uncompressed > self.limits.max_uncompressed_bytes: + raise BundleGateError( + f'archive declares {uncompressed} uncompressed bytes, above the ' + f'{self.limits.max_uncompressed_bytes} limit', + gate=4, + ) + ratio = uncompressed / max(compressed, 1) + if ratio > self.limits.max_compression_ratio: + raise BundleGateError( + f'archive compression ratio {ratio:.1f} is above the ' + f'{self.limits.max_compression_ratio} limit', + gate=4, + ) + + for entry in entries: + self._check_entry_name(entry.filename) + self._check_entry_mode(entry) + + @staticmethod + def _check_entry_name(name: str) -> None: + """Reject absolute names and traversal, from the declared name alone.""" + normalised = name.replace('\\', '/') + if normalised.startswith('/') or (len(normalised) > 1 and normalised[1] == ':'): + raise BundleGateError('archive declares an absolute entry path', gate=4) + if any(part == '..' for part in normalised.split('/')): + raise BundleGateError('archive declares an entry escaping its own tree', gate=4) + + @staticmethod + def _check_entry_mode(entry: zipfile.ZipInfo) -> None: + """Reject symlinks and anything that is neither a regular file nor a directory.""" + mode = entry.external_attr >> 16 + if stat.S_IFMT(mode) == 0: + # No file-type bits stored. This is the normal case for two kinds of entry: a zip + # written by a tool that records no Unix mode at all (`mode == 0`), and + # `ZipFile.writestr`, which stores permissions only — the producer's `metadata.json` + # arrives as `0o600 << 16`. Nothing to check: the member is extracted as a plain file + # with the worker's own permissions either way. + return + if stat.S_ISLNK(mode): + raise BundleGateError('archive declares a symbolic link', gate=4) + if not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)): + raise BundleGateError('archive declares an entry that is not a regular file', gate=4) + + # ------------------------------------------------------------------ gate 5 + + def _gate_5_extract(self, zip_path: Path, root: Path) -> None: + """Extract into a fresh directory, containing every member by resolved path. + + A pre-existing extraction root is a failure rather than something to clear: it means + another import, or a previous attempt, owns that path. + """ + if root.exists(): + raise BundleGateError('extraction directory already exists', gate=5) + root.mkdir(parents=True) + resolved_root = root.resolve() + try: + with zipfile.ZipFile(zip_path) as archive: + for entry in archive.infolist(): + self._extract_member(archive, entry, root, resolved_root) + except BundleGateError: + shutil.rmtree(root, ignore_errors=True) + raise + except (OSError, zipfile.BadZipFile) as error: + shutil.rmtree(root, ignore_errors=True) + raise BundleGateError('bundle content could not be unpacked', gate=5) from error + + @staticmethod + def _extract_member( + archive: zipfile.ZipFile, entry: zipfile.ZipInfo, root: Path, resolved_root: Path + ) -> None: + """Write one member, re-checking containment on the resolved path.""" + name = entry.filename.replace('\\', '/') + target = root / name + resolved = Path(os.path.realpath(target)) + if resolved != resolved_root and resolved_root not in resolved.parents: + raise BundleGateError('archive member resolves outside the extraction root', gate=5) + if entry.is_dir(): + resolved.mkdir(parents=True, exist_ok=True) + return + if resolved.exists(): + raise BundleGateError('archive member would overwrite an existing file', gate=5) + resolved.parent.mkdir(parents=True, exist_ok=True) + # Written as a plain file: the archive's stored permissions are never restored, so no + # member can arrive executable. + with archive.open(entry) as source, resolved.open('wb') as sink: + shutil.copyfileobj(source, sink) + + # ------------------------------------------------------------------ gate 6 + + def _gate_6_structure(self, root: Path) -> tuple[dict[str, Any], dict[str, Any]]: + """Validate the extracted layout and the metadata schema.""" + metadata_path = root / METADATA_FILE_NAME + artifacts_dir = root / ARTIFACTS_DIR_NAME + if not metadata_path.is_file(): + raise BundleGateError('bundle has no metadata document at its root', gate=6) + if not artifacts_dir.is_dir(): + raise BundleGateError('bundle has no artifacts directory', gate=6) + + document = self._load_metadata_document(metadata_path) + metadata = document['metadata'] + parameters = document['parameters'] + + self._check_metadata_fields(metadata) + self._check_parameters(parameters) + self._check_model_directories(artifacts_dir) + return metadata, parameters + + @staticmethod + def _load_metadata_document(metadata_path: Path) -> dict[str, Any]: + """Read `metadata.json` and check its two top-level keys.""" + try: + document = json.loads(metadata_path.read_text(encoding='utf-8')) + except (json.JSONDecodeError, UnicodeDecodeError, OSError) as error: + raise BundleGateError('bundle metadata is not readable JSON', gate=6) from error + if not isinstance(document, dict) or set(document) != REQUIRED_TOP_LEVEL_KEYS: + raise BundleGateError( + 'bundle metadata does not carry exactly the parameters and metadata blocks', gate=6 + ) + if not isinstance(document['metadata'], dict) or not isinstance( + document['parameters'], dict + ): + raise BundleGateError('bundle metadata blocks are not objects', gate=6) + return document + + @staticmethod + def _check_metadata_fields(metadata: dict[str, Any]) -> None: + """Every origin field the import records must be present and usable.""" + for field_name in REQUIRED_METADATA_FIELDS: + value = metadata.get(field_name) + if not isinstance(value, str) or not value.strip(): + raise BundleGateError(f'bundle metadata is missing the {field_name} field', gate=6) + experiment_name = metadata['experiment_name'] + if not (EXPERIMENT_NAME_MIN_LENGTH <= len(experiment_name) <= EXPERIMENT_NAME_MAX_LENGTH): + raise BundleGateError( + "bundle experiment name does not fit the platform's limits", gate=6 + ) + + @staticmethod + def _check_parameters(parameters: dict[str, Any]) -> None: + """`target_variable` is what `model_config.target` is built from; nothing else supplies it.""" + for name in REQUIRED_PARAMETERS: + value = parameters.get(name) + if value is None or not str(value).strip(): + raise BundleGateError(f'bundle parameters are missing {name}', gate=6) + + @staticmethod + def _check_model_directories(artifacts_dir: Path) -> None: + """`prediction_model` and `data_model` must each be a loadable MLflow model directory.""" + for directory in REQUIRED_MODEL_DIRS: + model_dir = artifacts_dir / directory + if not model_dir.is_dir(): + raise BundleGateError(f'bundle has no {directory} artifacts', gate=6) + for required in REQUIRED_MODEL_FILES: + if not (model_dir / required).is_file(): + raise BundleGateError(f'bundle {directory} artifacts are incomplete', gate=6) + + # ------------------------------------------------------------------ gate 7 + + def _gate_7_content_policy(self, root: Path) -> None: + """Only the file kinds an MLflow artifact tree calls for may be present.""" + for path in sorted(root.rglob('*')): + if path.is_dir(): + continue + if not path.is_file(): + raise BundleGateError('bundle contains an entry that is not a file', gate=7) + if path.name in ALLOWED_FILE_NAMES: + continue + if path.suffix.lower() in ALLOWED_FILE_SUFFIXES: + continue + raise BundleGateError('bundle contains a file a model export should not carry', gate=7) diff --git a/laborious/utils/bundle/steps.py b/laborious/utils/bundle/steps.py new file mode 100644 index 0000000..35c2506 --- /dev/null +++ b/laborious/utils/bundle/steps.py @@ -0,0 +1,55 @@ +"""The import pipeline's step vocabulary, and the gate-to-step mapping. + +`ImportStep` is the join between three things: the machine-readable step written into the import +record's JSON detail, the key of the error catalog (one code and one sentence per step) and the +value the frontend switches on. It lives in the bundle package rather than in the activities module +because the reader raises with it — `laborious/activities/model_import.py` imports it from here so +there is exactly one definition. + +It holds **pipeline** steps only. The two failures that are not steps of the pipeline — failing to +claim the record and failing to write it — are reported by the failure channel under their own +markers (`not_claimable`, `status_write`), deliberately outside this enum so the +catalog-completeness test keeps meaning "every pipeline step has a sentence". +""" + +from __future__ import annotations + +from enum import StrEnum +from typing import Final + + +class ImportStep(StrEnum): + """Where an import stopped. Values are what gets written to the record.""" + + RECEIVED = 'received' + DOWNLOAD = 'download' + DECRYPTION = 'decryption' + ARCHIVE_INSPECTION = 'archive_inspection' + EXTRACTION = 'extraction' + STRUCTURE_VALIDATION = 'structure_validation' + CONTENT_POLICY = 'content_policy' + EXPERIMENT_CREATION = 'experiment_creation' + ARTIFACT_UPLOAD = 'artifact_upload' + REGISTRATION = 'registration' + MODEL_DOCUMENT = 'model_document' + CLEANUP = 'cleanup' + + +# The seven gates map onto the first seven steps, with gates 1 and 2 sharing one: both say "these +# are not the bytes that were sent" — an object over the size ceiling, a key with the wrong shape, a +# digest that disagrees with what the uploader computed, a wrong magic, an unsupported version, a +# chunk stream that does not match the header's digest. The catalog's `download` sentence ("The +# uploaded file could not be read, or it is not the file that was sent. Upload it again.") is true +# for all of them, and none of them may borrow the `decryption` sentence, which implicates the +# password. +GATE_TO_STEP: Final[dict[int, ImportStep]] = { + 1: ImportStep.DOWNLOAD, + 2: ImportStep.DOWNLOAD, + 3: ImportStep.DECRYPTION, + 4: ImportStep.ARCHIVE_INSPECTION, + 5: ImportStep.EXTRACTION, + 6: ImportStep.STRUCTURE_VALIDATION, + 7: ImportStep.CONTENT_POLICY, +} + +GATE_COUNT: Final[int] = 7 diff --git a/laborious/utils/connectors_config.py b/laborious/utils/connectors_config.py index ed3477a..aa2a019 100644 --- a/laborious/utils/connectors_config.py +++ b/laborious/utils/connectors_config.py @@ -1,5 +1,6 @@ import json -from os import getenv +import tempfile +from os import getenv, path from typing import Any @@ -90,3 +91,72 @@ def build_minio_config() -> dict[str, Any]: 'retention_hours': int(getenv('MINIO_RETENTION_HOURS', '24')), 'secure': getenv('MINIO_SECURE', 'false') == 'true', } + + +def build_import_config() -> dict[str, Any]: + """ + Build the `.sientia` model-import configuration from environment variables. + + The password key is read as the raw base64 string and **not** validated here: the worker must + start whether or not `IMPORT_MODEL_ENABLED` is set, and a missing or malformed key is an import + failure at the decryption step (`laborious.utils.import_password.load_envelope_key`), not a + boot failure. + + Environment Variables: + IMPORT_PASSWORD_KEY: base64 of the 32-byte AES-256-GCM key that wraps bundle passwords + IMPORT_BUNDLE_BUCKET: bucket holding the uploaded bundles (default: sientia) + IMPORT_BUNDLE_PREFIX: key prefix the uploads must sit under (default: imported_models/) + IMPORT_WORK_DIR: worker scratch directory (default: /sientia-import) + IMPORT_MAX_OBJECT_BYTES: gate 1 object size ceiling (default: 1 GiB) + IMPORT_MAX_ARCHIVE_ENTRIES: gate 4 entry count ceiling (default: 5000) + IMPORT_MAX_UNCOMPRESSED_BYTES: gate 4 uncompressed size ceiling (default: 4 GiB) + IMPORT_MAX_COMPRESSION_RATIO: gate 4 compression ratio ceiling (default: 200) + IMPORT_MODELS_COLLECTION: MongoDB collection holding the model listing (default: models) + IMPORT_BUNDLE_RETENTION_DAYS: lifecycle expiry for the uploaded bundle (default: 7) + + Returns: + dict: import configuration dictionary + """ + prefix = getenv('IMPORT_BUNDLE_PREFIX', 'imported_models/') + return { + 'password_key': getenv('IMPORT_PASSWORD_KEY'), + 'bucket': getenv('IMPORT_BUNDLE_BUCKET', 'sientia'), + 'prefix': prefix if prefix.endswith('/') else f'{prefix}/', + 'work_dir': getenv('IMPORT_WORK_DIR', path.join(tempfile.gettempdir(), 'sientia-import')), + 'max_object_bytes': int(getenv('IMPORT_MAX_OBJECT_BYTES', str(1024 * 1024 * 1024))), + 'max_entries': int(getenv('IMPORT_MAX_ARCHIVE_ENTRIES', '5000')), + 'max_uncompressed_bytes': int( + getenv('IMPORT_MAX_UNCOMPRESSED_BYTES', str(4 * 1024 * 1024 * 1024)) + ), + 'max_compression_ratio': float(getenv('IMPORT_MAX_COMPRESSION_RATIO', '200')), + 'models_collection': getenv('IMPORT_MODELS_COLLECTION', 'models'), + 'retention_days': int(getenv('IMPORT_BUNDLE_RETENTION_DAYS', '7')), + } + + +def build_import_status_config() -> dict[str, Any]: + """ + Build the connection to the import log database — a **second** Postgres connection. + + The import log is `public.experiment_run` in the database owned by the Spring Boot BFF + (`sientia-core-mlops-bff`), not the `sientia` database Laborious itself uses. The table and + schema are constants of the code, never inputs; only the connection is configuration. + + Environment Variables: + IMPORT_STATUS_DB_HOST: host (default: localhost) + IMPORT_STATUS_DB_PORT: port (default: 5432) + IMPORT_STATUS_DB_NAME: database (default: sientia-core-mlops-bff) + IMPORT_STATUS_DB_USER: user (default: mlops_bff_user) + IMPORT_STATUS_DB_PASSWORD: password (default: empty) + + Returns: + dict: import status database configuration dictionary + """ + return { + 'host': getenv('IMPORT_STATUS_DB_HOST', 'localhost'), + 'port': int(getenv('IMPORT_STATUS_DB_PORT', '5432')), + 'dbname': getenv('IMPORT_STATUS_DB_NAME', 'sientia-core-mlops-bff'), + 'user': getenv('IMPORT_STATUS_DB_USER', 'mlops_bff_user'), + 'password': getenv('IMPORT_STATUS_DB_PASSWORD', ''), + 'schema': 'public', + } diff --git a/laborious/utils/import_password.py b/laborious/utils/import_password.py new file mode 100644 index 0000000..5b61e8b --- /dev/null +++ b/laborious/utils/import_password.py @@ -0,0 +1,98 @@ +"""The bundle password envelope: AES-256-GCM, opened inside the activity that uses the password. + +The workflow only ever carries `password_envelope`, so no plaintext password reaches the Temporal +event history. This module is imported by the bundle-opening activity and by nothing else — in +particular not by `laborious/workflows/import_model.py`, which a test asserts. + +Envelope layout, chosen because WebCrypto gives the Angular frontend AES-GCM natively: + + base64( nonce(12 bytes) || ciphertext || GCM tag(16 bytes) ) + +The key is a 32-byte value held in a Kubernetes Secret and exposed as `IMPORT_PASSWORD_KEY`, +base64-encoded. Neither the key, the envelope nor the recovered password is ever logged, notified, +used as a metric label or written to the import record. +""" + +from __future__ import annotations + +import base64 +import binascii +from typing import Final + +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +NONCE_SIZE: Final[int] = 12 +TAG_SIZE: Final[int] = 16 +KEY_SIZE: Final[int] = 32 +MIN_ENVELOPE_SIZE: Final[int] = NONCE_SIZE + TAG_SIZE + + +class EnvelopeError(Exception): + """The envelope or its key is unusable. + + Deliberately one type for every cause — missing key, wrong key, malformed base64, altered + bytes — because the failure is reported at the decryption step either way. What it must never + be confused with is a wrong *bundle* password: that one comes from the reader's gate 3, and the + two carry different sentences. + """ + + +def load_envelope_key(raw_key: str | None) -> bytes: + """Decode the configured envelope key. + + Args: + raw_key: the value of `IMPORT_PASSWORD_KEY` — base64 of exactly 32 bytes. + + Returns: + bytes: the 32-byte key. + + Raises: + EnvelopeError: when the value is absent, not base64, or not 32 bytes long. There is no + fallback interpretation: an unset key never means "the envelope is plaintext". + """ + if not raw_key: + raise EnvelopeError('the import password key is not configured') + try: + key = base64.b64decode(raw_key, validate=True) + except (binascii.Error, ValueError) as error: + raise EnvelopeError('the import password key is not valid base64') from error + if len(key) != KEY_SIZE: + raise EnvelopeError(f'the import password key must be {KEY_SIZE} bytes') + return key + + +def decrypt_password_envelope(envelope: str, key: bytes) -> str: + """Recover the bundle password from its envelope. + + Args: + envelope: base64 of `nonce || ciphertext || tag`. + key: the 32-byte key from `load_envelope_key`. + + Returns: + str: the plaintext password, exactly as it was typed — not trimmed, not case-folded. NFC + normalisation happens at key derivation, where the producer does it. + + Raises: + EnvelopeError: malformed envelope, wrong key or altered bytes (AES-GCM authentication). + """ + if not envelope: + raise EnvelopeError('the import request carries no password envelope') + try: + payload = base64.b64decode(envelope, validate=True) + except (binascii.Error, ValueError) as error: + raise EnvelopeError('the password envelope is not valid base64') from error + if len(payload) < MIN_ENVELOPE_SIZE: + raise EnvelopeError('the password envelope is too short to hold a nonce and a tag') + + nonce, sealed = payload[:NONCE_SIZE], payload[NONCE_SIZE:] + try: + plaintext = AESGCM(key).decrypt(nonce, sealed, None) + except InvalidTag as error: + raise EnvelopeError('the password envelope could not be authenticated') from error + except ValueError as error: + raise EnvelopeError('the password envelope could not be opened') from error + try: + return plaintext.decode('utf-8') + except UnicodeDecodeError as error: + raise EnvelopeError('the recovered password is not valid UTF-8') from error diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index 7a420ba..9f76141 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -10,6 +10,8 @@ The worker supports multiple runtime-scoped task queues (via ``sientia_do.tempor - minimal_retrain-{runtime}-queue: Model retraining workflows - drift-{runtime}-queue: Drift detection workflows - simple_metrics-{runtime}-queue: Simple metrics workflows +- import_model-{runtime}-queue: `.sientia` model import — registered **only** when + ``IMPORT_MODEL_ENABLED`` is truthy, so the containment boundary stays a deployment decision ``RUNTIME`` must be set; it is passed to every ``prepare_worker`` call. Schedulers must use the same queue names (breaking change vs legacy ``drift-queue`` / ``simple_metrics-queue``). @@ -30,6 +32,9 @@ Environment Variables: - HTTP_METRICS_PORT: Prometheus metrics server port (default: 9090) - HTTP_SDK_METRICS_PORT: Temporal SDK metrics port (default: 9091) - PROJECT_NAME: Project name for notifications (default: laborious) +- IMPORT_MODEL_ENABLED: register the fifth (import) worker; unset by default +- IMPORT_PASSWORD_KEY / IMPORT_BUNDLE_* / IMPORT_STATUS_DB_*: import configuration, see + ``laborious.utils.connectors_config`` """ from temporalio import client, workflow @@ -53,11 +58,14 @@ with workflow.unsafe.imports_passed_through(): from laborious import metrics from laborious.activities.activities import Activities from laborious.utils.connectors_config import ( + build_import_config, + build_import_status_config, build_minio_config, build_mlflow_config, build_opc_config, ) from laborious.workflows.drift import Drift + from laborious.workflows.import_model import ImportModel from laborious.workflows.minimal_retrain import MinimalRetrain from laborious.workflows.predictions_batch import PredictionsBatch from laborious.workflows.simple_metrics import SimpleMetrics @@ -137,6 +145,9 @@ async def main(): pi_web_api_config=build_api_config(), logger=logger, notification_handler=notification_handler, + mongo_config=mongo_config, + import_config=build_import_config(), + import_status_config=build_import_status_config(), ) logger.custom_info('Initializing OPC...', metadata_runtime) @@ -235,6 +246,42 @@ async def main(): ), ] + # The fifth worker is opt-in: containment (QTZPOC-19) wants the importer in its own runtime with + # its own credentials, and a flag lets that be a deployment decision instead of forcing every + # Laborious runtime to poll the import queue today. Unset, the four workers above are unaffected + # and no import queue is polled at all. + if os.getenv('IMPORT_MODEL_ENABLED', '').strip().lower() in ('1', 'true', 'yes', 'on'): + logger.custom_info('Import model worker enabled', metadata_runtime) + workers.append( + prepare_worker( + temporal_client=temporal_client, + main_workflow=ImportModel, + other_workflows=[], + activities=[ + # Import log row (the BFF database, update-only) + activities.claim_import_status, + activities.record_import_status, + activities.record_import_names, + activities.record_import_terminal_status, + activities.report_import_status_write_failure, + # Bundle + activities.download_import_bundle, + activities.open_import_bundle, + # MLflow provisioning + activities.create_import_experiment, + activities.upload_import_artifacts, + activities.register_import_model_version, + # The model listing + activities.write_import_model_document, + # Cleanup and retention + activities.cleanup_import_files, + activities.ensure_import_bundle_retention, + ], + logger=logger, + runtime=runtime, + ) + ) + handlers = [] for w in workers: handlers.append(w.run()) diff --git a/laborious/workflows/import_model.py b/laborious/workflows/import_model.py new file mode 100644 index 0000000..39cbbf6 --- /dev/null +++ b/laborious/workflows/import_model.py @@ -0,0 +1,416 @@ +"""The `import_model` workflow: a `.sientia` object becomes a registered model, or a durable ERROR. + +Shape of this workflow, and the reasons it has that shape: + +- **The claim comes first.** The record already exists — the frontend inserted it through the BFF — + so the workflow's first act is to take ownership of it. Everything after that failure is recorded + *on* the record; a failed claim is recorded nowhere, because the workflow owns no row. +- **Fail-fast, and nothing else.** No rollback, no compensation, no resume, no reconciliation, no + `continue_as_new`. A failed import stays failed; trying again is a new import with a new upload. +- **Only the status writes retry**, and only because `UPDATE ... SET WHERE id` is + idempotent. Every provisioning call keeps `maximum_attempts=1`, because a repeated registry call or + Mongo insert can write twice. +- **One `finally`, two actions**: cleanup, then the single terminal write that classifies the + outcome once. No `except` block writes a verdict of its own. +- **The workflow holds ids, never contents.** The decrypted bundle never crosses an activity + boundary, and the password reaches this module only as an envelope it cannot open. +""" + +from datetime import timedelta + +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + from typing import Any + + from sientia_do.temporal.policies import retry_policy + + from laborious.activities.activities import Activities + from laborious.activities.model_import import ( + IMPORT_STATUS_HINT_RETRY_POLICY, + IMPORT_STATUS_RETRY_POLICY, + derive_run_name, + ) + from laborious.activities.model_import_errors import ImportInputError, failure_location + from laborious.utils.bundle.steps import ImportStep + +# `start_to_close_timeout` is per attempt. The terminal write additionally gets a +# `schedule_to_close_timeout` larger than its retry envelope (10 attempts, ≈151 s of backoff, ≈251 s +# if every attempt also burns its timeout) — set it smaller and Temporal cuts the retries short, +# which is the one configuration mistake that would silently undo design D20. +STATUS_WRITE_TIMEOUT = timedelta(seconds=10) +STATUS_WRITE_ENVELOPE = timedelta(seconds=300) +DOWNLOAD_TIMEOUT = timedelta(seconds=600) +OPEN_TIMEOUT = timedelta(seconds=900) +EXPERIMENT_TIMEOUT = timedelta(seconds=60) +ARTIFACT_UPLOAD_TIMEOUT = timedelta(seconds=900) +REGISTRATION_TIMEOUT = timedelta(seconds=120) +DOCUMENT_TIMEOUT = timedelta(seconds=60) +CLEANUP_TIMEOUT = timedelta(seconds=120) +RETENTION_TIMEOUT = timedelta(seconds=60) + +# Fields no import may carry. The project is a frontend concern: the model listing is the MongoDB +# `models` document, no migrated database holds a project or model table, and nothing in this +# repository reads a project-to-model link (design D22). An input naming one is rejected rather than +# accepted and ignored, so no caller believes a project link was recorded. The same goes for a +# relational target: there is none, and for a plaintext password, which does not belong in an event +# history whatever else is true. +REJECTED_INPUT_FIELDS = ( + 'project_id', + 'project_name', + 'provisioning_target', + 'schema', + 'table_name', + 'status_table', + 'password', +) + +DIGEST_LENGTH = 64 +_HEX_DIGITS = frozenset('0123456789abcdef') + + +@workflow.defn(name='import_model') +class ImportModel: + """Turn an encrypted `.sientia` object in MinIO into a registered model, or into an ERROR.""" + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Execute the import. + + Args: + input_data: the import request. + Required keys: + - import_run_id (int): primary key of the import log row the frontend inserted + - object_key (str): the uploaded object's key + - expected_digest (str): the SHA-256 the uploader computed for those bytes + - password_envelope (str): the encrypted bundle password, never plaintext + Optional keys: + - bucket (str): defaults to the configured import bucket + + Returns: + dict: `import_run_id`, `run_id`, `model_name`, `version` and the model document's id. + + Raises: + ImportLogRowNotClaimableError: the record is not this workflow's — the run ends FAILED + with that error type and nothing is written to any row. + Exception: whatever failed. The record carries the failing step, its code and its + sentence; this exception carries the technical truth. + """ + workflow_id = workflow.info().workflow_id + import_run_id = self._require_import_run_id(input_data.get('import_run_id')) + metadata: dict[str, Any] = { + 'model_name': '-', + 'model_id': str(import_run_id), + 'workflow_name': 'import_model', + 'schedule_name': '-', + } + base = { + 'metadata': metadata, + 'import_run_id': import_run_id, + 'workflow_id': workflow_id, + } + + # Class A. Outside the try/finally on purpose: a claim that matches nothing leaves this + # workflow owning no row, so there is nothing to clean up and nothing to record. + claim = await workflow.execute_activity_method( + Activities.claim_import_status, + base, + retry_policy=IMPORT_STATUS_RETRY_POLICY, + start_to_close_timeout=STATUS_WRITE_TIMEOUT, + schedule_to_close_timeout=STATUS_WRITE_ENVELOPE, + ) + + current_step = ImportStep.RECEIVED + failed_step: str | None = None + gate: int | None = None + succeeded = False + source: dict[str, Any] = {} + result: dict[str, Any] = {'import_run_id': import_run_id} + + try: + self._validate_input(input_data, claim) + + current_step = ImportStep.DOWNLOAD + await self._hint(base, current_step) + await self._ensure_retention(base, input_data) + download = await workflow.execute_activity_method( + Activities.download_import_bundle, + { + **base, + 'bucket': input_data.get('bucket'), + 'object_key': input_data['object_key'], + 'expected_digest': input_data['expected_digest'], + }, + retry_policy=retry_policy, + start_to_close_timeout=DOWNLOAD_TIMEOUT, + ) + + current_step = ImportStep.DECRYPTION + await self._hint(base, current_step) + opened = await workflow.execute_activity_method( + Activities.open_import_bundle, + { + **base, + 'path': download['path'], + 'password_envelope': input_data['password_envelope'], + }, + retry_policy=retry_policy, + start_to_close_timeout=OPEN_TIMEOUT, + ) + source = opened['metadata'] + parameters = opened['parameters'] + model_name = source['model_name'] + metadata['model_name'] = model_name + run_name = derive_run_name(model_name, source['model_version']) + + # The names go on the record before anything exists in MLflow, so a person watching the + # import list sees *which* model is being imported while it is still running. + await self._record_names(base, source, run_name) + + current_step = ImportStep.EXPERIMENT_CREATION + await self._hint(base, current_step, source) + experiment = await workflow.execute_activity_method( + Activities.create_import_experiment, + {**base, 'experiment_name': source['experiment_name']}, + retry_policy=retry_policy, + start_to_close_timeout=EXPERIMENT_TIMEOUT, + ) + + current_step = ImportStep.ARTIFACT_UPLOAD + await self._hint(base, current_step, source) + run = await workflow.execute_activity_method( + Activities.upload_import_artifacts, + { + **base, + 'experiment_id': experiment['experiment_id'], + 'run_name': run_name, + 'extracted_dir': opened['extracted_dir'], + 'parameters': parameters, + 'source': source, + }, + retry_policy=retry_policy, + start_to_close_timeout=ARTIFACT_UPLOAD_TIMEOUT, + ) + + current_step = ImportStep.REGISTRATION + await self._hint(base, current_step, source) + version = await workflow.execute_activity_method( + Activities.register_import_model_version, + {**base, 'model_name': model_name, 'run_id': run['run_id']}, + retry_policy=retry_policy, + start_to_close_timeout=REGISTRATION_TIMEOUT, + ) + + current_step = ImportStep.MODEL_DOCUMENT + await self._hint(base, current_step, source) + document = await workflow.execute_activity_method( + Activities.write_import_model_document, + { + **base, + 'model_name': model_name, + 'target': parameters['target_variable'], + }, + retry_policy=retry_policy, + start_to_close_timeout=DOCUMENT_TIMEOUT, + ) + + succeeded = True + result = { + 'import_run_id': import_run_id, + 'run_id': run['run_id'], + 'model_name': model_name, + 'version': version['version'], + 'document_id': document.get('id'), + } + except BaseException as error: # NOSONAR - classified once below, then re-raised untouched + located_step, located_gate = failure_location(error) + failed_step = located_step or current_step.value + gate = located_gate + raise + finally: + cleanup = await self._cleanup(base) + await self._record_terminal( + base, + succeeded=succeeded, + step=failed_step or current_step.value, + gate=gate, + source=source, + cleanup_failed=not cleanup.get('cleaned', True), + ) + + return result + + # ------------------------------------------------------------------ validation + + @staticmethod + def _require_import_run_id(value: Any) -> int: + """The one check that runs before the claim, because the claim needs its result. + + A missing or unusable id means there is no record to claim and none to write: the run ends + FAILED with this error type, and the failure is visible in the workflow's terminal state, its + error type, the error metric and a notification — everywhere except a row, which is the + point. + """ + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ImportInputError('import_run_id must be a positive integer') + return value + + @staticmethod + def _validate_input(input_data: dict[str, Any], claim: dict[str, Any]) -> None: + """Validate the rest of the request, and check it against the record it just claimed. + + This runs **after** the claim so its failure lands on the record rather than only in the + logs: by now the row is this workflow's, so a bad request is reported to the person who made + it. Everything here fails at step `received`. + """ + unknown = [name for name in REJECTED_INPUT_FIELDS if name in input_data] + if unknown: + raise ImportInputError( + f'the import request carries fields this workflow does not accept: ' + f'{", ".join(sorted(unknown))}' + ) + + for required in ('object_key', 'expected_digest', 'password_envelope'): + if not input_data.get(required): + raise ImportInputError(f'the import request is missing {required}') + + digest = str(input_data['expected_digest']).lower() + if len(digest) != DIGEST_LENGTH or not set(digest) <= _HEX_DIGITS: + raise ImportInputError('expected_digest must be a 64-character hexadecimal SHA-256') + + object_key = input_data['object_key'] + file_name = claim.get('file_name') + if file_name and file_name != object_key: + raise ImportInputError('the record names a different object than the request') + + request_digest = claim.get('request_digest') + if request_digest and request_digest != digest: + raise ImportInputError('the record names a different digest than the request') + + # --------------------------------------------------------------- status writes + + async def _hint( + self, + base: dict[str, Any], + step: ImportStep, + source: dict[str, Any] | None = None, + ) -> None: + """Record the step about to be attempted. A hint: its failure never fails the import.""" + try: + await workflow.execute_activity_method( + Activities.record_import_status, + {**base, 'step': step.value, 'source': source or None}, + retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY, + start_to_close_timeout=STATUS_WRITE_TIMEOUT, + ) + except BaseException as error: # NOSONAR - a lost hint must not fail a working import + await self._report_status_write_failure(base, error) + + async def _record_names( + self, base: dict[str, Any], source: dict[str, Any], run_name: str + ) -> None: + """Write the experiment and run names. Also a hint: the run is created either way.""" + try: + await workflow.execute_activity_method( + Activities.record_import_names, + { + **base, + 'experiment_name': source['experiment_name'], + 'run_name': run_name, + 'source': source, + 'step': ImportStep.EXPERIMENT_CREATION.value, + }, + retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY, + start_to_close_timeout=STATUS_WRITE_TIMEOUT, + ) + except BaseException as error: # NOSONAR - the names are for a human reading the list + await self._report_status_write_failure(base, error) + + async def _record_terminal( + self, + base: dict[str, Any], + *, + succeeded: bool, + step: str | None, + gate: int | None, + source: dict[str, Any], + cleanup_failed: bool, + ) -> None: + """The single authoritative write, and the last thing this workflow does. + + Its own failure is reported through the channel with the verdict it could not write, and it + never turns a completed import into a failed one — nor masks the cause of a real failure. + """ + try: + await workflow.execute_activity_method( + Activities.record_import_terminal_status, + { + **base, + 'succeeded': succeeded, + 'step': step, + 'gate': gate, + 'source': source or None, + 'cleanup_failed': cleanup_failed, + }, + retry_policy=IMPORT_STATUS_RETRY_POLICY, + start_to_close_timeout=STATUS_WRITE_TIMEOUT, + schedule_to_close_timeout=STATUS_WRITE_ENVELOPE, + ) + except BaseException as error: # NOSONAR - the outcome stands even when it cannot be written + await self._report_status_write_failure( + base, + error, + verdict=f'succeeded={succeeded} step={step} gate={gate}', + ) + + async def _report_status_write_failure( + self, + base: dict[str, Any], + error: BaseException, + verdict: str | None = None, + ) -> None: + """Route a status write that could not be made through the one failure channel.""" + try: + await workflow.execute_activity_method( + Activities.report_import_status_write_failure, + {**base, 'detail': str(error), 'verdict': verdict}, + retry_policy=retry_policy, + start_to_close_timeout=STATUS_WRITE_TIMEOUT, + ) + except BaseException: # NOSONAR - reporting the report is where this stops + pass + + # ----------------------------------------------------------- side-effect steps + + @staticmethod + async def _ensure_retention(base: dict[str, Any], input_data: dict[str, Any]) -> None: + """Ensure the bundle bucket's expiry rule, without making it a reason to fail an import. + + The rule is applied by the importer rather than assumed from a Helm value, so the acceptance + criterion is verifiable from the code that depends on it. But a bucket policy the importer + cannot set is an operational problem, not a bad import: failing a valid model import because + a lifecycle API refused would be the wrong trade, so this is reported and dropped like a + lost status hint. + """ + try: + await workflow.execute_activity_method( + Activities.ensure_import_bundle_retention, + {**base, 'bucket': input_data.get('bucket')}, + retry_policy=retry_policy, + start_to_close_timeout=RETENTION_TIMEOUT, + ) + except BaseException: # NOSONAR - retention is not a precondition of a correct import + pass + + @staticmethod + async def _cleanup(base: dict[str, Any]) -> dict[str, Any]: + """Remove this import's temporary files. Runs on every path, before the terminal write.""" + try: + return await workflow.execute_activity_method( + Activities.cleanup_import_files, + base, + retry_policy=retry_policy, + start_to_close_timeout=CLEANUP_TIMEOUT, + ) + except BaseException: # NOSONAR - cleanup never replaces the failure that brought us here + return {'cleaned': False} diff --git a/requirements-local.txt b/requirements-local.txt index 852328a..8d060e3 100644 --- a/requirements-local.txt +++ b/requirements-local.txt @@ -4,11 +4,17 @@ sqlalchemy asyncua==1.0.6 redis git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.1 -git+ssh://git@github.com/Aignosi/sientia-model-library.git@0.10.0 +/home/grezewave/Documents/projects/sientia/sientia-mlops-library prometheus-client -botocore -boto3 -s3fs +# fsspec/s3fs/gcsfs pinned as a set: nannyml (via the mlops library) pulls both s3fs and +# gcsfs, and each pins fsspec exactly, so a free `s3fs` makes the resolver fail against +# evidently's fsspec>=2024.2.0. Versions match the library-distribution-controller freeze. +fsspec==2026.7.0 +s3fs==2026.7.0 +gcsfs==2026.8.0 +# aiobotocore 3.9.0 (pulled by s3fs) requires botocore<1.43.57,>=1.43.3. +botocore==1.43.56 +boto3==1.43.56 pyarrow kaleido hyperopt diff --git a/requirements.txt b/requirements.txt index 4b2a2a8..5b70c81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,11 +4,17 @@ sqlalchemy asyncua==1.0.6 redis sientia_do>=1.12.1 -sientia>0.40.0 +sientia>=0.41.0 prometheus-client -botocore -boto3 -s3fs +# fsspec/s3fs/gcsfs pinned as a set: nannyml (via sientia) pulls both s3fs and gcsfs, and +# each pins fsspec exactly, so a free `s3fs` makes the resolver fail against evidently's +# fsspec>=2024.2.0. Versions match the library-distribution-controller freeze. +fsspec==2026.7.0 +s3fs==2026.7.0 +gcsfs==2026.8.0 +# aiobotocore 3.9.0 (pulled by s3fs) requires botocore<1.43.57,>=1.43.3. +botocore==1.43.56 +boto3==1.43.56 pyarrow kaleido hyperopt @@ -18,3 +24,10 @@ scipy<1.14.0 scikit-learn==1.4.2 pycaret==3.3.2 joblib +# .sientia bundle import (Argon2id + XChaCha20-Poly1305 secretstream): same pin as the +# producer in sientia-projects-templates, so both sides open the same bytes. +pynacl==1.5.0 +# AES-256-GCM password envelope; already present transitively, declared explicitly. +cryptography +# MongoDB models collection (the model listing); transitive via sientia_do, declared explicitly. +pymongo diff --git a/tests/fixtures/bundle/README.md b/tests/fixtures/bundle/README.md new file mode 100644 index 0000000..7a5cad8 --- /dev/null +++ b/tests/fixtures/bundle/README.md @@ -0,0 +1,86 @@ +# Golden `.sientia` fixtures + +Binary conformance vectors for `laborious/utils/bundle/`. They are **bytes the producer's own code +emitted**, not bytes this repository knows how to write: there is no production writer here (change +`import-model-workflow`, design D1). If the producer's layout moves, the reader's tests fail against +these files instead of failing in production. + +| File | What it is | +|---|---| +| `golden_model_v3.sientia` | `format_version` 1 bundle, 110767 bytes, plaintext zip 110569 bytes — **two chunks**, so `MESSAGE`/`FINAL` framing is exercised by the golden fixture itself | +| `golden_model_v3.metadata.json` | the `metadata.json` that went into that bundle, in plaintext, so a test can assert the recovered value without decrypting first | +| `regenerate.py` | the script that produced them, importing the producer's `bundle.py`/`crypto.py` by path | + +## Password + +``` +Import1Model!Fixture +``` + +Recorded deliberately: the fixture protects nothing. It satisfies the producer's export-side +strength rule (≥ 12 characters, upper, lower, digit, symbol), which is checked by +`crypto.validate_password_strength` at generation time. + +## Provenance + +- Producer repository: `sientia-projects-templates` +- Producer commit: `8d3c1fd117c697e8b352c34f11c732809a4a129b` + (`8d3c1fd QTZPOC-7: adapt to lint rules picked up from merging main`) +- Producer modules used: `app/src/operations/model_export/bundle.py`, + `app/src/operations/model_export/crypto.py` — loaded by path, so every constant, the `struct` + layout, the zip layout, the KDF parameters and the chunk framing come from the producer and none of + them is re-implemented here +- Producer pin honoured: `pynacl==1.5.0` (this repository pins the same version) + +One deviation from task 0.3, recorded rather than hidden: the bytes were generated with the +producer's **code** under *this* repository's venv, not inside the producer's own venv, because that +venv has neither `pynacl` nor `mlflow` installed. The dependency that matters for the bytes is +`pynacl`, pinned identically on both sides, and `mlflow` is not reached — `regenerate.py` builds the +artifact tree itself instead of calling `client.download_artifacts`. + +## Contents of the bundle + +``` +metadata.json (written first, no indent, by bundle.build_zip) +artifacts/prediction_model/MLmodel +artifacts/prediction_model/model.pkl (90 KiB of incompressible filler) +artifacts/prediction_model/conda.yaml +artifacts/prediction_model/python_env.yaml +artifacts/prediction_model/requirements.txt +artifacts/data_model/MLmodel +artifacts/data_model/model.pkl (8 KiB) +artifacts/data_model/conda.yaml +artifacts/data_model/python_env.yaml +artifacts/data_model/requirements.txt +artifacts/model_card/model_card.json +artifacts/train_data.csv +artifacts/test_data.csv +artifacts/raw_data.csv +``` + +No directory entries are stored — the producer only adds paths for which `is_file()` is true. + +`parameters` carries `target_variable: "SE"`, which gate 6 requires and which becomes +`model_config.target` in the MongoDB model document. + +## Digest boundary — the reason this fixture exists + +The header's `ciphertext_digest` is `sha256(file[164:])`: the producer computes +`hashlib.sha256(ciphertext)` over the joined chunks and only then writes +`header + stream_header + ciphertext`, so the 24-byte `secretstream` header is **not** covered. +Verified against this fixture and against a freshly produced bundle. Earlier prose (this change's own +first draft, and `aig-docs/03-sientia-bundle-format.md`) said "everything after the plaintext +header", i.e. from offset 140 — that is wrong, and `tests/laborious/utils/bundle/test_format.py` +asserts both directions so it cannot come back. + +## Regenerating + +```bash +source ./venv/bin/activate +PRODUCER_REPO=../sientia-projects-templates python tests/fixtures/bundle/regenerate.py +``` + +Two fields change on every run and are expected to: the 16-byte salt (`os.urandom`) and +`metadata.metadata.export_timestamp` (`datetime.now(UTC)`). Everything else is deterministic — +the filler bytes come from a seeded PRNG. After regenerating, update the producer commit above and +the byte counts in the table. diff --git a/tests/fixtures/bundle/golden_model_v3.metadata.json b/tests/fixtures/bundle/golden_model_v3.metadata.json new file mode 100644 index 0000000..cf30161 --- /dev/null +++ b/tests/fixtures/bundle/golden_model_v3.metadata.json @@ -0,0 +1,24 @@ +{ + "parameters": { + "model_name": "fixture_model", + "target_variable": "SE", + "input_variables": "['feature_a', 'feature_b']", + "models_params": "{'alpha': 0.1}", + "nan_treatment": "drop", + "scaler_name": "StandardScaler", + "train_size": "0.8", + "test_size": "0.2", + "include_ar": "False", + "retrain": "False", + "start_date": "2026-01-01 00:00:00", + "end_date": "2026-08-01 00:00:00" + }, + "metadata": { + "model_name": "fixture_model", + "experiment_name": "fixture_experiment", + "run_id": "8f14e45fceea167a5a36dedd4bea2543", + "model_version": "3", + "model_project": "fixture_project", + "export_timestamp": "2026-08-25T12:08:59.533441+00:00" + } +} diff --git a/tests/fixtures/bundle/golden_model_v3.sientia b/tests/fixtures/bundle/golden_model_v3.sientia new file mode 100644 index 0000000000000000000000000000000000000000..2766539be3d8f4d288e390baedae5615b7d62bfa GIT binary patch literal 110767 zcmbSx({e5hpljW(+O}=mwr$(CzS>>2ZQHi(uG%*Df9A}7ft{HoH+h02DpF$ds!~Es zz`(%0UCJ81MVPM2airK-7SZrP{{iv;$^Qo^@YV1;DZw1oHV{(=&KXQdbL+w?D5=%X zdQtP9eCcC%+y5s2ubz2;Q6ZtQ+OFM0ZwF{1VyQSt?o4DnVgHw}33$AFR`g`TF_Yrj z#k1UzWI{Uz#2?Ef9WM5$G$WTElr9bIT$AtIV(BB7LmmAvNzPmXE1r$fTMouu-TeY5 zn=y50x^k^Jzv7R1TLcdn%(16>V4Z^j*K8ciHWzkooG#a1_QKEOt|2q=fZ08p72>~o z>1=2k9D{A_oAg3|J;X+-t5DLyAhOv`aeO_UAzY3`D&?Yjn$`4AqJSvzau=&iAu$U#B5G-&Us4KcOtGW?U( z&9a-UZ@!(4#%h5mmg1A6^GMKl-ATXS=KKr(*&MVeOQqF>4(z^pd3+onlLnM-MKCA+ z>w*%|mV#k4pP_;=ytWGmzsdw?l@Svht0V=hqhRA*{~e`lbBqwhW$kSL&g@tWHY|?o zeDedVN5{eDev$4bvA$QL{Fer^$~K8$swlOQjeW6gfvtpH%UCh6X}d7>p4Oon^S8(1 z;`L9gp4kZl^grRH2N|!h)?Rskk)+hdlUU7NI{vS@xU+Od{52$?P^KULuQWaw$T@OJ zN?mk2ZX!OPBz!BoUeWzehLXC+KT9jhDwnTvIk#0O?`z%Qq`GkCTxJbjB;Hw_Spn3( z-`sT_Io^V-Qr-1|_cbxdRPO>w>pn2LyKI%N*lD_Si!K` z8q2zJWsQ3@_|(L<7$`8rJfu8=>r1S)beuOtZLtkq5XXQCeg!;OwX)RTby@?u){)-^ zDryWl7To$n(gdJ|74k>@r?j1EI<@JmcWSp?-#4rBd6sI$x5W@JZT9k~!7Ib~;uAC} z=0Jyl&!F=XBXmdQWO6iK$;>^DARYOZh;l>0*IS z%3bn`Y&kZli6GO4t+5xkB_MjY5m0c7lKW+SmL8=yCZ`VV2c4f5pWe{1iU_XQTI+Hw z-p!Itr;01Xc6ls>`l;jRMV~m{_0#?;b1BT*BAv#Ni-w9RIo(^-aKTR?d+t&eCbpy-Z(zPcIv!{zcb5L2!;|Yf{tP!1}5vy z+9{a^&}Fi1(r=)Ug8T{#s2@`wfn=jyII^;A_qW^MH&i!>?enWZSPo7^g?U5?JA`|> zNeP|H6MnmLmE(!&*(NuK>>AM1xx-a7jc&|8XZY8kpx9>9aHCOEr9V{}80+6bj@_vo znn$nTR61oav%#b`6yT&y1G-5nnB~s|T~7rCyom7*Dde5II>bkCCavc+zh>Yqp3Pav zjncE)z#K_6WT=e&6E7Foju7jYr2|@lQQ0UTK#4+=bq^`<)CUZQs7R6&MtX^u<&+wd zp`qYZf-C)ce{;!ir)e0jUong4;=YrYj}$+~>CzHahjlA;#jf|<1g)F({HxlUv6=&a z$Yf{V#+`0{Qdez*;ocM^)e3T(h*h6JKTyT7qEXg}2LUIHrY*VIqUa`_Xzh|na`b#(!&aWmyo0>4cW@?bf!kq_#t9&zPwP5f$gU23%B zhHUlj&cq8=u^V9Su;h~by!t#gXXIuw@4Ss|D{g0wX8KUz^~9FV@rKmzOz`%yqbwxz z)^G~`qN~F-_5g~C1aTWE{JW@jiGY&8qB;farPFIsAtpnduRsy;CE!L#z0h=KJRGW5 zChwhEM`b$mm@}0>7H5t650@NQy48|vU8JDzG7)te?xoGj5+Gl$r{*6LHNEI8NJIDU zt0#ji8zB%A~51okHn!7xEiHU!Yw%J+@`0+Zn9@oOBK(@KWd+SZ=GkE^p!u zhq{qToZJ}13w{@LO7q>Sw^H2}U1G&Z9j-@9jiBNO$oxMw+&h{)mI&?lWFGVCwFM=^Pf zHy-AwBCMrAOT8nWiwz-diqA3xyYFz{Q3EJ=5$Z`R?c`KL@U=)qijv<0*s|C|Q{hNH z?vUy~c@&^z2GYf`?84VxCs*xGgJWnGoHe_#uFcWSG*!m69plf#$yECX!OFhIjvJDA zQ=3<9AH|?Hphni1=WLkzA4z-`!&j7!Qui=kdN8|b%zpI@dyM=qf)j2Z>`@bZjswgT z?aqogi#f8TnndIXCR)EbTrpO}N&4cqGR5vs(u=Iv^}_NrUDtHWtctD&QvAFK7L8(4 z-)9USO_vwDIGN14jSWm*GrKb-o$)dh*H?Jssjxh#zyJ*0pU3{Hs3q@-yuy^ZFaS%k zjQb5!7=^NwPb^H^AKA5|;-E}ykAc||UZDx_%)H8#(3bp1MGqUy78I%>8SZ(RB>y4r zgKX{>3cK*JsguGi&kXm*S%vD9(_|Iv4`8F2wPX}t)lUrMzr2&f7+0&He9E_t86&7Q z+VrfmAb!wr08i0gQ|%5bmq%$fs%o$N8Rf}lo1<6Tp`E(p(-z+FCd;~P3h2;fVAm*E ztt~r)Ne-P_nM#X8;~j^Ket$ERnTU92b$d1asy~^%Ic}lo8gjH(uvaCjP#zPJyPhxn zb@;nRSfY{XPdQKTF8gv<;7uZJF#~_taJK`Nq`47_#eG|zRo(;6F)=xB3;sOC#083VWm3vYN zJ(1bk8xet;z^jPzs&`f)@9IUr%%jjM;JIn@saDueG$(Wt?btr)qkOFFHWIfiG-stM zOBe{TgrQCR^0C(u0`B~f(`+}<5=9RSD_pOEQ!s@Hia-c7oR-5HpPk3`>Cov7px2c+ zd8Hci=yf}Y01|YkUeAvvoLCG}Lc*^-;ta6+kky#?Yd`i<^pEsx#rZEg8EmGEUdc&9 zo~l%-sVm`R^nLApEqjF#E=8svkT+{l&41aoW1J#oh(6J286R*^?45ZXfQ-noztoX{@6qPKX)xG$tEUxmX?{ zS3x^hvY$1P)~c6T1|o~Pb!udB6zUtJ^$!PObh@L?ZJwTXr#NdF^MwV`c4h?019(Q-vq+NpK5qZFT4^34nd-E$NISC3q10X^$L%$Ns6jbl)jLkk zLLTqvI z346EVp^ZiRg@N9j3Ivry!%H(^ZDJ7g41^Y|P*vHcxfYvIrI%>Xf8fk#=TaO@vk6Eq zZnh5&N;QIj?sl8r7wNq6K5>ZdA23FS&l$i%x}~!&3>=zr*C%uLGy|CEIWUK$setx% zn@u;fJJV~AtG6c|BKl05+3;6e42##oYe_{owzR?VS&q0=6=Y5brC^PsUBUx?8{lFx zD%SIl7|yQ;`vh+JmCI(lVI`*@mvN8!dO=7E!LtTm=Kex-Sc~G_uqxk*3ZFbDe0IwW z8q9|vH$bn%^y<@99Ho|{2TmHi^WfAb{y}csrxQ;5+rzBk zrUE5J8}`{+B0^OY&fb2iK)tF0Xem{Mrsa5kP@N!(abr^fK@i((5^p9vc2K%8{;Yyj zE|)tRC%Cx=;kqRfm@rbKk{KHrEbMsZbhzWue?cc)#b2KoiD?HQqb14Vo_yk$UcvzEgX0FjF8OsTi&nu5f^uSt)z;x;w`-R|R_ z7@5@(C-Yhcx}rRYK6{cTglP6BN2?*KDj$)I=LgNyAjAQ^i}ZGAK!KOAYHg%Pw0Ca! zXqYdhnF}n6iKw&RNKBDr5cUet#ixl#++3D|@1??!OUYQZ2b)6(aqE4*9 zZuNx7)rwhM`NJus)SR29fhUtij>aprIlb9YX=RgP_^dox)A9huQm-=hqHl!10jIBo zzW~h4r$bCGK3BVJn%O0nSPB&x2}|kE^RWtki*1+*!mui@t*Gg*K$ zcL(o$?wJK*1@h4UI)1TQm6di`K+_{5W6CR4n(0dRc7BGNn;YF-4!@?kQkW5G!TJZb zely9Fwfv|#%u@e8#^ww~GYAzwmJF@XM$uCXDJd`uBj^ia0Roz}H2IZzaj{QLewy&0 z!oWOjE@|pkG~fkwf=B=0Wh9AqWd7k$uur@ecr}nrFLUEjCM%G=AoJ2K2fotTazXXB znC2_9qn`(~l%U)BGc?QEnc4BAy2ibF(K$!je?xs+yFis1Pzs4Wb^0rSkrPd#CcvzI z7-xXUQOFOP?h(C!?X--x(ToQpKW0_GqqL~hr~KRxgrH;?e8cN)z^MU?22xIs69ZLq zkVin>LlFhcWsRuJDtS0C&{)%**6*CbJvhyUX1YdpfRF4`l*?wL$E)y{oAj3Nu})s) z^9JpP(@>iD4*a1$uf*;hkHJM9IVBM?N-O3!_#o9Lf5|a&50lkyo&3a5 z0UGZCC(vpz&6#`%8e4CreQ>0KI8*h}wX3U)F+wKcBS^G=(>|p~h6GH@4}7as4fQ^G z-{wF5mcv@MC-GK8`e^daG!YmYL~ z$lqX??3n`^+kILXVD>cK(5~Y zoZK?>S|-|?DIH*!t{pw}LCOS!X3moX3I5_ErSbPLz0P>WLA#n+!mvo?H2b!*X4N6H zH@)Kdd(;!OAEM@aJc8}_P1eYd-5OTrvSbbuCEspqPq}GrtMvhViR5wZbdSFy<)=r47}(YZT^rF7qrN2j z>Na~AU{tGWhilwSkEUoQWz4o!UeAl8S}(3K>rlbYgGMYfDbfjnbZZiLB1AU!Diw)i zAvOkoEg5=2jCbaK+A;A}=Zbuax2MnW>!~Sffl}(uXiqB~8xh4pv?6oTVCv3x6Z~1e znqgac@)R+S>=0_>vqgg*nzU+7my0VY46rn3rw#cyz?ZF>6BfQdtY#T}h>x~-B=vZB z*~wT+116EhK;!KtLBKF=(i|WH-hakl_=8;qaVWofps0bz0*k{A`}u16=>hO#58NYUA0^n}y6W1_aZ| z*4qez$~E4Epx&|Gy^nPgeH;h!Bgz5}KCuPjuciysR`()Gj!7CW`fS3kBgfk3izid|uD$%7TZ_Hs?@%GVif21=n5lRMs(2NI()>aAn5cth(PiJD=|NW&I}XJ08h} z!Qr;lMe`Mx1^0sGrRLrvvuu|l7X$YeMmuf^-*B7YAqkGBn~eCp#%9lsyp$*G5msUte%GqE zSnr;ozC?X)I9hQR;d;Y*yWt=c!(2)5ws^lG{Lrfbit70XT+DPJalqD=wq3%!j3;^K zn)9mSz5GHX%c1H=GdS_r@r+kEXGUM~2qVpVLb1hwt4`w|GwIq$37&hJ*x?_OQ87qo;sGe}4b6 ztH4N9mFTOf@;17itHjiirW1odbN)IMJrEi)Y%;Ry5^$gaSNgg&8I{H51E+bNCpJgW zICI$FzOp0y3`bqAZRWVj5EQEuw6)T3a(P0$bmqxq;5G^G4?;FVQyhuL- zqB{f0jZRXCLeRFShxmNREse0kOzd!{jlE6Dr)1wAudeqn^}+{MCy^px!upXaNhd>UxVGAOW6H1SSO*Roo7 zCXf;rb#P$~Z$l!U{_5bZ@?j!-zktT1KM@Cl5mF<#Kb7ytbvUqj$!qlM=956>wo&j0wOYDs;c0Bp2F<{st{U)y-E9})zhv)y zU&rU470dR6+;I>$0_yn0s&f9dc&OnGo9?e15JzYqpyqFd?_Q{}=T1n-&U)SyVks&>!0_J>_rp zbP7MsNh0!w65#;>R8iId3Ju}72e&%%lN_eh5xoIT0FY$nq0O1A zmS;Wgvlv<7cDYxxa^y4)&cjoh@ZbfGQSwV|{TIdfvE_t&R?z3iVDt@E4BMUf%AF;A z+HCw(~1W(~_}C*TaGOP>i!)&O}V)i!rNs)q__p71?X@4+ImnErX#y7zeA z`4y3b-Y5nJtUv5ocOif)9|4rBNe{r*Y3HH;PwfxkddLyQVkXI;5st&RJ?s{%TU;t5 zwq7s7>F=#ihwj^7k3U_KZU+cX*(}Hu&UVXAOD{p{`{5CX`G_R)!!u}9T1=SLPDr(J_aA z-@Je0_>k{fj+Af%QRs&Ampzydm1n@mtB^?*4hjSK-fEA=rgZtquO%pp^Kb9BStj1H zcygI3j9_|?f&AEQqMGZ4w*DkT6S5yhW5||3R73w%^Tfcw*YLg@V%2+jTRBR5MK77r z3m>_uCW#-7rXc_Tt&jAzVM@%$P3S#xCh{JH=|^9-5k(hKTVlUQKfw;A0h0U?$S#K6 z87Ae>FXj_%i(07cAOo&x8AE6Zvz`ytoiTk$m7E|Rt82pbjV^@@zYgMFAWS|i0z+8= zaf3n^Ev-l!L+~!AKhzgLw?A&Vo6dWy+1oDf6~LI#E)TQ0O17L0Ex>HJep9UFq&$lZ ze>P+_A@cW6y?dDV#Cz6KVvpYNLy8f}peraEqT5Z`|J6!wdENcm-%>zAf>ee7Dt=@+ z)~YAaG5wAxIhW5mdMxYll9IY1qW4~*(#@n765sv&=%I%`i*YHg7Q zOt&Z-EA>v(uWT&t@@d5ZEOriupYc?<=@6cV30E&N{a)#=cKkNINk?q{S;Qx33aKoS zuN5H#5CJxj6=N7^O9eSuGaa9ZkfxQ$l`&HqeoZQI#eaXdO;r9MMcJWO zBV!HAv`UC!jity66w09r#^6VeUmeu@i3`uRff-diBr0*~*^cie*j>XJv-Gn+0^et~ z8P?JGenMr5_O5xcKBk6M=1c!>(xliM<6@v)%i9Mf?Gy=frpCAO%Z!QgvzW^Q9adQc zhN&H%n3}D2=JOmsCtNW4Q0s_qYZrr*v@Cq4I{R&G{w7Ll36DI}>!rv!{);{#<^{?j zBH}}S1?}T_}h?z zY7&#}Q5F2n+aGMfwO4<2*Co6v<=m8?s;65k?5ov3BT%w@J4t8Av}h3}(~)y=mMXB~>4(U;{R^rM3q)|Cg`Jg{3HOBze*3y! zK*X@@`t~vV87}MW2Yt?`2KtYTW}S;5Peit2FmS97vkWAJ`UOdghTV+GA`jngZRZ%M z%D;gKe<)3!IQkEu2gO7{M^zWgzJrwKJ#(UL=h2l8rExe;uZ#$nuV*oD3EWqKqJRh; z`%kW}EXzs55$9(|@x^u5L?Zu<@?LcfvmhPVw9A$mceCxl%8EFESiv?f?#pkga_ZN% zSonH?XO!^f1?p}pVIS82ol(lnG+Rwnc(2R}Qj?V*+ETc(mLBhE^^vwfySroPv_)j& z&5}*4*qH!nTIV#2Ti0d|pKP5TmsOMlkRp zW-jdC<zmaPAjzZ+^{pdhy9b4LMvi$67UoR!&vl+I>M z9o(8aM@~Gh-jRV#SFW7`y_b8rr_-}9=-&H5$ z!Jg00vunIJqZSRW220Oa-kvw+*E2V{B*j1wZ&7Y+3qX0d&)1r{uUJRmeuZ4R^{9Fb zpuA}^7v#=j!wi8Y`#KnA;RLd=5a44 zIRk6_r*=T;07<0gca5eVaq&T965e|4B+Qf(r%s(%sk_PF;i5^Awy#Z0fXjobv4?*o zE%!c{+uR67&7aS*w9E1je{6u($*YgHQTkVdSNiS9 zyxzVe14CQyF4lC89LVA!1w(o^`Fh!O#=>T(B5eilu*=2)Ax{uqF%!f+x=+K~PZ2=A z{aO?EupHr;~T}$06z?L*bv7wp>g&K-?j?x$BM{Ia-KqEudX-Iz7 zg)8AZaZ1dvicNm{`m*4kI8rbYgPHZ+Y`$r8_6Ha;Q?@IS1;1S|Q{ad==4%l@Bs|Xy zWhG~Lo(4SBuB?gnOj+hi!E0Ym6m%1F)sB@=MFUL9r@=h?osz{3AZf3PcPc&Ya_6n$ zPwPHJ_jA;0Mtw`s3HY$QQ>^%}xj)x{YWVst&L02Cut-sR5aY0XWKYmoHdWa|R%#DoC#=;U^6G4Dh~MK4m3tQDOaWzvh*+O6^pBcAiBr zna*m4+Dav|bW3!tk`!PA*1npPy@4>%kk#y?&abk(n@pjbl}x9?v;AE(n@Ao_=)29p z;b^}Pf>9O!L(+euc>80n*c?4E|~xh@i%;BzX${zvv2L*HpM zXk~FOzszf?3!_QK56fW6V4jzLKB3c$E2&EN6}7B>B{62t@82P(3KR>4DgIyQ^I*Qk zCXS4+bvPC~y1Wq;ca6$HL$&rb@q8p&%d|u0>X_Xv3}Uo%KXnzl^%cLSov$;*i+&y6 zJC&BCk7W<_b2j|cY;&EcR)NH(#bbtF+n#2JqD7KjUktFBr-=_B64-R4f6;aC4>)RZ zvOr59zdLB`UzYK$8`xEVK*eyZi;x1Lfa!^74u#6H5c(ng!>Hlk)ss9we1fu}g@jF* z6sOQ+Jzxw-m^cl(@07NP2}FiPnOHPtygFQRLek!S2&>VBb@?6&4w6r72NHj1sFZwn zlUfR+kjP^ZcVwTZMo{=p(X-kz|H_%z&cSmpB+e$w)2M8NveY?>Pie1;qV^9~TgHG| zP0&%uS-ovFkLNxhiRoXy(d2?;bK#o+?3$mM$wq_jzX0ffyyNNv(#5y=^;&13@;RoB zP>OG%oLDqH++N%?BnO&ld*!4Puck_9t_Zfot}tP)tm96d^#~}lH*-aBsmgdPRO3(z zLz4hyj>_Cpq!@dD!_r0E1`in}i5c%{HG9#L`O0!Ko_AdmQ-Qn`cJvmLMhlIk>N<8R zCZVD8HBJdou|pz@Nv{3)w*wa)ft8HOyNzvo`e1~mt)yZ^W903%BqklM zB4E%`+*F}ckVKAOuL?#=ms@Uy;;;N@hOTmCf^<-+_*jS;yLNjm_0~AEa+iIJg2YjYuBYh{(6E``NCzA)%A{5xUEYxI_<^$rY4aU zM(as}?=(g3@a<+3zRMn;@DRk-hE3)Z|5#l=ZA3>QPgz;gxiZy#(+{+aLTHm2;iBhn z*5cz41<1~=QFdEOvwao^^}O(`e$FY@p_uEaM&V@Qu&0V!+f9A8dg!&gw`qi(tWZXl znVQ~i??{M@0wsn=(Gp$4zj`7u*Vr5*;n6T~TS|Z_ti{4hJ(*tKMqK)-ZE9bjD85e~ zAgGj88BOJiZP5xaC-+7C`R)L4!5*L{jZh zyNb=(uFY-;g64tvrcVLU$3DoiHu+A1tyz8q6>$C5sJ0@F;ORi|GK}%Yk@l&9-k64~ z(G`!+!uLr87q@ZhEvmEKnKk+(N;wtd#A$QNc7IgkYy|WRA8xj-Os1yA3JA7dYl*m` z0gI;4H6E+aDNBaO<&?ch=rb31Q_=A~MN~?mpABSR?-56>@pIT~HmWZ*B*jwp6D4*P z;}m!3iY7w=E+J11Jc?6Q&MXsky_A2Fq*udh+6R5-j0{Lhzsl*9Z$7ThV~*%dAtfy6 z6+_4emTIt)IwW+Sl(-${_Yj#vpxz5`Z+(MgGv?OoNMA9;Y`$(TW)ist*p2d}Ji&`g zZu^~NJ0>O5FKCmcQO`fKDSsWI9}(ebJtTom0X3&@CPH0&jztch&SvY&9|R@4DL5FS z3CC{hpgx}*))+42x+$my(nCco7lN#w{F_^cdW&R+W% z2z?s2!hzukXT2nd1R;9=Qu8Sru|(>)k2tj7Qz(w)&Z(o;`e>6&G>#@lQ+9pu> z0E;a1jA0*k&e*b71P+UpxC&^BrWRCSSA4ym4~|Dk{n$^Yhlk_a4Hx{AKFPu$hn3E& zjZIH&c^4YC4Pn~JpkgcLr}pbv?3k_4GlLX0Tti(77N9e*Ce;wYvQbq=HUl z_>?Un_8Z!Q?oujo_OBtN#8U-2EO#=kN3TpY5!FBcB6iQ3xJ)r`j|)5k6O2naGsKAo zgNrs)Tlz@Vxv8sf;U-z*^r;K&mXV@@1oX6}IwbY+L)_fA?^X00&w}Vo@qQSr&5W`( zG-=@VVCgUju$5Y2`(L9Hj?js}P<=_J8eSkO?<_{)qvo`?hkFe)C70WN+IfUN^~LOt zwhO80MjxuWzTV4YR6JEwNB#iY1b*59#0vG9 zMi<*)dM+Tz)%4*a920xoRB9z!i1FcCDUuO&3j8lfav58fQuogRbB!jml-)0-d>VKG zX}0Kyfh~CHi84}>Os6G6pD4RH{lIPP==48E;Kl&q62HAtNokkl!18X*N>I@TfKS~) z1r;t-KWWy%>tSJ2yYe+vOp={?qp+ZQb%x`qO=3*=po=uGX=m>H=r|f7aTr+ZV<#5{ z8tx8^i|X~XE3bGUeTK z1A&MyPnSL}&e^pJ)@%x#)q-6TB$W(=JD_m6ArBE{C6zYloo{xxLDrjhB)|Z~b$nd# zh?u_1S1fVTu#GRE|I$wir;7`$wGcnS)eKor#ldBdqz>nst}t*c=QgQ(z3iLnxChk4 z(}%-RP+W+iEgIh&crLy@O{%tIe}r<>zmzygBd8a*M|zC_{L#}Cx#N}g-j+2)?VHS3 zs`u^Fd$UZ;7kfPC$p!Q48LxxUNBbN!3wYelDEg?|*bJ)hci~{vq#|X)T)uTnUsb)J z&80|Jhj9{*9Eaw)Sel|toM-F?(`Mj&F!-lW{%atJBJ0kpF%Kmlnh|3?sL!6Yt@@dJ z=qGym6v|>g@Ci9TLX#m#Xf&jkLA9lv?d5*d&8lPn_j7n%=~O5*ZAQ4(ST_7~d$-iL zRcX9TB^5+c6{P-PPXxBU*){3YYY&&+c{v ziuB%;))L!6i!DlFfe|yE&Yri`-fmJx%O4{NtH z3R3162Lb?LDAb@4730=G14z6?zCAp1NK6|y)u&|U14`9745bJ~J^Jb`0sfq7{BKUK z7W?Vjdf;3xNrjlr|B57$=nuf!>28)A{!|~wNc!qDNE+jTLU!Q$hU5)A$nHo9eWoN! z>T|juPIX#+=42mfvE9;`l$ypO^38w?($kX?`WBFBy$r8xOJgpBvO5%}alH{0eMQvRg?!)uh@pp0iov*>#HJ`G=A(UY9T_GHBw! z2vXyk?4<#G8Z-xD%E*)gG|Gt@^F-EgwwePm>e5nLd_q?R?8)Nn3S6aslDY(KC4~LDpcjFWe zj?P2%DK8gju5i`Km2*%>es|U$PNem!IrrDLj55iTn4YShQi1#0<`;47%;tp(<05SM z=7K8X#090nlK|fOfq+bbmj!E*tWIK`5ve8~L|KP4t?qn|KNFcgf53!o>Ac3PqARD2 zt~U>SRNPPmeRoLML@xKlZqfi5pN2|j%$RK5@Te<}CMY-@8Ga;z>l0||BrTmkHN(m^ z-@x=q@IUR^cy>K%V#mD?!Sxdw?j0#4*BaW7u^QQ-cV^%-KRsccxu!B*teg9`gIDS& z{9+y%?p-npu>OkdZ2Atw40RcAbCO>My5+3FEsqn^s#%7d=*>A~ULblk7987O$fOTI zFRa91)(ym4pKT}3dsCR)^ux-3N79QRK-VXqS9`?Y(JYo>L0m)eoue`!KoGzUdZR@b zvI|Q7YHP3)K=;?!Mj_B)5<3vjwE{eFCg@TCHjp~~>__}5kVmAqfj(wYK*22?7N4-l zEK!lTRMY$jL$DPB7}KWDygnJTJdM(?i4&rVYaC%f7wdpHI&kBvZqRAP$5PL@DMoAV zQ##NQQox3<56m!r1?%;dwZ+Wk{)UgtOOL%~6LH}ZE$&b34DfjBB{AA3B!h|BU4<%- zVwp(pVKV$Q2gsDDG42d5U#>d(hM9i~R zo3UxfQYdMz9%x;Det0C+v5MjSinE+4dcs*=Jh4wZJ6};${iHe%?&1;$o%~{tb5}X=kTGn2 zoU?CGf&8|z)caL12#mJ9+SaBKe=)IpY(53DgK5Mh-XczO3JTFdW7{rFh=(7`cLhoX zTJZZY0vv|hDY_wayJFR2~DhFcb@iXzndxt$Q=`tLaUtv<_TlFOBBv zuaq=r-y;!VE7j&u=4M>{e;G@E1s$A-cmT|IV#AYo1ZE z`NKAgd37fN1D6)kns3yj@RokT3oxIGLb?ZP62+DQU1gRKHls6kFl`D%iT-Xp95jjF z-$*0+c3KsPr}8-DRyxNtj!Rb?6-TfEfNBMj54g~9P12h)(wMS;z{5*vrPXLut( zP$HaLCAZ?9!xERm5Bt09E>+d}5!z7Eu$zc7v)uP|9MET+7bY}$Py)5v_?xcYy8)mn z)c&Wd-rK3u+%lx5E<}Ql>k4$d-h>Dj_fVw#AvE?)kK)d9`B6&z--my#4&qS)RC8?A zl|@yYeMCuGvQuL^<1X5SZ$4jFiEK7;2>vjZu^#-J789)b--2nf8!#J+b1)aJSkd{P{%&)_DZ&g;*$g7)jtkT(HX`s7 z5%+5VY3bH0Z_m$=z2pMJ*r|id=g>oqR?QQ$zAECh0z1yfuKgsf= z@URgmSC%I_{W_<27_--^ta3Vqb5ed6-SxVloUZP*l%tA=PxA~qs~frWeMTw{9h3gW zUn~||`*ah8N-~e|+|V++W`45_vSvHejD)!P99-aCAl(?-M4v6{P0<{N-`oZ>H?T6x z;@T~tb$S6g_N^mmWNN6ynQjX#CmSs_bj}PcF7+Uu;h;t`(N?ki4Knzx*AcgmMr4dPKgm9@O+tzdC z7TbsZb)!So-xyGN5S1|RCtsF%QrQYrC%LolE7V^9^iXl}(o`Iyh~{xuD;IzMSyH0(2GC02`cA?sb9hlQ-ZcPb@8M8*t}eEjZInb9Pdb<|1K5< zbb0z6eQ4vb;>ITb4Rr1z_7l;!uA>y&qFlU(T{*1=#LMSz@BHjNTr;G?G^ke0r%X|wRz$;D~Ocb}MhNbgC1d9F2jyzPajKwV zt5;Q?fU0E?>y~#n1^(Vt;~0cnV^b#Kuy?1+Lqz`Ld&<*fIJWUa+b;-R}oCzqjw{MXnU z3EBlZaOr0X0cv%zG3Ael76)-?JU)*)8}8%Yh+45%Nrn$PZ3avAPzH+gu&Y=yv#)>! za2JN)y@?}-u<;#96GRw|xONnbvR$)rQc*lRz*?ekb|j@@i;ZL~m#bw(URHA?U~n1q zG;Z;7ijv~sd^pkh;j&4I&Ff$jGXUV18Ns1K7%;JH`L@v?Jw$vHLhHHLgC-5 zB(#0@vD`0d;mzx2_(}GB6RNE&C@}cwbLOA}Exn@BZXrAj3jM}lsBacd?sntVz6Tk| z5r{PIrbU=6m2a*=xH;}y=&cPN*Cjoi0k8mSQ=zD=$%L%qAnWcCEZ>(Su^{Kaq{tc5 z6A?k>0c_Qf^1?@(TyJd-2)x2@;Z$YAqme%gN_z_w*aLe(>S+;j%8&pJp(`i$S|=#* z?M6VDz8K7`0{8HgIB)q zO=`95Y08yyB^tcAjn9)oO46T~&e-mA*>^!U-RQSf_PHb^rmg&Iq9H5d6>~hjOQE1j zXK1XTL2uk<_XDr1;0^}DgrVobDy+Tbof)5Lde*ZnW|PV0XYcI1xYTBRMlKA&?E;C3U<`-#-u4D= znrANcr?t_RY8tAAlcRx+(llw?wJRA`b#UwSRbBJ7*(J|Quxk=mW9X8NqC25zNy4!= ze+b4h%muFVC5*QUA`zBFIVH3CNS{GK<5m}fvBBx+n?cmFLyH`)MDAZ%H=m@~_dSeN z)_x#cb(!Y|j@Ao>E@HU<0V_b%zxeB!j9kxPGX$p_|H6;48C5Z!A)pDQ+BWe++)sy&VWvD}+X$2`S`G^voR7rrI$@w6oX#xVM0f0doIk+F_Q@d~K~^Q*7ksf#KR2GhqYlE38EkYsKC z9TI(_2GRI>UfJIn)1WXq&r1{ioLd6Bn@E{4kke}^Lt2N^6y!aR$B3Y?9iSgg19Nxw zDUF;8>cJ`#3LdRkVoW(m&I;(_q)kE^d|!9$u<))O$qH_hh4FhSd&@-9@gC`i$uADp zKEQ=KB{_diMA#sttaeg1LzK{b?Y3?#>{D>yd714ev+t6d-(}52jH+TEA(zQcdK6Q* zDDYNmEEBDvO7J;lvV#cT4E_JDOiJa(mks92y;H1Z=kW^63rT=dQA0(GPPRBS^h;Eg z#h$e~v^gy0)cHtwc-L|!>GfvS5KJWTW5%D>hqf;0F*#W?)caLo76Q)XQi;pfomZ1j z(qey<^pqv+;;%zA0c@a{rWLvn1o`o4*vY{nvMDQn2(eUcZHNfr1x?Fr`N*3L0PpU! z$2vTX>m6*N_81GJADQ00RBu^Sx{uzZmyzo7y_d3Cs))j@R^avcJ|s_B!ghtSwqN}G z=X$tbGwYRAQ95__j7B0h!G~|F)HpAR;(TsofDXefONosG!--j%ltr5Qcr9K%EbQx7 zU@NLD1Ok#C;V}vGziZ@rv5&(-{9ScI^f+pFmof7J#m!+sk}RP~v8*z%Nc)z>H)^f1 zrDF=jbxyRQs*)&sYvX9eD^UeH6F|n{ka_UXua0d-=)B$)&E{GuWa~H;$CHFnJ#oN1 z>H<&U-y7s-Fyt7tcHnk15%zJ|NiQz|5~9Rc;FuGeg$b2%D!Out2$d_$1VJw?8z}59 zeT?3XEOUVlRbHdE{6^5IhZKQ(_~mE(i2kS%9!N~pb;S+uPfS_R86Wm`@d0o*mxQ*H z1jd$q`}00CbE6XT7k=J8lG`2%3PI&6d`3}6atW0)iT>~%dzMT+VRW?o_)&lE`GA!k z!|_BGNeq5weu8v*wL?0=*>ibigwoupws$5|1>9bMvl}&RQjdj5G#8^55tPO3n6H)T zHzp+(s*=!2SIpK>X}qYN9g9N~5)tEWM6#Xx{0p6ZH&PrK(VX~!`YOdRlpioSh4{sI z7bh05C$Vc)q<~D;ItCCmaQvkvP|+M`h;`|TvE-v<9i$g`pJZ+}i&_p{&=n50QhFXH znCj)bg7B}9=)oV1!PK>xRNtSQ8>MQ#I`;AF2YMzd1q(d&)%(tCT3`LjHYZPWj5Rr^ zM-}01sU_uRfAHf7G5!PG1ZtC%>yVYAF@`cd3T4#@-w~J3IJrOGSka=?M2kQaZfO$X z_loS1tV6=Vwl%q$ex?H^_GXV*(kH+=>_=0_oq@gvB|8&4aK=>1U|FIQ|{49 z7L~qx;Yyr6SvJy(f-YU|FB{hxaxz4;)Xs(;R@4nFF~|CU(|dsUIb2COE+!t=*e*GUn(kfK#_fnO1fvk;iG zhd0`WDvzsWo|p~V+hT#ttOP~F1G}bv*Nop0J@@SY_{n7C@5+J4MYTpTRIW=&=cCeC z4FHjJ(5Y@{Y`l~A=HtztnN+8bQ33xq@jNwgB#4dmK1iiBKla2(tQ%LC{sD$aOJffk z4{MoNTp{j93JM?~MmQvUk_I8J+ zGbMLi8%X3f1ZQ=knG~d$RhR#8xM*cldVdbp{T_dBYflW#-{YhOj9@rl-MPEUA9RZ{EMq^31{=QwLJ* z6A*g}y1IVlK4Hq_=^;~#856RG6!qFXt9H>#7dC(V=<~EYx*z**V*Z3*jWPVGsFIoDYlPDm3Z%Se-C8i1T)u(FL1i8MI;Lcwh0i& z3*21AkPRw`!-4T{&AJnoidK6dUbqoWH@?2WR!W%~V3d}a@^G7fT&X^|@gQKd1=` zGy8F1CSNqW+lw{zq&CkSbti@0T14OMB4(cOxSOtrlf-+qL@Oh%Htxn%NSB+@dwNTZ z1X-{Cg?1OvqFHUrTp%>Nyv7ZwsYRUbnnW9MIq$z#BOYTS71>xY)ibGxr7FYbJHpf_ z`mqTTpvO$GRGX}+vCl(>j+foEN$kBEjBR5V_ip5{4*7kRaq$Rl+&Q~!zNt7a5|Wx# z?$M+l=l{k5j+!etd6GzFNg>+)vxZ>k8uza0$C(W{;|dNUTE~?I_;;hN_dPow1WvLy z4X?;l?@QJ@DCab=9^~fxkd3c#|0qq02`}A=)Gg| z{JXs=j+Yd#lbX9Tm>`geYJP`SXLa~y0OCZgn0(|E9REH-=Qn2ecGIq zmLkG=l#%=X+x!459l8`jTJ~4ZgayN;k)_$z&>N>pDD?}()8stLvIj0rP)mfU32Ju} z=Yk}E6uF-Q-}mNnW{hN2u3#HY52Je_I3xlA%pY%uAuHC^J9RM z;9H;dmx+27l-1lYes2jUe-K`6dIBut>w)40h~_+(>&{fTvAU$Q~7osAmO{U8LUE-&2F2^DDxCC(;tx*?ACwpHcdpt+Ui-* zMpY7fh?x6^Q_|vt4P{SrKJ7B%7FW$$BUfa_L;cN@*5nY%7hg%miDj5xwaM=>jU^3t zo2G|V))WvG+!nBprb{`2P@-*ziLMEr!UK3vz&=dj5JaXYJ3&@2n+6=};eztIK|T%M z9&ojNoEH>~I9Qd&dK@siDgq+;dsNv0%5B9rRzw3? zNZsg`_vQvkUuymttrIkQ`sP&j)VzIrOWLo8-QhZOqMj#vhnI`{Iq`p@HRpqL1lu_@ zYL~Kprr!=gnxgOU(DpVGnfU=O4T81n`la&)T-=t1{ii1`(4;`_)B;f*oDvdYMoCZY zdi}4CUklgFt{i_!@u__t$Q-$bX;JH?kd#FV^u(gQk@|BL^i*J;wK@9=eM8r|d~Q9p zAi+i>*oa^{O5_dUPM}AB-)x5H<#=XA^KAHmzF>d9BDd9LY0= z@-+pH$4qH%*8W$^ zCcHjG?-}(KX$HTgAg>BaGZcGeH>pN%1;11u>G6*iZS_9tB10Pp57r)4=jSfD8r``o zw4JlWL_#&@2gT)+ZIS=+$%c=$O|ODg z#5?GcksN#R*S6@iUd@FfG<5?CpNJ&v%cPKU>i=iHNXW#=eynv|eBVwuw1K$t*Hjx} z92f-_QMkCodf|Z+=N%*HJ!Nap35(cf{V+6uc6dua$k9*c+7^eynYCpJ|FOV- z`rfM3Eru)|O|KPTvwK?>yuc(XHd|aE;|4$|=MW4b7X!M?ct?l_X3xAK&TYseRBy{D z45aTBeqdpdnwc!&ywZ8YizM&IWl1ixI{Su{K&!XsU1iM@09B{+H&7+pxh%O}T;!T) zl(60j$+$wYJ_K>bO>eClH#YrM4RkkijSubNGR~qh_$%;TQwhDt=Y6wyatdI0+5VwN znXtafc{1Y+q>lmKSH8u9G7O79WIsRGK^2w}ep|B6@@2lD=)LSNUbqIvc2)m}_w}4$ zvm8-M{SJuU;yQ}`1g>1a`mA-QUS1SH(XoC_uL|6@htSbn9W!zc)Yq24G*-EcPeOJ_ za!I~w3t~g7x7AI#MP_fJr5wN7@FqLIS>iAl_?mrLqfqny3yyer6#3L-zpl<_wN-kZ z&fX#OjS+^mw=JI^k~t(MhNgR`N|r@DoL6CqJl=DSOQz}+fI~R1v{AH1CT#|GH5aNn zt0@>U;>PRZ7zn5Q6ld)(br6{^MIz)a{pHI$G3L#uQ$q-fgSIV}p4nZ!Qq$uxz=_Cg0SI!wzQ6zc<3X+e6C1dd9|yMU84Hme zey1q`c^)qH&&e}oUe`!o@ENKoIlJP{@?R?)=SCLV>td^ak9trjm^&ZA#PcTWRF)A*#xpO3Y@sTdJ*X9AW3vkce#yBEMGyjxR+Bmcp3q5$;EFx@y@jr=O)jAV zsEUZ7?IR~BBZS>lMfJ$OSCg{>HL%fsR0MSjP0>gyAN_iVPezqfSSmg;JC7;L8YZRIlUH4DB zjsFc~c+3{?^=wbRnrIZk3vgJ?1IORl5Rz3fNT9{9svS@;r)uA0yt=`+Y}$OJbA}vk zPN`m6$u@e{H|Z7UNAwxSRv-5gMKA%6y^!1*?F^kj>=7Elrd}mQa)-GClvlXxzuXA^ ze=|)tSAp8s{2!#$XP~X^)xJj&66ZtrRW<|KZbX;Mn{h53&4b=Wab{RlK0`K|BnCin zX{d;BT{b#ap^lIAwQ7nPN?Nc5-y*sHZ|Z=G$9-~6%`;80oO6DXg?Pz;Xyr6&9;F=t zv6_fxL?NKvs!Xl&D$1lJxIHcL#xq6W4WOyjJ?>%6V>o@seH&`7v2TTQboLa}p)H8w z3>!~{5IX4Kr(MoVg!#j`xb$%tL!9%fN8F}JSvX$)O@oMkV#23r3T8DS8P{t+Nu^j3x@KST-%>rt?zcY zZcwiNDMss|U;yXlz^0od*r(ZP;ZfR4spE7iNbVnhHB?R4A z(CWtiWI;|fx0zFgyo33n(zkEFDA7!4-gXG70bZ&{|He4+TQ2f^m_xDhs=vgJtK8y+ zr`>}cF{>WY^t#$eWB<~$k+VxCUHO^}MImeL*C6!4VtTE6ixpIAO2ud{=pAA2Y|l2hFgy@Mh9TzvlKHIg_A1%b zCy#UHXYPzA+Hn61)v_T}_0&cE+GMemOGI6cGl87}VEz^k@kQ5V77a4Ir*bzuM?mB! zBX&MZap?Jd*Sa+EsjYfWhW#n2k`>HQMi2NRt?;h7S(8EC?^E!(_Rk0H=0^Yb4%bbQ z@f~;>F>#>ofG^kipzzC;{Bz4|*y)u2Wag7Yq?WwUIJ4>^zw04Y+S(_;Smy+?5jmtw z?l2E6_{+Vvu@;`)I91DbzWv(!>PMb zA+N+E!!+i(rLdcH56I(o2hDPTlb{||OFqu*W!QkK1fHv{aI7X)vX~V9cTqeR#y)toV@9z?5*A+uV}ZFb8jp-OP!*#R*z8d_I5&Y(3V#1 zjf|r4+XTCphnR5uWBHRyD7O-(Y6;-G$V{3d%bJRDoTBKwg`O>+P*78U7xLGRE8Qoi z6F%f+DnIyaao6viFE}(@xFIY6&MHo*9=BdqfF28|^ch2M2xlX9i5w5cbzt9I}k#&^D|#IR4YfZUg2Iz1ni_%Z%IZ$nrL1@?1-w8@who z>;GCgy)VEEpb7E2`Dl?a;vNjVJjt>)LQJcp)t*$2kCF2|^t1D_MtkDf>)M9RPmJoE zhjo77(kZ0PszDq9keW_WwM&FtcTIlOBrJ2T8fl>I_@W>Zxf%~hPCu)0LLm~9PM*% zP4VwCeP^?6uH3$u#jU6?jwjMoQTm6Ks8WQw`o~_>i?!ET8}xJuVFlm~ynxtrFx5=4 z(G%iU%W+!S5kXRXETeon(@X2@)HEAA?6;MISkE1$7!hUD(KJ{H6aeXAN@}#yJ8o)P zlhMw6u0zqL%h9#g@E-&EGa#=fyfzK;$vEIAoI+VlVOoddl1dTKqVR#R4J`N*eYJ zHbh3Zn$=~n8ziF;7vLIYwTZ!rcHebml4{O6n(xC|H0o6=X|FZP`e&4SiIBsF%1wc} zyIz=p6yiT55q*$Z=NX@3;PG*(|1I}v&K zS;e0jdi{_yAzONvnTr28LtD2yr%d_!HXf4irGd+r1$ac6+GiRr%` zOW}XI*A+m&4cCP5EGLt6yRfw1(R^!937;urE>v zMOOJN8MS5D^4Je#8L(JX!*9N}jPm^dgz@r)+}m78%p#+pZLhu{zcwwq&Q->PuEk_{ zP`m1KSnjGe&jBcS$|9CgK0=+k1tbNJm(@A-*auO+nnN(D+bREz~cc;XC58{)123`M0)oT=+?_pC8p3o?C-)#zmnm2FVfu^TZjA``I< z14_csby16|zA+#o;KmRLz=_2!`&E(&A}k(DseVk%UXx*M26Mk6B0>7Z3|G96!z#Rj zz(#i@vOfv?2;l&9&!6)7@!bN=f3fB}AWzC@k6N=6x#qrX4SUiFNJwrEG)Ajr6&CNW z!UF3jwm6(~8{X*|s6LH(#2IAb_3~)YX>=ci8X1B@@o`+RDLzk1mc*0eoo)_FwA9HB zHUioR=C7 z#_Hi%e4;zHACPrnx10M9R%wX*!>K|or?*=}tV{YX!!v9m3qe=*F%VB_<$2#U5*_#| zQVQ&oZtxX8C+pZQ^n(L4tQf@kAhO;b*>iwr3!kQ@Vs29C?t>aqphEdh@6KN1$NV}w zZ<4I`gW))cqSU8p4(}YARn#3ujvOj3>8KQ!KoAodF4$lfFJe%akr}eZ-LGS%AOh)N z6~bH!#Hv}kCv4~D;+ZiS;~#@>MHLU<`h}`6OF9zPQ;+KHp>D=7N;7+bQKj>0-tUyxT(%!}TsD@2EQW zS~U+zT*q+WHOO~+F0!?os0@@O@cdfVR`?L|C^PhocsPc+e>CV*CgD~(Y39Ft-tGDO z)ma<)?(w}1vA5k}K@YcpJ9&g5^Wja#D)v}+aC6&2ZrQC&-;peRrQMum?)Ev0gFB0U+4}UXbx^5+0Bp}>J0*rA^JC&c4AeF<&36x zmuLIPhT3Jol z+yFBk5TqPtvWV$izzk+GXlsTGs5+s9>Pqcpjj1&H)4uWz>a=H;z379)+((GrrN{afJYxs(qSNS(w-b%z4*3C8`sXNdI7z&&(wt1`uLheE>lLjOC&nSc=Yf0@wYA&ezOplxE83nDf=bpZfyyAO(5z$Za5Vkm_ zV1{PI?@L&Lf$1Vo$qOsUnU@tbnVw#JTZ)QQawm}tlg-do$O@h?F}VhLvsTQABzs&y z;UaO|WTvBzSJze*s2WL*Cz+r*FuSRRAqygHv~ewo5B|259w{C2HkH4Ze$lh)Gkr&> z7~7u)@Xo(P78qrVv*^|?3KGhkQwuF7VC@RJ7mANohcPD}4s}a)>TN3JYknSt(&T%nJEv}L9` zfm6e9yAN-sK5l&AgS;R7$@%AcqS% z0j&@1klITI7_F@e+?jyqEwYQ4je#BbPpk2#j6jtLMIFi)m$EHLn!uh)bSPv%+DY-f zhn9{v-%+6eKlLVkl%e`fGKnj?1;O?pT5w2o*xmgBuxwy8JT{T-i(+DKjaKK@cTt93 zj8=2=a9Z>g-Z^ZY^s%62WaDqEUtMh{WpWmwOaqesq3D8!^#}Xs<1WIr%WTUdXCT^! zQkN;nF1j}v^*OJFSw;PY6F@G-(P!It4^T+RgoPgaOH+-hR%axv*rh0dd+}({qmLcW@sy6O?#U8$sWg zOwgaeLC9zdIrk!xTmP<8G7J%n1S;w3>hi=Zz21lWT7TqhNqmVzlf^L`q{~#ZHfFK+ zv$13@-x=g$vQy>VKn7hSgkeYGj{rRyjSy;Xxv7Ry?yN45tn@G%+T3{^6WDo9h?R$8iUCB+bpK80k+OC@7hJ<@9=coN(=zf9V% zz^gE1#uQ|L&A^G#nQ20^DSxcsyeA$(CrPPnDak0+63n?0rvWd5!p$RS#fV7eUn&Qn z1-Zt{h@$N0L#c0%X}ABUR8uiI0im0m{MBK!^tnwy{PT9pJqj@3a5A4q$`tu=^M z1?xaRVB-BlhoiNEBD@!5xkTKkz&%JDiK+4n_8?oZGOw5mvE|W3n7B?Y%90Q1vwDd? zvoO#)O8zK!oemMx<6k8ACE^&SefGj2Cpn@5k?Jbolg|W0i;5&!8F1G1#3SSAquSO4 zl@Cz~&H;Vgd#U|#K{1U4r!omQsyjf-;N7zXQtv{Tx^YG_w7<2!2ttOD)+|)xD8!2n zW#BJGdb^1qoSE7h)_?Tl0Jc0;O68j_;nF1oerzDZZH>`>?Uenj)jNA5wXR0>=ca6{ zWqGpKa(*d7V0K?!n~m+T=B-h@8aI#m>-DX zfk34NUpI7p{E}OC3Z$^rIr~>4SV_ZFWR4$uOJ$Zio6@Py!v!_kPfDEnR=p$Le$pwH zcYy6}J30EmR=f0|**jK>4Vq@&$Iccl49pKqWXx z>#vYC8BWi{o-@u_z_4r)m{j;doV<4P(YsYjUNJLc{)qoTRcz2jk5euO+}B)YJ=Bbj zOyEoWT=3mmQE%BzLp-l}+%60$RW_IPuu2;x24nR$W6#&&7wDnk0pJKuSfdCt225K$ zk^xlbwB^(3)b@;8fZFk;-Hh!ejr)`OuT5Z1_~Ce~N4B22+d)70yK(2vpp=2%6-v8h zO%e(U=_<5DUuEoFzMX_N+Djmv0)W9Pj_)M)gyOs!iBUt{pI5wYsKg-+TP7=E@9g$SV+?dn#9B~Ubeyp;Z|EZ7z1 z@@_|51-e}%d zWFi?{i%dN#H^gADRhJ)`yDi<3J7Q{9EqbXbF9)nnNE4P-vukkQ1kfVZ$f)sl{zorg z$nsA?v04slAybQa>wH0d`U!BT@cV0N`$`kpapSGF79}(bJp%U%mxVmHH^n%SID6gk z+s;VHW%=|Jj9jTd=4#EOYF;eKTFuwNaL>HQrk0(rW23Lfv6q?w36XZmGrwXX&JTK+ zIy&r#1p*I_Mux>V7YK&u4AsouBERdAP7NGi+)Xl6>&vyHPra-KUR=tS;xtq@Sl)>)!J?a3?E9l+dSC49o#u@JIHra0 z2&@!QL&Vg2%r9%mg1rC<-+64>YAhN1R?zN6Qxk$K?Z(#sZswvUls6^xCgs-zw#Byx zkyYIusF*5B!U8b_#ZJbMKaHqS_Ouo`kO*$LK&}lN(hY)!cwu)(bc`i@+ji{@t@=MT2xD6I+U#Qn9Fm-`a8rh# zHQzg)n}k3@2TjMV{%QuYy_fJddG+e3>VJa9w|K@*(s0_a!brb|DLs^^7$7;e9GUKR zcoo8Dw!Nu5ilxrKv#ZG@bL|C-tWE9ub%FNX2U08YBGN6l9;dyR8BOb_yZp#sdWOC= z5^@tYQtKX!M_%tX+gdlgnYW&OwwWD?u>k8D?wglMV%#0gT)^gV}l!i>x(W`#DQd&)sabwwE z$o^AT5Cfi(t^0SoH<_u=8Zn=n=|f%vTLvsDY8JzIABHu&q3E{zwZbTf3_RB6*CmEL zVT%gZNlEZJ)TSI5cmpWeW^` z85!DQx*+NK6$iZroq1sMq8Lh-Ux$}jQXitttMXRrxCgkEdYSQ6n2SpMeU6Q{@ZU+I ziR;FpI~@;DXVAOB&6ML&*R?z>&9_3c`V%~nKRx7>1E{rMq2IOL7+iCzI*xQRZ-7&h z`e&Rx4)-heXp}gmB~-CF^kNam9#OdB1Li()9U;iZyC)*i0IGuARlmq2Dh$cJ3bTi> zMX>SS!HkfUOrlG@K2A{**TyRiGSF=Tz(*lO&W#6l9U_{L)JCg8diu;8dUDIW(_IGX zIR%fUfJq0DZxY^KyP5XkZL_M<&L?N6%0S}o%P3X5FXk?xDlXU5XcviNQMZ3cE!{LS zCT!-r5olylL|kj16$*n?1X#&JH6<3S#=cEF5$Qy1t6_zFU$;CI2e5-Vx2sDTSHtY7 zD4;BIbwrlZ+M$LzeyzB9w9_7auLKY$M^!I5=XQ{aDF{PZgJaPPq5iFJ$%0EjW6k=m zrWr+1tDOdFF%6?Qd!Be)Mzlq|Oz~Ufl$Pc@{C=gi|hTKIF{L20OK%5wxCH__fe@vnDdFT zDDQ*mBBK=5v^uu7$?y+v$sgwujEay>Y9|Hyi>}+Mz-X!;m>$4lZOJvkfy^c95e zZ2HzZqFzEHPYPy|v zdVr)Bi1ngV%wi3Vv2{cItZgw>e(3oyts5$uq76pKhyXhd5KCB2F*>7A!qsa;ff;5Y zexft>)Hn*CdlDq@@>td`I8THvEk_N+2k*0^%rCNE`Z#Zkd$@(LZO|)1_&C_uhykm* zT9ut23QGK_5Lh0!6iexCjgl z50Lp$k6t>~u5#WMwrJkX00KDNvVZ>z#&cL&ImQWo(_qohnAVoMNCpa+PE4J(Q#TEW z)?1U0C}qw2qgwI#olUc)tr`%pIfBkQJmV-sQ5dP@hg&fZnPN56@dcOqWZMfFw z!5dWXnCIfJyo>dSE|j?_t@a^_MR@Jx>_p*_|IC*ALY7NXP4GRCTArN@fQG4=e zvl$g{*NZEP(;VN*5l%{qbe=W_uV5G3yiFa6Di_t3Y41+e7XbXEf6g?)3^bgr6ghJ~ zaHSIyg=s!*s<=2m)5av|E95|Imii8C20eF-!_9!H6NR<=hRQ}`u6Ye@H}8RqLz%Uc zK``IEY`?uZ0`w}JU!Hc0sk5yYiVb0g8V%>D%jcDz-nb=dDY;UK{ZSMHdylPNbY$eb z+B@*u@hTXNX!IXCV`lj2-y8Ae5|S#09VosmvppU- zCa~~*{(lZF1w=^XG@C+fim(|;ZE@7vtHB+nKVHXLs&w%r@9r*@!Gv3{yjwpdj-BzK zvZ+wZo2s^HzRR;gx3>|dHsI5MKeU7ub@9jvLoTy0MCFXgXv!A6#SAeM>LqKXMCLsg ztf?LMW+&O)CR#gFBpVVSL8&lV@4E_`)rbEdUb>UH2^tW-5F=vH&JY1*fvrx>E6b+& zHB0(Bs!Gcb8P;34vII_WA0QM1YG=cw<$JP0ku|JEolvTe3W zk}4#^HMbL!;SV$$H2mg{zx}uR`vBlwjq-1&gxQM@W3Aq!6R-(5wn7c2sxw{~a&RJK z8~0c5oRojW`Ov|34_Z(T$SSEWyc`3&uj~}gTsvxsQsc*ueoWeiv-y2pu9umeyK2cA zp@4kI#D1vXYXvx>a8Up2?+Prt*@1m+);VeK|9T$e)WhQ-dl4M4m=&(9?WMAP9A3#o zq!d@(3^lZ!`ep387a{Zt{8O!YbB5&NZivruc8jyeNUzEkoC8b4yKld7AQ|sdG0nkp zaS-j-sU*9aV_R&I;8SK`#&K1Y7Wt+^zRR@y-almc9N)&+T>r^)UsJZV2Kal1q2?QK zk?13y&se?*KCn&x=OUNJl$R8Te7Z8!(%|VP_vjGFdtRi5N#6&+ zr9w?TE>}574} zT=pqwDMP^ej`CMs^XhgcONb)bkMT&zy%KS9%}v)iS_HEmX`^?uJ$OZ$EHeL=lX~W! zy%_)*rDKh${)%D7gTlw|Fq7W6P+&!Of@n1h;ei?KzgZQf*loiO+U7E@)=jBxJ->+e}(EXP_g^Smjt6PP&)mcM6EOn8fOmte=i;0!V8wC z&%k}4OebC;Rv~cIadBhQ+NT#NQ7(YG?bmoACu-M1VX+!pAWoNXC%TAs-P={MuyQ;by{mlLLWjbDwANaBMT#6` zf|u;0eCwbB#VmQvDakC!sK9_19JM_Lb8E*&U_it$xTpY~d3HIoqSvoC*z{;1!)+tp zqj1Aul-`13i=*UQ%WbX_&-=oid(ut@a$0Ovw5_nqY`}hKXAGr*rN<}A6qNx+`v|QS zhD&}1N$DL80}d1#=)1JgjGV&U7r0_*CQiFGYAceGE_0+Omd6~87y&!0E-^rkE82n7 zgLM^#UE-TAAYsy-S;hab%d#8p(oUe5VJ$*lV}eUjL5w2jO;)n$0)M$tN!|Wk>zgmV zh*NV!n8xf+C@p0bAhAXpd=@^QvQpZE2CCaYeYaOu9kq(jWWBD^=$kho5GzB_zn<#AEcH+L*pr%DYm~9WChD$-iLJpZ1ebXPRK4yBDsJmuk_{Zq zS6Qk+87k}3C(RDL!&|M{R^#n~`>|R+$zeBL%&@zmOs5xHLqRxY{c&9}FbXq7PsH}s}{T`mB zM5Q-wTcoD27l;;{#Mp|N(ABB#uH4sTFy!Fo;w&98W~GwfPTog@m)E$;;=x3Vgc)U9 z%oNE*kSif`USL3$1;=o!DX1Y1wO7bJWufbYrq$i0RWyK3^0IT;g8OMLTW{|RW`;C6 zV49ubVVaFNrws$$NC~9FJMwt{qxIyT-cbvxX^xHCDCt-CLzD0_h>v zjTbdkCBz>VW}s&oH^hXU#R1_H8PfUpmLJygwep=z<>T9m=i|G~0!O7qRGp)0%KEJS1*h= zeyLs1aj0x#tAu(3;?OTN~nYOktMX5uD3^?qUL$fdXP zynj55u_%YlbcDl9$yin>wz8M&LD=pVpN#l~B`&$1K{zXn`XX37i%b&tsbNp(-K!!Z z#z>j`q?P&<(4U(MYYBE8uiO3%sdAr%JGf0~AeHdgEt~-O%h#zPY${nTdOQxlG-kkt z`{};1ib}DrFv_U%#27|6jdt1OtvTVqxuC_zc%kn)MHq|F7v(MYbzwB6Inhp=aEhVM zpJ=^jZT(xkf+@w0hd_r_9xEqiTj?*mIEM(bS4oqg`^LKJFJNKQDb&7z5=bsLW3Tc1 zVqq~K8L-4kh*aOVclw}<3kMUYrly*hA?OK(+}Kk*JaX@wL4%JEmU9cx8~!^S3G>RpG(9APmOFFk30IMKRhT$>PC46mjf^kJXmbFcc5 z%F9|)eA!oGVX9bjGQ=akPf6&;opRi0PN{Z~9@RQpw#BVDzW0^ffe4AW;uPDdxv zmaHSM$(?C{vFVm{stE=?t$VPf6oV)n8(|!Lc$c!YU=h(y*|WiiM8?yI9w7(aacqcV z5#+ND8~WX)C<;&B!^6H)Ha}8CsPJ?VVE;wPoPt5QO6qd!A zT$2@%oVNdh|GE({JWm^tjQF8_>+(~iU!q}4a5PO4!GLc+*#>@1x>WJ&Wi4+$rUNfd z1tCT*kI_=1J?C5wuMkY5_qT|i@7z||;wtd(m(^B6>7Y8@=34^H!K1J^pqf9_yT(8Z zmrkBXr2rsADIZx%jr_M@wfssRM#F*pR~Mgf+MZ5|uDTf{Cu3Ie(>b z6d@{FTzYOZKgiv=F}0@5QPB;F>%#v2l=eCzRaS0Vo9Yj!=!>7R0sPZj4_Ai~3=t?M z)`itxfmEw{mT}j85SJ-DxEKdGq)8W#PUY@t+2Kos7f)_yv!4h@DtdD;*Gu)}fdD7m zz88hb_I9Ojq>k$We>AUKN5p|WLt%tflef&ii(@x8H(i)BsnHz%%6D#H&BwnB=aseQ ziXR#`$oL&W%Z@;%W8NpnvrtYVz55Y%d6 zZK)XK352j8NEqaM*6(#jGz#9-HtGRdX;F20w2ixM8A#YAKAyMZdJ3u511+%S9l95# zQKZ#2%CFMW;r?@WtfscvPbWUUf#dwIBA#nAO2H(JfaQ{Zs2%C(-7=2j1@GH}(%mxK zbtaIrw+Vi8jWsGu#dA!ypabc0%zy=a^61+R-f-|tVzbyQ&3E|FM8_j&G1_H|!2q-L zkG(|>kto=e#(nUp86$!ySR(HSal;l9EBJ77IkPqk?)8PZ~~sCc1mvi19VeACE~a5SNS!Dfk!GbF3+Q&Tg|QDjT#zehu1~|Av8wXLJzgikkV3UQe>l}W~wo_xE*2p&HP!P8C{b2hTyc@+IU}5 zsk>hb(wfUc%NaYmO^YNX(QU~JxmF;5v^pu6ib>!@K4N^-!uoEEYbZ+H<`lUXXK8}` zGS$D$;=7wlWBh_K6m27eOK07$cefrJJ7E;XZur1}J z?)OR^*1S}`f$X6N%A!H(uRa83;Xp-8V9-FZyA*eNt#Gm!bGiYj$jt(elP^pHb%rz> zXsd&KfRq9sP!&eBLcLJ*`)vI!w)cKt*Y=KxHLyPhpt+hCjg6dfMs8mj-a5}8wihb8 zrg^rdiw=Z(M5+YVYMUGvv&Q?|!D9HUW*7nRfG;>$4uGC>L|_6`B<8GdjI!M&RgO=@ zXcou0!Y0<|gcTfJBU=FOrN-NY>5e!$(}UDV?YXJHeZdBzQRtvA4bL~s&-Bu9NZdM| zMUcLwIMrgWay{lp?!jG*tba3WKUD*plNuVWm|Q+seBnZ}bec>U$gy&U(Yn3z4u{8dDm9c_>!Q59#Nmo9P?Z=J@=@$MSWHzK$&o}!=XjZTp`vhzNE|XzvXCe$|N7b!t4BBI#PT<*jD>1n*60{ z5`)vt#n;uSX~BWd)m-H+F?1+Biwy*L^%#M`_uf~;>90iy%3uW4gLD`EI_Z)8F55$3 zSu}8&T?d1W50dljy2#u3$vOiX6WEeN2f{@ z38?ad;^OM&FL&5>s56OWwC9DK^iJxmZU%z?2s(=Du%}Ix6AFeA@v7#2?fU*1Qx!j^ z5GAmD{-X%{jKZv+_P)@5k2m`~{8|AMT%$p9oXgTD4xR52*GpN8)*m%#J#2DCA`>4$j9uj>mGH`wI;iIiVXA5<%w*u9TxUiQdz>po9X-IQeMziDPb; z2-eMYB>f2Dp+a4fBn3~nb^biLtGDRcd>SX8q0Q}|yx$$TlW4|<8Ii-Pki*@UBz7q(d(FW2T*$$ovjZ3VY zlAiQ|Xb|q};oF}2>#QnTH;+wq*_YnEaP_dlS9@G=TtcDxFYkK*CI_}3%7HtZo7;w^g-d%D zkVBnXT!cE}t19khoosgm%UpD+(7=Mr=h*#Q7X(3P@pY=P)5QLIQ$iCg0T2hHu*3wP zeR)~R`Y1zehmFO(7Qk#=cOD`zO0TU9g%N7}3+#~uJGr(?3u352v$rs!#JbV!3wxfE zac;XCfYR@|-zIJOJIt)O#H4WFjKp$YeNofhlYj{h78sOcb+%|Z;}bum{s3+%iBQdE zdZIBkZqaDkRcMU=d}I5DmN42P6&IXqQe9|W>Cj8axp|MZ;276OwkLRSZ=oKfmqr9k z$sxM+w2=5a&qMjRUg{y{W^MsI;nF>>T`8uk1#0e9m!qkv>jgNgTb_WU@K5_?!kQ^f z@o`L>6lP1{;)tQ%YmLo=@x-xBT%baZsHhDIarXcISS93Qjvsn4>4}+h`x*#BKCp}6 zb07~f4n~v>*cVg{iIUN8jE|BZy25n z0=Tden)5P_19(wrI9wBJaTwbErJ9DDe3fQf9-LvvFvk4We7(LMV z0f3f?Df+p9bZ?FnPO_ZhSs2vF5@*=&)B@O*miE8l7g5Sq6ik%$Nxv8Pfr=x|=s2%h zjeDb1#SOL-FnC zbvvL}c2i%1Kv8vE%P9WsMOA(m^pG5|LjkIgN&C00P@}0R_`yjbXE7At1+kD-rArwK zSY+jBg`VRWHxx*THum7~#&44hX5eU`3&+c`{8OsNDihnAd)QC_)Hj?&r z^wh-%OAP3Hdabwq+`L@+2b3PvoPkmI6x8e8&TUZZFz%WQbk@u3SX=MHrqW z(7s^+=I4<6H~{{aH8PR^Tp#Qn`-v)k&iSM1`=U#UUzo76Qsx1mWICGIfK9FUg_0?$vZEHC1^lUuzC5yH8h#w( zuKEoOgu*OmtXJt;HsUi+kBu#A5f}^SsyMud$AvB*&#{|YiwxMXZ=5x8_5Cz0>X^$| z8KOB?GSf0_bqA(c|0!~`gCdH1s7e02iOu@hi_z z=Mwyi>jw85^px2K$BB7_IMHHqcA#tq>gm>eQxd@`BYC?$yeXfh6VpA^EaCp#h|!)y#e&;s|*we&q*GpTw(Pe*ceg=zYrl&^pvltu>PrB{=gd2A{27(Nj?qhIh?)TW zeBEt%Xu~7&rg?9H!Z2479ct26R|X1o4n%(H8o(kZ*Wj|~cO?#bIeth^KDq(mdE+F) zN?Blw>qROlT&V{`$?TLzqz@6hnLj zzh(;NE`Uf_+!D2kOfwHol^lp*%4@7cl`e-*PiGM?^|-U+rx>eL$*(mvz*%FpX=ETn z{lv6>{`|&M=%na+&(X`c5fPH_&9=klTU>7Ek5beymbgQ$RuB8_N_|1d@#fWnkoVJn z?umddi0F?*E?o1o2k@qHfEP#kzCtG%axATYTCezo{)j{c_moUWvZ6678#J!pmqdYbURqHQOO&qoW9>dB(!eboaS&Wqxb{5!Lo+3)dj zn+xbw=`2-I7R3VhnEc5hY@(Qo`IEC@0}Q(*TYGR6eI*v{@|9`L=$hCl|C*v-{J+9g z4PaZV@Q|$dE3VxmvZ}iH3H8!4#$f!(0CFU!+r&v(v+_t9Y@4*v4!G|PlZ5$oWS2hh z*du?wviH-%qnW*(cQ41HLB7mLN$TY?RdymG^{0`l>c-*5XR|hur3u=taF4&CScU+-eCiUq70938-+jovkx9Bo^3~gw zhl8h!b<~t68mz1~_zqv>9xc2Vmd|pMSPbn(o$=?NDco)J9r-y9+dQDx%iR~Kac_3n z!#U&JFhXmLJdr{pNpET&)#nF)J7CD@NWNHN#>WxW69=7m018^Ahj%<0Y4K!hT{M2T zDvxj0(}mj&dS)?hXE&HfVl~#?daHyjZJvr8$okHZ0Rb6tsyn4^yRnuomFYVn!^RA?f)1B+>pdop3`{v-d%1p zKfbgC3E-glgW4<_CBVPtbL>bWW1PpDQkXFbonrtI;VNGA8I ziFPHKSi6JEGtpH__=;uyVXdt{MteMVkjI_iy4bQ>*KVn_B60(c@B4sS zBLsjNwhDsG3>{c0r%y2)gZA0QuEH28fvQ0=x0!D^ZrRGB`blmksS|Xsfmx zbB4leG3lR?O-Zxxe3Oi%`{fHKnVpxn!oGG0sj20n9nTM~rFhNk(F^fLT1%kQs^Naz$z#F$uv(d&W&=$7q{1s2O`Ik5__lL-7 zBp(aEY)-v z^-$Fwo+kTmp@xHll}JF4x#bG{mUb~`X>uCR4O;CEO}`2NX>It)QBFYZ$RM1)S8@Vg zfRTXPPB4pqot)V>Y#U$Cy2a5UF<+k$v$7beEx?NK+wmN&{2hbz4T(W!K^D)=VOmk4 zU-I@&=Av<)a)fVVYTAb519Wpd-!>5`6l1G%i9!Jxyms5+U&cUHCDQBSV@?IJ7IwP7 z9Q0ocO=r7eMzJ;gbvYH;-W95wRkdf6r)!aF^QJY>ra0&Mnhy;ogJ&!piSX{VGGarK zIMlJeaYH)}$f2<7%8PW0(Lfy(3Z>Vie31Y}7PbS842U}RWNj2bQP?h{* znnD-PS18;mDTl`k*EXmsk4qm^Ugr;PRHbM4+D-5C^lGJ>y6U?shrUT%bi*edL`z`t!hExD+X_!vs%-Xm^;> zCO8K3dU3R*>#H#~dgcwDZ^DCjDz0+aWsF^Y`l;57#%KyqlP zenWng3Rl)))pLW2qf6BH{vZPXt-DaVaHjrZmRT`poiIc*w{;tD7R`n6m&3Wu67D|n zCeJKdeRRlcNv&eUy5p+PDMc|PPT&ESvvU?6w<($IVeKaW`A(VjvpN_*+pW;r2B zCt!_RVh3M#(BwNnBNgQwVY^Ssvn^7aboFtvxmFV2_o+nh#Rork%K_`{NJTA8Dqp1d zve8gje}<%|frEIX$lkx0L#e8Hp@(CkK_;Xpi5YDYDUx!C}H z1%JWVEH&xR0`N}LpQZy6@z<}Y9JhUuG5-ffez!{%VgM1SeSCVqP!|86GDtCL6<6BM zf97ietSqyw?nG{fMLNf^RJ6RfERnSxV{Wj z{fE3Iy+V8PB#U(`A0 z$OrxaS7Dn`^f1ajRpCCF__3uk_3R|(5VH!#Zv%W(7U5>y5g7H(WLp3 zj4DS#xu4U$bl0AbGE-s>5L0?Em#GcnB8ymoB$hm6g-$f<-}%K0^N8i^L8H8Kh@^yGxQ#W{|OIAwCc6=oVwjpie&nSlx;UprjZXAE5 zlJZu;xD?F?=$3dPU(!a1PoIx2>xrZE1wCIJ)gM%lo+9Dd2h5$y3g0Es)mx$-Btr^neoO{w6H4!oTT>4z zNcT6Ub(W&eMgaM^jD*8lx& zBW;G@Vml@J)T2>s+%NEQ1$nR0!Byve9LWRFNA6O+&RxU%BQcC>U7sQk)<0p=fRv?z zMN@Dx-?~@jPoz&kx0N zSFoTj(*v^!geqr(r*26y#4IMr)&bpWgn+V{ex1f{px?-W*HK2#tTwVu;W{GIA8l#q zul}f@fG2;8xsW?N*SAcKXmPoxfLGv^wKcf+iM}@MVYeC1dK&xpfgR_CQmL!qC#0$>s1%-}uG>~>vniqGdQa-v>=dnU*5bKKTaZQJhk z3hWw~eEzlIF<^3 zDE50CO+H6pF~m*&LcfGIOP0{RKFTHoe4Co_B$>2CyX1lplo`N zGYl)jL-;qekoKkOX8Y?_FQJrH0WfPl4)*EAFW!fbV}+`EU<1mp9~j9OF^0Ca%x_=o zR}MT(gu6QwzKRwC(bvp5*o5V}qwy6SIoE~!Leq;b0CKTS%)p?ikf%z9$ZM)9Kv3sk zs6FM4AQE4kywhzBoR~@)a$HkNICErm$|h!ii`E1Ki+GSH^>siim%4u z6&}NJR8`+FEVVsUFv^Kq;6rsFmIff}3vrVW7j#sTWk#(exI97(EfRji%l{tevYE@m zSSP)4Al9zOrm5*KXuE%(Qnnf~vulY)Y}FR@&xtT%EJ%g^2|p1?jn}0<5)b40>U^anUdhrz*1|lIStH666Q6eGTk6Jnm>Ws$)BJLjha|g+_mr;4&{Jw~P~=OrG)E z&#;DzLZn@t^{}&JGf-B(vO43HNu+duRD}FJl=fTBOtNaKD!`G7l* zZDl0wwzyN+A)RhpqK2F={M5A)nCguyNLTmGeWZR(YdLJ1u=&qS5Lq*#T!(HX)d)Co z3=vv%O`2*f6CQ2$nSW#{=JxAQ{7UxhjP*?iqY>yt-w2-+#vcNy9?ldwD?)(Vou!ev zdU61TgwpW%u-7#BV@ewfR=3As00f4&Xz?}x>3vL-4UP{)ZAfX7h=oH86T(OTsspax zz}3K{?&G-@-2iOH5skkE1^rq~UXxWh>tQRg`T+HS(2^n9jSB-Z8~4b?KYH|SGmy3u zmI<#p@m*cZ;g7FIE4t5%^kn&y|F5gK?gP%T zwJ?L%JJO6h*3AD3AvJ7YqfXHILPk1yft#dR%BP>I3^l2e9vrsy&lMr%FU;MlkoDR?$rPXlg~WG;bQ%KhMv^JNTM zRLd6oPvaS{d@C%8Fhf3sO1t%lV$3F!_3gLbRN)trnF@bvEhB;GQN%E^6{brXYs2}- zlRl%209L!DDF@H9t^nygEuF9X2c|)Gh7Ce@{Y@^`X zBEtj=>ioiBxdImVM!^s0D@!Vnq0fX0`9m&Tns2vC7TUv|gQ`HwKpAkZtxhC(=U~1T zMdBng4}9xPaFJIeibg>p`m;l+jM%*j4-= zPL%9+MFGrh_gS69E$&~5b2G<-oypY(sBQW?`F&LXYwfP!l|Ktdg=n4p)&q zG~6~*@QFGd1;U}Dh-lQpZ}3^ds#2oykoN|=Q%_s6ATkc=Mbvyc$1Z+H9Qvu<;Y3b0 zdN%OnLi#sb88uf;s65NlX~0Sc8s{YL+BEnbPaHYdSd#7j44!{jm&V4PJX-gb@(H)? zhk=G;sxVU|6T&6H9YhzYB^p@o)P{&nLDw8Fi(16SM}9wt-cQ9GH*|YNq-4b}2F*F> zz$)tKD@+Mv!nPD{vJ&6N2n`_G1PT@m^cznG3kmD1$PUjlO

rI5z+Ty_ zmxf<+&f>(yFloRzpT;D|9?!6?peLNL6o${m8HbT~obrAbnL1^@S-wrU`)%THYRJ!m zK2lOtfEo=BucR^6;TFLX6h@Z<_Ug>$?J>9EMq|xI!-a+A)EZ=4sO4TxS52D)+!Lwj zuyCO#tHesVqe!SkE{^>!sG75}=VJ4%6_p5RouPC`{$p_Bo)B^6Qs%LMH8`3|XP-rF zCRGp>jy_`$J-n)GV2Te=?F{Ri>Nl?Gl)IW)T8wB}oy2%vT*C;^JYGgt8zaso(hRM# z+{p8y%;^d;?`0*V!}jK?05~Z!W`KTKln@ZP*!I9ucyu5?kH7y`a@Hk9cnA)({&ynn z6Tw#QG$3}vF)jG>ibxD5w4szi7bgz?YwzR}k5~FT;34hk=*6N`$s@S78Do z$SH#UvDT$B_qxESRrQ{XwBFfUAjLeWJw%Iduzj2BK@nV5y*)t=$?1XKnrG_Fq@!o1 z6P2uTx(>3p-NMsXpx_RRUYBaKM%gXGA&CSZ9*|ZV;9`m&p8^)@OUjrWf~+}D;xxbx zNp|SDkh|b}@oOeO!{71TH;maBCj(h$B~CUtfqQ-Fd{;66Mi>`@)2_+- zOD^o1GAfuoVGu`Gy?fiwsD?8F_OgD;Z(0!NX5%qQb=BtHnXtcdQ<8rY%0Kk7FQhpv zl2vOik$ z+B(Br8rcoWex|-)q9o81<|Wt3DMoK42XOGmVZ3^9F?n;9a0?^c<&{4hLj!oD+vmKQ zkV3#U5{B$DBfyG(S#f-B6|$0)S26egF?Crw6g-0gFDNbtB zf_}1%eOM>uUqm#w9E`=1}T*UvlsHL9+gA6M#nReH`ws zL>Y{P3o?;8rs9my@dt!XGllC|c)({=PG47l^fPo({8Ppq@h!Qg>*UL^%k1QfBR@X} zKAs$;+Hc(+7^>|U$*8@^|6Y!Y`FQJ|&{?cfZMu_2+b`fegar=SIrE`SyCDIKP?_n| zPX9wy#7BNh0&y(e?#4jmg#Gq^HsXPLYi9=@Z4t{HGF}5}+rz&mUPj6Z#8eg6V#p1q zA4NLZh?pQbPmMIff3+mRzDz(CQ}DJXTk^{JT<_ApXFQQ^Ir{e^KFo=<&!s^JPg zHs_4sA|;u4d}W$OLe2jwhgbwn%z^Z_0#aEZvy4;!u=qtC`J9gTZ)0_nePWbbQ+8s3 zg=v2GY8Hh|7T#^*7nJHsPQpICXd;iTEc%ic{p)kN?U3}B@}*9EKEczcSJqzE#upGQXP+xH=~}g7c+}r!yS-55uCZzQ2PC3irlcFbFLD*I`dbL7 z3=Hbq`>XI1S$&W_C%KsjLH|Wy)`G6umX_gn8Hr{eCR~^bDrO6Y4bIvC>vc}4x$jiR zLYH1&ec2kKJx$wXwE485W*$>36SA6nst;D!t+Mr8Br|@hv8{apR|H6d82b*w3YliR zk@sGk&pS5aEa5(nE}!pb;!~9Pxe#mo%(bFPVTBKp5yJp3Xv|l7hssv}7$aYv;Q+Xn zzLLF~NLNE#?}OBoeS z&d;Sr3LKPk^T~mc1(qk|G*P|PiQ6lQZRT95ZeDAd5SPKF#xnb`O&fQmuvL=&rT8FH z4U@E#3U^ra;8QHA%@!o! zEkp$}?`*5Ayj+ZK-BshAk~)Ti;PYyGi!MXg&N4N%q`w=qr3nh6ek#TD0mqyAt2;P1 zoAQHh4Vb4j?u4G$4#ep>?_`_UH5JeWLiL#|h|*Eto!Z`KQ4$H{;1w9s?1wmfY^-gq zE_|&+YFWg*Z`AL+$yQQT!Xf~l5ePu#i$$YzKDbKaB8&1goLBj? z{tVvr9+C&|K6J8RE~dDI0mNSE|1k%7*OF3zV~-?_D)5-1MybZeRZk#%AwaalYBKjVRP2|lLBU0S-l0yxe@bxyI{cY)xx z{K;vhsK$Gb!c&(!uLG(9Q+-wLRZO1mSlG}2*(}x5UsrliU%(K{+WiGZa;|6E3$`!! zhna*uJHxIy5yAb*s*8U>_%(fNa z(Vn1(XA^!zZL%}Wp+p1-kn!+wlvsJ#wdcU96g$}BYC`T*^!CUm?)rBWQ$3L~m}!Fe zHP-cytHz?Kqq!WPZu5+0V(gee@)NK}CcNzpgM zC5TOqraWkQQGU1W^!3)%jPX>Yg`|=0Yb<#YQnP-7-k5TH)~>P6YdYjG4)t?hxw6kf zsMaz(uJafsw;}aC~duf02Ya-S7fjv(z%$mE+n8ItWS} z2m%luQ}6{K_Kp|Dg_GPo$q@r^zOi#vsk+1gz`$#LV3)O#P(b#{;^u?JA(|m70sEENn z-~aoPtX!Z14(-)fF zIO36Ua{+@O0fX|cW&_N?gdNLmgTVP*B4oz z8dZk|3wSt<03OPR>tTKw@@OWTjn`yJsP~r4QgcI-JhC9-3$5#79DNc81s1$cG#I0W z(^#WEr3&inyDf|~(y&9OYvF315~!*P}q8vB+iQXE0dnzdt(|VS;iO%E} z2=wOI4MM9!O)B$oaELobM3fP>44wl9wB}=GQk#Y#$ezq$1{g9a8_(U(cF8IFqIDM- z(SC)FkcXC7#fb!ti5F>oa(H%r%8aydwmA>bk-flAA$qUuLr0|IVB?Fy!X>)ka3N{l zKu28Fe}%r9ChW!TLov?>x`|!v07@5uu>|}_3F|b;M9a&^FTe#Hq^i-c%X@m(%T+W(+|3j|FFejiG-C!Z{N zE*34KTq1nLb_UNs>~v41k@TWDc)Is6@r=-fDgK6igOrWVp_4Ehc7~!a9h2E|F6&nI zve$Zfl~F*!_Fp7G-s`lHPw@VAb#7g?1)@z6=XeLLmMSx!x=TR8-Q*JTXfRu7#zA=K zn$1s|Yfn7@s)Z<%?wkeq0^&9IkeBh~UNqg28GFPR zL`z4#o>!$kIjSD#J5r_J%)F=K7O~B+E?z~t5uhIudOqYGg`CDn^FCQ-o$tN{Ul3>N!NVX1msA28^Eb~-Px$ZhuKfO7e zx$^yUV-HIUvV;L9{3HQ2EGfMX5;Q4QowNNOn}MqUtLj5WDT5eUUQoceGs<&}rjc(@C{Lmfem_?2YEu@Cr|v z7f1n*2b>y9NL>9XpJUa#c9Aldp($U?kvwt`pDF~@SxH~NQsSl3I@9wYGCB+>gfQVk z9`L6&CVeb7CbW2s@h6QnTsKGk@vy#!W3Q?5oN4~58Vi6yKttVD_ zk|nDY#p;J`Pq(>Ljx_c=%6KVA`7sDIB@_$v{~uYn?zsA?*kiMZ>r$O_R#gT$}(sK*GPz-2%s_AHs*u zt*!w?3ND>dQ7nKm!Soo$;1kS{>OC*eD5gjng1P&`biKIXz|EuN^kXT|VpKjrM0bJx zPRnaI(m1&)rFp|7RRNkc+R@8DDnvh)>*qc*I1NpN%&*-+m~?n zywY6idHm6kr5Isesp>nf<#qFS#Wn?>$V#R*YidmFEMn@I;$@ZU6Lu^AOkca}RTF7X z!W+I;(voh(Lqj_Z!y-FanlGd3KQyQUUYUm$|jmKYzIwt0G2^v!VGmi`AuSlfuZON3r2dr})WgX^pEWHcWIkX}I z@sp}7FmX9=Ujw^6BW=5$-7WO%m4=V#3d*K`SBn3Lt1^ybF27sokHa1GR@tUFjKZg! zxg&JT-SvfR%-{W`#j~?Y-uq4p{Vx_=!%FSb4nkEy+0+L2I=0XUB7Y2+zg<9-4c~4) zM=E&uY0s6Hm7jL8ru=Qo5{cY_D}P}x@uoo4e$@4W!~1{b3r+D8K4y5~i)xwXu-F`7 z9S6~(BP?QSFhPA^2bzAvRp3dbM1&*JsA7_q$xanJm6dGclF7EET(+EsM%|w9E(`+$ ztq|z-xwJ)hHsv5&<(=-Jkf>5hehJb_{;cMF1^Rx<0-h~A@KVtwDs&t zPSZNQXNsi}k5*UZ96(>a^DWz8Wm`5kyt+@EFW$=m2j|%3D~#dr(XB{#pi?kEG=#3e z=2O9~>d;w%BrX#`Jm5_8Jus(G%a4rdsvR}oJ%o3qv_yT{nR0pwct%}Ax4f9HQoFGl zD0s+24k$61mL$tm1u4<`xT+@^4gE4hs+BK$$Tbm$6)?COL5)kbO>jSr91XiOK}bM2 zi%;b9&*y0*)yivxegBsK+QI_0!(EtzZl0k12aKxAF@lmuZobr()wf}JnD-rU2aVIJ z)3Eqv)AZVePn?k~IyI163nOQ{b4Rm&^ho^AR=rGTX-_i_=wr)4W~%04C1#6VX0pZT z>ZxvK7BiGWV*Xzg`z?i(>iJNKu_L7bcRmmH$kCO=QptUkPw)g{Jsi6aPiK0}@&{x& zMI5{rX(Sza&JU^Bp)rpkVT$%bdCVpaG4(sK8%pmCRI9^an#hoiMo(n7Gm_sRnc6*t z4)9Fm>fz}F=3N}3SL5)Gv^9KYVYeMDEF(P}I{-tT0p1`8i4GnE-#Md8J>u648QQac z&Q}&(Z*|>~I|1jWcAz>!JUU*zA6Q%;T_9VM?fC{S^wpij;W)T64$^QTWBOwn3cPnw z4}^0fMtN~qei*|*Ej^u!CmtG7LTl~^{`Yfg0MGM%a(V89G+vKNj%!642QX(CPq*YN ziXT5nD9ziw7{eapK?;)ncM1{>Z^*O_Q+(e;)^9is`eaR|ud=h5DPLrVY}@LFe{o1OD_iwN-WF;HH3{>nCZyiy@oiL5Kwg$h6NMExzmI5Fpbr13;%CAw5EL7+l)^%vPZ_S z3PrTjJIwHx+GHiOnrOM8{4gdMcM>P?DR$3)R3HeA$yQANM<`Bo_!{ua+KrLwW~<(W zN_|6wlB}9tEA%iT8o~9vRlyG9ji*`y!%nUiC1Z2zumvy8Q*A?Wv;>1mZ$69XBVhp! zt#RR7!`$NBZE;D(K%Rsg0uAS)c(uQm&G`0hBK3KK)^hjiPUh5|1`UoAwulJdl(Jw3 zY)vW{ee9ja{q&FdAqz1)6TyxvHVqY zfIP(2;wX_J74`D{RzFqOli?G0p=0sSrPQHsS9F(_u?5#1H5fOy@0I;hsqfZ;< z6;~hDT(ktTS6LC+vzJbi?~UCVth8mhe$5MV)$Wa^=LaY8h`I})s37KvtYxf=Y{u>u z%wi>w&sOI@P^W#I%vp!ExzOFY9 zFyqq%`oKYd{H_gjx~3`nZ1C`sG;)LKDI;XJp(X6oSl?GjwDVvE1$;Agw8qq1^SM>t zSB|{Pv33@6VqG>`q#X%O!(4rBQ@Pi05T7-F*0yQGCodsR8g08>no{1yl2&sO7a&#n z2wm2<>nz1FPHzCUav_KSO9riz3vJBLr{BLP9MysT+R{)=b~mJLLmFF4$m4nk*v=nq z3nZoS=#Tn{WDq6OC!JW1OZHM6JPUF1 zuM1n$U2K1LtM8&NGb~*g0^@H%-ZUV`1$_X7-w#l2@MLGnMUI`*FBgERYlpiD^(1jL z+ULr2959fS(#G1qfMN%H$SqgkaFd&A0Vh@w#>E?F zfD%KtBgja3#&Tn7?dIU-8wCllwyqniE?5N6?F$?vW-4HA&J_lT2q2g$0tt5ByshqC zwixMss=>U_>B-IkPv9r8WYa+%tk4Oih}#x#jD)z z)1`2p*&{csk_S`_j%LR@S}iuY3;VMBQLs#XVeOzX=*H#7SxC79OzdQ<8o5n;3~+kKIA(nf9|TT16K1c|3N+ zcfC@^hUU9|_v@$@uaV#tC3)V$e=>4yTt$A2Md)Xmcg^!%qMb1LyaR^AMU!Duh{0qR zW%Y!B59JMAuc07Zi|ayTP@#oC_zXvz4NuAT++?UfzNgN8;ZVx8NY^JvmRWGNclBpn5)T&wmY}?+_34&kWS^fRI5bKvGkOy6aOgPyrPk3(PxWxzin<>*h zRABcsu2E)}li^^TYcC?M$7*0Omy3F`oH@^r2!h*quPz1saOnq=K`inBs^Aj?@=HFX%bo zwYtF{UKe?<9Tn*XZ{r2Q%@eFSk4XkX9X&Q_aF9TnWh;YBc3C8b4eo=D@*JwTzw0(s zihJO#sl`$ccC;GTC1CQWFq#$_x4e|fOenxEMj0>`(GgIFDgwIQ$L9(o?Ujbogl`L}_qS6E!LW@sFG}M68nbt?B4GP}gzLf&rnYsHU+0d+qPG zVUM268|yeUV*Q|@9hzdDg$|GmFAA+`6bs2E?K5#3puxerU@qP&A5W>Bi0;mOin}^w_P7aUh!zX~{F+A2{QtWvwFJlH%8 zWrZhiTNNUxMxWX}eZpX!cI*Xm{DH{Gvby~1bC_dGbFB0fvv<%BU$T5AGWmVbHqZ;Nt;B?ZN=MNO=xgSThkm9@< zoK?Ps+?@o(A)5XP;@P;5>gEf6S#RMNK47+93rIevISf~YE?A%Gxn#W96eDcl(0xgFxb z74%qbJNGX$oYm%~s$wSqDo3jORoBtCAhK~|DbK{GODD<6{1lX>JLz6Pe`O4(lD=;R z7%rdVV_*SB#mQ4$-%C9wEhVs%F|kz?}Wd^T?~1`*`1Br~`gm#TZ1S9ct?rmtEaCDt}ch z@AfaL{Zb6wF$So%x$1b06uIh8oA_*MqvSU-k6A{NbI;tK0*Nnf4N~GnY`f!#C>*)E zlQZgml?$~4#d4^#ln)vbb-Z^xuu1#5Q*=6ZCc)o@CqaN(*lfG@-*}}Io z9I=-pN+kWGdvM?wt=FRXc$Dr33eU^x`;fD3hRb~Fwv03LFh~hSbp0%m;oHJ~4kxV$ zCF!V$ZjQZ2fn7?Sx@&kt@TQek9DK?sH((l8o0ca^Y>iZtS#3TCWIO~F06p9f+5Kqi zOaO7oHe1?fyu&)X>d`PtQ&cu%Y1ycJ$qU9``kxlh%hibl!@PQ(b){`@DOr?x(n=ie*kRx z@jlJnB>>iLHI^i+B+*H0~$kGY9o(k~>k^H_x6~^$Nq! z&S!88M$BObY6ny-O#{q2H<8b#JNkqQW&Dr1Ro1Tb!_S~_S?0o1a?R1RF4+Ile&Tw) zX!MuHVd7Lo&xS(jgPhu~dNaZ@Njws5!GerDstUW_-Zb2GFQgpa7HKy2>NiNiR1nl+ zoLGH7D5lRYA3DX(=2wvKZTi?cR5Vd757c-s>-Ha^Fd4L-$+F_;=XKHwi=jX9yM%&z z5{7kf^8FzrMWjlk;}nK6Y8UT+`J%q~Y_@6`-85CpkXP+3QZh$@X@vj`xnIYpXOW&W zdZ6PzmpW}}x2npE=4||88rDPoR0#qxCRP08V0Jkp8Y*x?kpzoxPaKEjCn=v1v00>J z$k8%7(9Y`vF{(Kh#G=mH^dIl0tVN3(bINz@v^zhhKsy3UfvF$i(PMe>dfZGC|L>Xw z{h%_VuSWc)g?$KlK2aF3^sE|EdN{*?)D5v2lvo{I*zNbB&*L<^8Lpj(W#_LU5_)b4G9#kFS|p#%{N;{Vk%+Q~LEZeg`hW(x{$>H=TSf zly6{f(aE1OAWchT+*)~n81?jR3CK`o5|4?6^RGNwi}61lDAhw_JmU92UZ zd3Ryn+Krs~T?(GH3Q0u1#$yh)63)JhISXNg& z5P(K88)}yO1(0^C~NzeoX5NGi!0!-7fXeeYE&osunra`F`*?xN;|{XIOu zXQfyI-{4KUw|4`MG=KMx3RM>(_->pA8*T8QJ+Y5`@gY%Ea1yW|s7^7xPLi23Z2Jk_ znl)Y9m}&_7JOpG7rgtT;A|5HWbGll4H4XQvR$oXdE0l*n4{CkHJOEQ1N_RsEe*k<> zsuH#os;myL_trj_I)cT!5nXEU%}qJ;&}wSwmh1)B#b4`7h4hY-%)cdRcV=6X&*D_2 zcC2>8n_KnC@czp$Aa%|JAgF&O`k77$8cz7B>773T;K4#nl0&YBSGbo#601=(IK?P_ z4N!C_Em$t?%@yhr<)YC{m?V~-!K1k|*O_wNL7FafnooE1MSlr7XK{leh1la)uMVO` zt99}&qvQJ1o93^#t>>dFd!hCHB3=lPeXJNAY-A@%nc$6S8WN4t$88$V|0&?!L3`44a6T|O&g zxiFW$xn;@Y|9Rh`Hz8AQAVTDC7-pOjMI&u8o42tCjkjI0J8K1Z95E)j2xhxhSvR?; z3vI^#rm|jrs&voS;2#Rt&T*yw4kb`wzwaPDvO(64Hrt!5HQ|KL?k#Hnl{I^{!Gzzj zE>~x64Kto;HI0!6rY0;QnqXj&u0K2?i$-=uF}DWzzc0Wy1PO>R3ixVD^>7!x#$ZL1 z*0e7GAw*-7J}nz_Vv6q~ts%dytHpyiclAjE%Q`LJY%2(JC{Xh`I&k31mPM@A5fmA_ z4Jye@q2p0c9rzf;8Mt#cY-i^AwWw5xZC^tZlgF1VNQ?)dfA>Pa>rfEO#M@mgGxm{O zb)d2E+1w!+IltgHnes{`bX`}^5@?AbGaz$w24W#ewI*)nGAM)Ym%gFopuO4`K@09_ zz^mZ=s6h36=JDebRIItQa6Cr+$bwS@`><#UzHwV3{cJzek!yzP!{u zMVfxp8J*|smrX6QaSKKexc^aP1g6-(UitHQ5(e1|MOqB~!pvZWRjdvMd2GZl#b8W; znXFyy6Ok*_Gur~fv(iXWe30z4w=j19g0l?qp97%&g<6=DMbB)^mZ!L*3&g;a(g{4r zG!N%K@~}Xm;RM95ImM)Q4x6qFs9<}g0~%;yW$q8b6<+d@X^#qM#%iZ5v?a-uC#x&R8@`(!>MZt8w zxNFJ7*jhNkOhaUuSX_UvXsy3~+-hQu%FI(w<=O`@UAW4(grcI^;p~Kfk71De57xQ@ z3==uV=q}neZwBm}n)9clELu4nh@|Deo3U?JV(=(>5)${5%^(z|tfKMJ4>bgP1Vv5A zbZnhZUDC@DHPPTX&iy>nl=qv&+)soAH!2-N;tOeCWu|EWQuUdT$q*VgvC4YQt^#SS z;(@87z|*t;GYf`blOE6rq!u&}g~Y*nh})pk5KQKl|=P&E?Tik0^%b(UC0j*G|`u2sD>MKXh=cuA ztCMSU7odax#Zi`-hCUt@m?i@Hi~b{(#8~2&F!}*7Fc&c{*<-IQ`tg~dUDfxAZ}MAk zIH#%jGbOn26L~h7CP!- z_3v9lIKx$f0_5(GX0e0Y`~+puY{DxSe9!P?mZmmMT|YBC0Q7* zCwvGh<;qc2O&x*um9;%%UiOoC__y_KXIxyYlED?dL>?l~E83bTIkRMU=Vi^aEx%E}q{i=ThNf%zYO}Tk0_291Z zTKTg)btIitC|To~=3_Gs2*jYGt(cM@a6XyAb2N1*wcxW}^g}xD8Cp$UG3)U}0_f;; zp1n`LrrI=9Pb4ieYvc+arU|A7v|xB}>JXE>i<~tKk(5z$&cMX%rlpc@bWhvcG|ICK z`LDu`tE=|@^2&wid!wa+ldhK%<2-TjklKxbwub;PTi#1)HdDZvrejRUdt&Umo`9*lJdi%Th&l3l3pJQRM@Uw7$cfsL{FECuVD7-Jw+G5-$XkE;) zA9fVOIN7UiSPQ10T!DHi8 ziS&UZEnqpWLP^^SwH5g&J_02>pjG`J{;oV=L$Q#JwSTWu=xyf<;o-xmq4YJXdH>Ru zTXaZ!=nHvAQ2d)ywe^}%2aGec)6EKeEP`D?zI2sUI#QQbjQoPXTy5dwrauo&6-U{I zATT{!w2v#z#8qh|{jUZ*E`Z#7^j0yf*<(5417GEvE|GEPxZ#hy`dUv>oX=WjJ(=st zC$a%um|*pHFL8=cbwSE(nO1ranV9a&I~uq_iDZNeqW~weI-Xb%xdPDmc>@#!4_@j0mYrN&atih=!z}Tid5$f)@8q6pTH%gG%X_Bw`Rh_kZ@cO(dA1t~M71xL zU$PhesZb#kvdmjWR_0KwVNX800fzzyaS+dP@Gh@+C1S8F^vB2x)h9zicZJBcV~M)! zIq=X039k7cZbPc!9rt(idev^b)tp?mgp(Ipojg2y{P=?~J;anV#~!2(mONTABj{#5 zs$RrZbCjakvr*LgLjktjq5V>5JdldKsdpWip-6}=icoPP*__o$789VdJI%ltX481X zw#4+e%2{)ZpsT_QrCx3q#5s$Sdkj^(hUjsbj$tvI%oveto|<+@>9A4LT%2;5FmNYj z|Mqc*^9)om`<*CD@^QE0@;D~#1d2KmbTl=-xiHr^2mIU12@NcN{Vbqd=DZARDrf(w zJ60g7&5A|%4#K3yJS9ZjZ7oM_1ct8UHYjUI%pqPl#4h!OIDpZ(`(DxdwqOLkeY+Ly zb#o@{>32EQno26le0AEesE!+mp{x$CqugPsNMOjyJ_f>FDN z5J+IBZf(N>(mm**S}unafu@b=8U=O96J4fcP9;4uM8dCVtBz~VLRT%|5xeJWFkRCM z{cdD!`n{<{oxoz92OO&16W=Z-`n_?=?7sd_EuBwux#f%SOLG%>!>l%24#?I8-09S^ z*i|DWDFmhOfYe9L-lMYkfZpX)@*`vhuFXKZG(+J$Wmg)+_I{9zh#c(eCy3dO%6<=p z$DpbVsW~N6w6SGy=R2M!Vx=$<$`z`Uxb$vGd}hMRWrsZH7jgyKsj30_w)c;jxH`uy zEpo*)i*4j}o*nEaeu@TNJ3~*KlD|fee$eY!;PqJxb2`O?5FE({8CTIgKvoZh>iS<7 z!bjA1*W1f{0K{$@#u`DSE12?#DYqtSq`;ux9+)&lQ0|vUOvu0IxF>_*zV+{TY%{~`-e5E z4H-=W<&OiAXdf$~;Fb=kO)J(aI|n}&mjEj&80Cbwu&Kl}crJ6Tv}q?q^-}O=Gf_$V z2I%E_2QOWMjJpMWn`!}N9jZxDEkY?O4p#th=ZP+E%1z#&0Y7V?{jj{OeW*YiX|sSS zh3zwgb*+*_A1a8R6L->40JY*5yFa_L)c3xIg4+RLgAW3g9h@ z^Rr@=W#x3a2FeO6uYj(C~o*o@-dNvYT2kkBL%tdnP}@({0I7J{>7N zAb1+0W}8~W?@roRU8^)oAI5=D8L1d61n0c#9j#meNYm~QDW^gf4RFl3?TU|YxnsSf zXa4eS)yc6IhVLe{Qz_2|PnL@AC!B14*kUcBbEIW91<+&ZIkI3Y>=Fq$T7PbGyb3^P zS*nm@Q_k6x0B^axpv$!YJnGzMjpoL>o$pl`Ip6ijoMd1a$oxNMc`GxW|2+k@h@H4E zkD=x)B6Q?=$Q1!~2L9`lb?;|q<0#kDBj6H9+abm9se{M@Zz038jU-=6n91je!RgeO z8Jrl=rP9j#rtLb0sVnc-Dor;wV@*r2$3mi!UE7xsp@ zYw<({6u0@evapPRt1NFQQ+|e4x~@EOFM*;(76@*5#?15C&ucL$A%$58dk zy|_9?1;rXfj%sW+MIrX-8zs)Z*=372XE2?Fou`X4VA@sR!McE&t6bV?E-v;hqiRS{ zSy|kDy_}aHnBMgm$-LH1M5$prpr7C%JAAphRXuq84rdF%{a`7M8zkC8;f?AX=x4OF9m;>=7&*lv_>IKK-KXkbkR}m75iA5mc32 zZ~9ucw(&-ObW$u+*vR|(eKt{{st4YyLW?ile%y;8^-&~&r%)&aNeI+Dg75z^FvdoG>WQD1A?fCg*<4ho%Vxmg zS_AX8EF+@XCf=FcEI=?&jcLOsmR?rZ^=w%XQ-r|Y)SZG%xQ2QLcGue)5{LZX%to>h zCo5;jh@vs4l7l}S(25MkCsL6(_YXI8l;emiwi$k(z@aX)-Zd& z@>+N}?&oB`&oirh^cm}6Acs#D=TozDYmhmJY9n7{*S7D+JStv(yI{{CY@z}-4GZT`tjrX8K5GT;Cd>I`-Vj|>w9e-!x**=TQ%)SfJ-x9QOrVV0|5 zV&FDAnPUWtgiUrVLv{}{YnSm~hj+ENpBVR*XJZiYn&433Kh8Be9FmlSgSNzLZrsAJ zVH4I#<WhnF6?LZ^z1e$L9h)ye`a=L zo)2s=R|nx2zp`A*4H1p|=(!%44CcNl3IgyK^+&n(CeqKn@eo9t-*kO0os?YgD>v!Y z5}JWG_y!&uLo-=ph2cCOK z8~44n101xR!9Md9EO(eV_XNMmn%LL}KgRsSTmk<40c1*ooB9BAGLi`fTZZeQN~er& zs+~k96gmwg?YA1R>?#xy8YtZeZV!6p))mN}Qtda!LG9A_lr)L@E}tX9d-+pdz!EDm)l<|XY<)MN_7l$4(LW}L0@sbgVHwG~hwpF48KU6-t# ze@Bpa%Q1XIhbtkt6-fBAT(=PuQPr!q2B!u!W|(WLfrnLf59Ut??9RQ(Z5$Ac=V+Pb zD8JeTI%k{pYBadLQ2tA~2{-8%w!+5Ad)s#c=r4plBejcGA1FxMBe-n!f75px;^`0Y zyDARoC4{K)!O7veSHJnJVPgB&`!Fg?Uz72=q%%zToRKIVRP_kmt!XSG7-j@b#T>8o zHp)Bcg)HPD84uZq5{uDbw5fO2M7Wf6P+K@GifEh43tjGnHoC1WWx54DOWq>FOhnvR z7BTz3^o;w`=S#?U(4m|Q@vlTXA_q6W4FU^ZUx zBxUxS?!GD;xHJa-S;TKg_8c(7$4@{_7FKr1FO90?b&x8 zfy2LMTEu9;^afHyztySfJ7D?T(ZC1?_f0GI^hs-(b6IUQ?zn6D{zCj+hmvaI7f9mN zBby02j)d-hz-(3akLzJ0(?zU|N3po@RWQ!3Jq<&vT-A3Ky2VUGcE`+L#(F(9hn8Z` zZs8+zmheB5KJXcQm#v}!&?s-atG6`5yZUBH|0>eIyz{H__{6;u!YLL>3XNWY3`578 zPR68m^&cEK|B#@zxzl{1UwK$jc5WLD{EHLr z%`;LSL-XA*DFSfry!2h2_rZGUKa$qTWri-N*ZFkId?Y)jqn_lgRShjQBxc7O_^6S~ zVdFkZk@fr@fGQE&PMiE``b$u^k3?F^Mg%YwQCROmMtcI{ua{%v1he|(XRzeKn$p*T z@pr>;SUwLbfU?Iv*|bNfU{)*~ndr&p&+s9{Lmb7oJ$1HbxqK+>{2W_INwF{48tI2I zZ537j?nm2*$oI$IWY9fh(?(5qgJ!H(jpc|{2pWhtRfO)ilKPZ{1ewkg_I!4_a*#$> zxH4Y_nZ40xjxYsF5MCS%A;;y<>=!xDbWGkN5TPbSQu9MH%#4O#a*T;UJRa;#d@NN->{E8vzg z{meTg@-ZL;iR*79(v54p^cBrl_SNvgHoFESfm(@*QxZI`j(3CMq9KSlTq;Q{G$~JY zzjKSHuimdGi+5s7pm@o?ohEKD z1+zSup~!-UNNi9xUs8ggi+EkrZv1VvzGr<^NKq(6hHmirTa}($9&9$FibxF$^_Pn$ zfQ^tD?j8^+f9SM8Cfx`@QkU|dT`0+6);5WVme7+=K_~HRKCk+9YI#9Ij?Dn3@jw;V&7xEvP+w@&$lFrd z%{rNP5L*`dE8`&Jy){#^ij4;#tR+EPJLm_++fkI25B<6su^IX}kqcLcHb$)$go zr+SjnYwPc4Vm3i2fJ{J%{QpB~W#Oe3u`q|(qKFSzg+bzb{ztDg%Rx!iC^--fk2n4% zGW(_=7nKJOyEoRj0WrhTpi0-)I%X+)*+nHA!Lm$ut+@esP=oK6vQfF~$8}3_`3s0p z1}!0-zH-6Fzuh4Xy9=qYsk17i8MZT!`4<3NM;~^bTJ8LOR8)@RU`j@G&m@5U zNALD)&8ad-M~m_~*X~AD1^h7g6$d4)DOQk`@>lW|aN?#ot?veW`)_xK^1zP=^GnQ^ zr9;Fcy)MW8L#cn}u_iTA{(n{6az#VUqOzW!VS?2j=QR;h#qr#?^;tu&3DVP~im4YW zweslhGwRyt6%4cNHxl2>H?3z$+$G`X3h&nbYpQV4LTqj>z0q^Yjsde?rgOw1_5JA;VG6}zLw?WzJn<4j?I1Y#CBF@iRTe}JOlkjpV6-6mDqT9+(WpO(xa%vh*6xIkD71i8@I z-sIn;f>00k5D4}6s^P`^h}*zdUWK|FYx7j(N9bzV&ooyJW6R>;9k(MxhPU1iK8HpNWuhy5=k34D3i(FQU4A@E{-eFfBAX9r%=)wD_5Xa`ZuE*tm^%}LlE2P{>gbI@#!X5iY!c{ z7bjne88n+XdLiPqC9`p{KnHvzoIcXmC`(v?=F)jOf(2sJ7I|0fwUdoDLFq+WNAf4oKm_m`9w=||qLX5>PHWTj zGYHWVs~H|cCJe%SBE?hjxHa@1iQq&`bvDAv`~L=JFd@)(=O=Lq4cp@N-J4-($(S>< zQG^5&u`E$KsZEr+{%>5i`3#lB4O@Oh10#=8{87?QJIMSb2)MS&3U%hsP;PuPn*q^u zUr-s;#XoF1q-P&1^RiX4S0P40O)WHL5{Kj6Rc5PuRBfLaZ=bd~Ak^ zUx`YMWaiVk;^=i7O&-Q4jpG4efW%exs#=>9_6TxQrBWr#rj8BXz(b!j`ayAwQl)}J z_t*^uXb|NMto%rMv*wwMk=}pVB4Guq2M-OywL{84*W}uhf5R$U6QHEh{X_Chdy}kI z&53MqjlsaXX7HR7e6QL;>$BokvLGpzGPiy&E^12{7O7Og!_7g*V@ zIg-U@C^E;)Id(566WQjwByWnJR5QmPNGpvNn1~RVRL;X(=yuPookXuJ7oFup6*sr- z=Z&;Xg=5*pqGc(E)a}AOt3H9BJl^o)ksG>s8Q7Z z&>fs4(z6}z@pfZ`b&{V;s4=|yR%81=MX zA5N#hBtu+t<3}3E&)OY|B5_tmbK@pW&hK<+YnXQsq%i%?yOD|%t>BgK9{zpeuosd; zC=ftWeo$#1h_!&3B)Bu9)IG-vYUU2m(do%o?yplidYX=f?)5d9VK8pCTiGH>p)aia zkU2IA;ZZj3r3>|o4@`kbbhD$YrrhT53U98&MAWI>Xo5!zJ8is5VW)g~hpa1ptt;j0 zA`+*IU$dHX+*HkR2uh~$m{vSBz4#h5jrjAD5*5rG_*N@j#&=BzJ_15Bx5U7s?x1*J znW_H}I}&=Q9`d#s?FiBGq7K3a-0e{j%<_eY>YwSaaT(8WZt%4IM$Ubv7Rhgn0=53! z_cfIQnmP6=OyMK`s=dn#K+Mw>8-;4dnj!D%3H5<(S25|X2ZK2lj~-`1r!yU4DM3vK z3Hb!8jwFgGkO(YSGJ1s#)Lyfkx)Ks2$C_60IU4bBS{05&i_4YndVpK%JaI$Z84hF| zo4)P&HK6~!d#H|t_$rZsxvjomCmD^F{wKIM%r$#j2Wls|mzKdi3LiX;TN-79)vh;hrrL6dw1&F@^C|@! ze7Q118x0AqDMQM%i9*Pe?>)G_G*EjZo@G223eM|by~1!K07hrQmQw4yNLR12?Rd_4 zqmGi?;+yIP9A6x-@q8IRZti<@bM?dyjNme(gnqJ9Kd6)x6)92|MNJhSz)2&_gGLvtNsU0CHNGHy{~pE?&^F;27-~Y%VriWOQ$rfEJBO=$7FxMJJc! zLM9aF1m^1d0@NXS&SkJ1Y1ZxAAAhm&a;ay3?*x-=$J@DYkNk9O_<@PV6y~;T@gb|d zA+}D2DYSximbq;qPEbix%aGX{EO$pP@Td{FOqjldV1qcotaeSaCC98Yl$5Lsee`=of zNzZCHtDR-%g(%MiB%Ej~0o}2oN*Xg$6j4s{CdEbr)w3|F84CB;Ky;}2%-x;q5;3}q zoo`=Sa!QqEkOlu}D{_SZhVZR`r$9d#V)bC6k5{Sb*o6TVMg7MXcv~jaLT!C2vZ=WI z(w-O;Vy;akt#f{&)xCCyO6x%eViR~dLEH-4N~~>|G$^gg)?MMK=Q=7*|IBuzuKSOg z4TZl$)bDlC!AlrJ0PUR!B0fr5JICn+2lw%#2F+tVxas27RCZrfsyT1%pD!#R{s#CM zRT-x?cbXk4n~4ahoZ$#qBFuDN_*@F=t>>Y%@I`d9ipLHXq6MsX5Omc#apC>>NU9m(U%^h} zIO(a?0PuRbJ}i)Fku?++w;+;`t6=%T)jwt}LH4vZ_{l;Sk?wSZ;!l}qBmgBguIlY4nb;uxq*d;4j)9<5r+2LDNmG9Fynn zu)JMT2uecBX=^s)!DD9 z0~u&sAajg_`}f)CP0oypu5Z`Z{>P`$PzQ%_4gZ!MXc1fNz)xjn-?`gD$zoKYU0nAFK#v#K^xxDq_*^=$-a!T-o!~&m<>r^=pV~0nF^=gk<#KoRX zadp^;5T??b+S1e!_%Iez410%PwVjzg?|^}Z>g=_e7-7Gve2vm$lDaZArh!% zHa~|p2;^3NoP3R&F-Q{8A3lCxSc}D+tsW2iw{*Riq)&$B@^|Ir0*?ob?q!b7g&ku7 z$57c(kw=uqTa}RrfA04$e{<|xDADsSSefchK$eJp9k67gY{=r64f+oE3oF*uB^_QA z4}W7%C6s;Pd#EGY;HhKnCW=C8>`qxZigM48z=r2f)K~1%+8|zmh(da7^+d zWvlcSt9X_58-Zl$B3~u&1YA4YW$0m)M}Ik#{oo%X0crlyiw7@M-J8Z;$eO_Wq?WMh zy^pc)Q*$N}N&}5OEwyWC0`7Kh_SCl#cd5|{Pz(W}cDtE|`Y7u6q#PO@U{o!AOJ?7> zDvJ)iX z@ZWC4xVRHSezFlwr%2CFUh@w(h%7C2=d=J>`2aDcT=zCR>aIo-3pjv6vp8a&tlKh) z_Wvl7OJ8Z!Mg>N(m4%$f1`k~3ozg1v0{_(V>nq;XjWX&RYq&f@#YstlwoXY9n=kx7 z(7LCm^Ap2QinQnlJ*Yw#fWka&a8hVzLS1Q8V8Kh?6?5nyQI%E=j`K+lY z+qFDF;uX(L0U!?$(0#sgR?fQVR=|7g{QP|44ufDxGRd~k4dZzkR+ss67 z-a4x$qhFHmozd7M3$SPL4Ss-_Vfy{$s49bYh_GJb{slwVJ{GdyF`~d5N=w5zP5_H) zgC0j7GG;>zFx(7tXyhW#48uiJg5~m0VA?j=XbZGh9qeJ`e#d?|@0Nyllh=W9;t}^Z zuwBZ$^tFrh$*+8;<|{I?7Lq)Qxs*MED>M45kzEXe^Np5oLw@)c&VD<>DFzl1YgaH2 z-H<$X{3kjs0hV^0%+ZP+o!x>Q# z55TD3)6w~^xv)TIj}doAISmMmp`+|t4*8TwZ!YyD7(GsvuM83REW4P6m)$1y*O*WN zCRMJ!8-FekGE@Mz@EDmAwsu6{-(_cHTMPjyN5P!AX-^oxuh6EPKCiU~a##9GY3^w= zm-+%aU*lMW$vvO*Vx-SXv^@ThvgDF0?QisthXPOV#nG7fhP0C4GN7X+VyI2&VcdJ3 zD_E)6FLruWNktO-QAqICS`gkaz`YbKU_G8(12X3K6(v2O=QF%~udbTq&}S(Zh`HrH zR|cXi6leoglD}Ug*RVCrW#@YO{z7lU!5~FB4TdTGAReT6;Cb(414@A>L0MChTi(NQ zDz6hY5#OL+NAG@3k)!Tp@PkugNw>*a_#!`Q#>EWi=-yu90k3@N0+cMkb=x6Qt5&d1 zSkoC$zXM;?2M1{|uJCdUySHER5AF>gcO5pd)m^Oa(gbd$tm}(|KwC*LIt3l7i3BUn zB6`@j;W?DBR~|C%JWXLiC@rke*gTrG*~&-niO5wx#>Hg0l}O51=fY*Sqdn%y^H;;nB`4 z^Tug$ip4Uv)^uDaztoC2)Ba8O5|+~SE#?DBCd|Mnc+biZ2&ws6w^RmJa@)DA_54~4 znFn@)caI$B#>a=6_lCRKmw#YKK1u{mwO@E;OMN9p9q%yK=yYC9$jhG(#oM!IKP~iS zTgYtc_zaoYKulfI&gfjS4K`WxIsFNth{iBL6?VYF%q7u9 zN+rldqnp>J?K9o+p@`&u^gk==Eg}SvOF{8pXBJ5csJ3uKQVT5tL~T((;;)e-Ducu2 zxwb+jY8fUH(}wi@r7)?Hs8j*JE3Ry*tnlW@vxyyFJr7fgi2Id2$N1xRc}h)4{9W4) zpo$#YHqIQIDJqVKuftDKbH)yA<>Nec{!dALrk0=B>7GNj+)Fbx0pB1`psL=|wDIb# zE7lhWT0@*ukL{{}s4JB60xs01))({#mvt`-n)wrY- z>9lDl-RFJK>4(yE_REkW+6N=C7->WBGOr=~W^&5m+xKP8mz4`fWpJ*wIcLIfOxq(c zngA(-VV&>yCSU*OHufPs#ws%i;9gT`ek9jXKA?+SI_NXNPO_`lehS}X7vmfQ0_9=; z0b@DD)M>u!o3+qG&+%`7KkvU{T}VcoziUcfZ|h;IwA@u5&lB1G=7XE@w~C((`zvjK zBXCe&cOOEr1MY}W)@^0$q?Q0cJ@A^CkrF;WPf$uWQEa z?X)O@8hUYt*+=9WSJn@M|F36-#ObNN;j>ONgk)cAcUqs($X^PYS@YUtMaSg<`G^z2U%`0KmR+QtoIU zc{otmh&JVs1hE%*)Y2Cbf?uUvM!8-haG{Ph7;ZR6w7gC#CuJo!QxynJM8^$-(y7Yo z+-Ca`xr&QH`ZFqaTuVWIVAm33Md2azQ*DqYB1_G;P_gUCq zSrA*S0r1IJr^LdEk5AMDuLefNPMWnzneA*u6Xr zC^3Ul=T9TnV?6_DJZMBkOt08KK#+*GH<_%^}AxxB#j}n#qh}Pn2 z95(%PWF0?6WZMKrU_k+G7TDAL z8E^j+XU1&Uc5?v6cO5 z8u!Z}Pr#0DjdR-JRWUu=CAB}{T??2OZSn{b#V;OR)~k@-1vz0GchTsdZU_GxFjsX@}KY<=Peq~?(=w;cJk^a5D%IQ@KUX?*H z`2YUo7X^)G7d$BF{|uSGq4P^lNy;~9OCm0Ppmm&TsW)rjFVzoFZN9Nw61?y#S-K7OO8Abh;!P4ot{`Cj z5_Qg){nT&tcT1#NE~GZoT^`ApRKL(?DSZ63jPSR5p}ga2zjB+sjN%|OQjmdam~rai z20}A5m-Y|;#FrZcBTXwxJ7k={ruR>C%Q>MI7mw4qp22f{mt1Lx^WAfiv#bm|Nn)|E z^z&FI_kCy{OukMXtuS)s`Zj-VhC#2rN>c?6oGmx|&}h^<9SbnjR}G#zHjS#2t0XO| zSL=V_1(n4a6ppgItK@x5fGA*kNg^gVH&QP3I#vg>B3U+LwhV-30f;ePHx2@)vD(zV zpHRrR#Sc6&8#P!TE-ZA2Lt2Tive)dAmCm`x07LM6FBt>#+S3DB)OB={HzG{*j4u zdxGv$WBp#D3{x@a^w?{SCqYb(*<*>!!dFrAxra4>r@A>E2v1-A*e*tf3OPs7O(6q~ zw0HB+>?WCx;|<_|H<154PvwN)A+6bJz8J)W_W+MG_9_lb=teI?WO#Man|!1iwITRH zYDr(Y+pPyBCNEtOjjf5o5kvv4jXO%~p*ArZ0tFw^qJ)MP`ntETJe8x?k_2=Y608!8 zAfImUn1-pWZRqdeg;00{!Vkdn%P{9VRkwdXdAk#@9(@XK)T^;vtUE{K@e}*66gMWL zwt!6$hMj30cV%!;eD#{WAaukEjT#3OF?ORoQ%_VXn72z8`uH6Y#-(R{E2o291;X)g z%Iqs>vM5((Vr$XgWT?8tL&5zv8QN)OyYksGrUW5c-MoR92nM$Shw=%l)-<(k{|tfw zt;qI|>&*TH?yhD1;usy--e$~P86_rF+QOzwowuJkzHh$49|3BKbWStMQ4vfE4oE_w zxy200kXh_fDHQ$jaIu2Ld<<(}ZStybn*_lMfL!v>&y@7FZ>JU0P%74nxZzK6Dg&ws z^}%5s^TEcI%>Jb_Ui^z-f!)tmo{URwEfFZekD>esLN*QxDTPHR_+WWfv!c%MTM0Sl7O^N>Lv$h}Mwgj7rRz>*Wh>I6e3_W= za|7St+L}@3&i#h7$tFRigFZ?M(M*E+wS5f```>UM6Ta?Y{HT*)x@th-WKAR~S ze0fqZp_o#rFz^(C-?s*OW~Y2H{KfD77t)H=73JhyW>om_V}#|aOi=_o670)V%A$*J z(C%$J^BCNpSZDS~8e-=QtPr63-=`2{5A{eAJpHT}g>ps5oaR=JSHqFsN6fo;J+=<9 z@8O?|u2qz<6?^+$dDR0lMYR<*n`tVajw!3cF0v9){8mcYfVbg>vxFjVH!v~KLRHv~ zO`By>Bd)G1EWbipW`>w1Z;s$yYfvNZkQHSM9PZ^YP+l#Qi4!%xB-xyqDBBWl*=OoF zV6@ob8*YY~zELp($CHkep9F7u^syGWHP4BT{!-d4S9FOL6~+BYn<%X$Ldj^Se?-p7 z96VIiK_SwS=am9MS)8k;t#@z?Z)_x0+l@V{h7xL~o73@e2|_H_M4eSt@=;TDSc+)mZevN3}c6d|y>u5z%RcKnR>)}gJ#&Wor*9waZ%jC8@`@xByMQS9@(r+bF9CrvG-=v!vqrMHEj%G7HQ zA{|@|N~P_nmK&QDw0*g{Wthq=pI$jvtY-Y|54iXTbptU-ckwI>$xwv2-PbN8{Ckl? zd738>YE?_;cR##bKjaT5L*>Ws61;tuAOM*beFSAEbGsi!2Y3Zud9q zoy&S%ku@d&D0zw*NgotnBb*tg+yoAJ*%G$|4mMZ8Y*2Fhm6W!|kW?{_@uosUd^BlaUn?EIuV-gKa;E$K{M{LKYM>DNQ_sODrK_p9?f`^j zhjqdd=7(FNKS6Hx)W3}JLlE91@IPbKs(10s%zA)ijxrYzJOLHbC70nyzdDhFUgV0= z9e+=gxfduBQPnK~N01%ZYZO?Y@OG`8Ezh6bHLs|fxn#hGmCsNFP33B}1pPsEZsJcK zM78s^xCUnq9Py!cA#)?EY&4*yR0NMAal^wH|Fwz0^d-1*c{~UF)u%=~>D@zh@<)*Z z0fR;w+O43n=V%^zR`OSelK2v{l&7L~!tEJvXK9fOe`nUe9x%l|h;7g(53Rd4CNjLi zUY#X&vlXb5?9`XDl<=u4ZjuF-b8<($gL`%%$xo<$Gws>BwJsuy^fW4f^!f=o|Cs$; z_N^p&nRTH8$+vq;tpz5SqGo2b1^CpOMgkbi?u(yK!1y9{4e0_jyfII>;P0(JhZ^4F z1n=rIZ!)n%(f3i92~Te-SSe0&S|}w21s)rh z&R13ogzhLWanBb@idxqC03r%oPwI$L%m9z)gC&B&-amE-Yaf?<+z|;sPJ1jOBV6~| zLdP?{mDLUG9IcBKC6(?8>7_Oh{5YKJsP>{Jxhcc{T;&sMm~|PNKX+&cK}NA0o_qs> znN+X*2b{c<7YXRbY2KGbAn3waJ(zAaUhbzcj?$a?z+vrI03OY=7o0_2Pj zsB8KcyxHC`wxs+^9N(0GQoAse0XHIeCwnY>K$2zZTz_5_8nI4b?&yWYV?Yj$3K% zX#4vJ6vTxQWS*KKDzoSa^edmsmXPF^EGka)MQpe)_*l;^0Y=1mSDk1uDE?N+MbsY| zp*!|sd(|nN*KEc=13PoDyHo%nv~*`H1ncTYQH>c8j8L<4yD)%gH37seiR z{gn1bZ6AE3R6gLw@ki`prt?)CR`bGZfGtP_UfYUhMQS??J86g_K36vmLlR?58PoMS zz&30j6BAnu&dzpP>ySP-CvnoLUP(eY!{ED|$;nXhqmLZGEFGyhu!oDhQkIP&=gCO# zmX1iVvso8L>){OZc79!qAFZKw_Rm2#i$I|hihBFUouT|YiEd8Y(CMXt?JP;YaME>m zuRcg^PcmI?N`XtyapFE6kV}ZA1WkSlAx&db$HKId2$Gx;S<^R+Ao`utf;+&n5`&JU z&i8PhvV^P20(^f>jEG0ZHix;TnISW5iy2J++h@_G#~FxoAScrpyjl0yiz}~dO-le# z-@D5rRP*I{X9my=+Iu0AD@ur0oiN`A_p8lF$9>;7d>>+W0ww^IL|aUH1w}C7<<2zW znEt+KhY6D%pf?%>8u!&p{4;lz)CwFXVOr|rxnvh+?e8`-WtXXhL8+Q+JTHXFp`kET zDwG{}@7Uj9D zHbCQEpxWe*8N1Ri=mEQDJK#{qx%-T+|F3FWBH{$&geIlM``L%W8Zb|}Qn5RZF5CcD z!$iZ_n6e&=@UUXdkMwq#TL=BlI0dlpIK^rm|D~R)c^;)z39fO``ir*=7e!9V0b(!P zZ$|)dqOnB>!_Y+|w4>7|fD{GG{zK=k!?Wz8tTS-#tMaQ%mk?PQ-h76Y;XA1Jz6)5D=}$0W4u_n|BMTx1wmzH{oXQ2 z@uI+#m=&=vc(wr=eQ1VR|Cc5(;BFU3j?UJ8ey)geDN9b^m_?!erui?iJuc2p5z`Jm zcC~E{6uVZ(i^XzL)4fw(*~n&~&^0Njh^#+FDFhEz7X{904jM{-(eXryi9TjVfP#)y zV7hufoJ2i0t~&_!?g#_;4POD4+jnj}vJ3UYo8$QK2SEz347j_@$h?K6gT4I-vR_Sy z-q#zOv~Tny;ZdUXCk=KkX6z9PWXR@$^XM(e12CHn3QK?x=rndg;~OMbVOFQ~#h5gC zKf78S%N@uLMwABjF?upDQ3VjQsM$XT#*=C8qMI_jjVGWM-1fH4e1%9bL4eCiUy+OT2bmuz)KVnI838g71gFR3nnqtR9nX5CPyTTE3K9e61f`5O?;0 zcEppnH?qJ_K$l(w)|z7btQr%EdlRnzHbMv480`Ht>Y7+yPqnemESL?2Y#D<`s4c#a zh1Z>iG`s57v5!u6?s=lX5s{WA-^!HHAEq&qByNqs;B)XF*tU$f7~@zgxW!nTP*On) zY*3W?yRa3t&?leZj3)|DUNOZ>ij3K5eALv0bF?~eHg(J9l4jJ_^q{{}!YYs_@QGn^ zm&QwtUmm=cj#NaC8Rm{QH5U|cR-nI>N7e;TQ`2ewR@q|^i|uR2)Q&-b6cfbAS$E-m zQujs4CwUQhb8JG|L#{xvQ|E+t94E{Kt#nEob*wqG5q`>KlTBNv&YC7Ph&cvSBQw|Y zPyp#t8HYw@?BmxJ-e|{<@q=O{A79vE?zInl?DucoY&u$?Lh;B{Y3iL0WPw?VzEGME zZMB66u(|De$8HOI=q$h~Z{-+fdPfhEO4aA^=9s#dd~rZf>aJ+JL4)>SS;_(AlOpiY zW-a-n57WEvk-A(TRGzrMbBRvc#(@ZD_FTsRrpoog-|H7zonoVySwMw3SN=yJH5^7x zA3~AbTmNc=qa+ip{LHvvrn>W{f^u+It6r~MfnmSD&%O;SZFi#I=lueQ*P|< zPyinZ*Vnzm1#hN8t|lRQ1{Tw7;#R|xT2HFp%T~tWr|H&XEdd^R%FnZ6^c=sCZQ&oY zQ8|Yz<0k;r{HawkmC#P;#Zj!ovPAJ(f;m%!cU;BceGQ@~nBxs&0XpeeX9>MW;}gzO zq9Ir796t%0zSD&?@4*8HGdR%-^b39e4tJyuwYaVFOi}f{j-xLtE(46i`QO|VFC2Pa zt;P-x)U`O{N_mQ=b?qEtKElGv(BiN!WFJz4M0}ZUuRos&yq~OVFACw3o3pOB`j01qE$&gaX}ZrZV{alU$NmB(3q$*mm=#G_$9(Kff1R zq@+|)nb>$j+AjC!!eUhw>FjHgAsU$pT-~EH#*>lg=V$j~YYgW$Br+!r3`Hb%ce@qU z#C1MJ<~~)l4WsQj7!HUOoX;gQB|fO7@`KU49e@5(^QSrmz>ccd(3umA&2Yr(p?8 zAcbRj;r~N4ULYhT+(kaTyhk_2q#P$&0KCBrZ5FmmUBlRp=rJ9|TsT?-unb$AcvyEJ z<)GCY&pzrRX%q|$CgXb4rUC^sGUATfiW%DTWwHG+CZwNxRVSQIgt0t7d4ggKPB)>9 zj0_n}h!D^<>L8a&uLDeU@8G+Ee)&)nXIuk+-Z>AEp7zwmU@HdaG4f{2R*!V-MQ26J zx^*C>A!3|l^aI+(?d*n-U{d$PI-t_K8A0=?nHQ=7@lE7{PJ54puATgn%4xPa&t zm$Q`FH3Jh}UO&)8*z8f*?k$hd>+^`J$?H%$C2hpd^T6uZyv#S)M! z-tyN3&pKesUfQec|Kgp#Y?CIknEr262M9;;Q=C9kY$vT=BSO`#v~584YY~+YU>%8n zJt>s;vocRJ;S!D?FL=sn*+NCyD#N=O3+&QXOW7a)nfTRQ~8i5-xH$G&5Jt^%{tkDa(gXyK~B;^L1(z(pd?^y&*@Y<%k{lf>q8_Sb?nk5nf zD`d}6XaGnkfc^#_z8s&f`F4`coN;XfEQIOzO(Qz&L(Y_3ejx(^<=>}GvmZr&nzT~xs^?dv!T?UXk5=t=|6?2FG#Rz0odRbo zw^H|mV*2PK9?h0$KO}~I$#)2-&7#r32=k~ht0Du9(-I!d^{>u024*=SJ@FO2z_YYh z`wRAq2c@l(G#2F!pFxv~5r7CCSTEkZST6fzWiD83ioEj;H+NeVzrMUB5nrW#Q0?S# zrCc0Y8}(+fNtj2tZ{bp^4)ebYNUc%;4T#gR&6>;!w;Y7=E(losj&olA`Z@(2k8K^I z$&_5L$~HWq_crUhl+n^VUwzyl+~xxx?8ejAvFHo5Q985(9QU!)kOnmsgxIRrLe9v4 z_59T)9`F%T&SGVF0`FPDQsx*Cr&FQlIJET~{l63)Xw);)+0@Sfp9$sOzUOE%y|pN} zW5<%ESftLT1U7x42F`{y>k4wqob&6=BY=$OXe@jDzw43@rhbkrQJXz4U@)ilxcZ3k zn-z01(L2QcJ&CapETlimF_?rq(FO&ddBG|(q2YU>=se=_UX&TmQc?|DsXYY#Kpq8BHvM2dX=e_*lfow7C<*L+X`}Rsq z?2e>A&J>JH=O`9*;Cqw9f+cB52rmd&b7VGv>~_!?hJDBy;P)II>vFt-33#<%T-4-4 zH%H|IXZ&7B%S-Lpb?v-tZN4qOExTx%nN!jCwu3Y#22kvnFe8Sf0Jq#RLyPmiU62KO zAvfMk*9%?0zaJ)F6APTCSGK%f6pRiqQHTp_MX*0;r~$#ZvKCtD!R$5+d8uCG(?r>O z2X(k@-_AFkF=WQ$7o7IkL%yriwmW{CI12V|8@T|oL~3lwRJp7MHKJaJ&Fo^A;z4}z z{`Epm1%3FgM7eOD?rq$ONJ`2!I1V5G4xFBJ*l@5@A$=P~`UXiK`4&X$7ycnZ^U0(7 zWkk0}Ldk|EDyj$7-@o?(Fw#t|XEb(_y1aGe>@TDy@a&faML)EDlu{Rnx5dZb)eyDM z5)BlJcD*e8_9L?8R+k6R|4O;U{R`F0;7R)rC4%7<>>zjzX7%N8=a6|ngFJl^NT4+HZ3 zfeku&vpy1?vr1_vOI)O*hhOuj_hRIJtXw4mSt=lYPG?F6+ri1x(#mN}sOPC8mqq%_ zMn6S6qi;>i7k0S+|5kfC1dGL2Ps+HP^o67;%o9C?T3d8g=`*VupFr22#rdV{;<(6Z zs=-6!^q;)2!XArnhEf?XF4u=bOoz$~8pioXd&+!~Dm^0YXQ6n=6REJLQ$NtKX6xeL zkSzS`m+i)-nG_@t19{d5-fqZ?3r+_hjo^B*Y^=9Wrv{NDM4r$5`mTB0Xeu^6l3~RkPKR`jwdv9~Q6HIl8S*z~Y?->>t7~9%tR#OmDADr;a)jNp}F|vIA=6Z zI8RQjEt1CMY752uB6yt3fX$3==A85F;Nd@+LWLs0n9Sb{VK3ux^B)DRo zt{x682h1gbxi4f%zVO5Te^XSIHcfz!b@$Y(`RP@52_1z@Mae-b5l85#-A!ABmnVza zQgN$Lkk4*Gxe#mMRb9LA8Lp7}pGB>*wv82ia|Q$(u&lEp98Tqix8d%i;l_oJQqi5} zmVs7^WCSW>SK{=Ys)_OmAZ9C8@FiRJF@#hR*w6du41ybOm zNOuRH3?TgAo>BA+o4K;EGXb-qfUtu$mqB|X2$Qdj{8aQU!rerKkbV8^kb_7w^C~>Y z5y?;^{&KI$y%rLRVR=JjVjO$rH?PsC zz59zYMWe(&uzh3&^anT|o>lO!wrqr1_vL6`xfjbFp|@2y+!n#+4lm`TE86kc({OeE z91<@z6PB3DcY$zx<_wlVQNpG@kZT;MRp%6;AuG_KE#7&1mlF?Td>Md?NY2(DTpTZK zyrzt!@4NAPgcXZJ8eoiYgiT*cT!nK?s97p?c`}UH`re4jNAa>edId+jP@A{4?U$1?WN5Yr1Y#HqLD}M^T|b zGvFM{7AJXGJ~e5LWCATT`|L4fTY1dlU1PKwMrC;05_>ZVFQLviE8KEBPtS@jy_DEZ z$YdPh%ZG@=AcAhkSK1$MSRQs)>Hek44lM9qKQ;8S3L8r;^c(|so=VbEGRLp>AMXTe zpb=i{`N(za+-7=-k!|IUeh41Am@Kb0{`})NqJ#3bl5kgno^X@znMLC)i66yO$roY~ z1lq7DIll@;zBn^$nZ)TSH%GY0Bh5%kD!_k5CB<4N4`DMTa{qPLq-jPu>iyDAeWB;= zUs)RPoctQX4?vT0c&yz6T7OZ48&$&UeW@m7Th^0Jo&PrZXh{?Dtr{pwXporTlCnW9 zlTQ&DCx|4B#`m|+jg)|Hxh=74EVvIFcdhJX2elo8$vaotYWaR|nG@gFDr*lsgM3Js zO$G&@vzMsXi8hgfXv{c8Bj_J@xD1IH04kd0qi)yzp+!y}gDAqtGdv#EfQ zBq>?bDB7dMM-e|99kx1XS%gLbzBuX;E-0>Gt@8C(;~JpnsmB#3G{*~r#BKkLfL_^_ z%YprUSmu1O(XFQ``Jf}qm?9n_2yLbU42{s@b17Uj$%ZuhU7d=AG@?F4;0DL^Sbwuq zTnVD2^M*bM@gxC33=R?;R2*)BGF0>v-$2fj7511`XszkU1S^xQ_%Yq`5dB)bNjGq? zy<8CX3q{LeayeAZgZdb(8_vbsse&LddR-Kx@k7> zJ4Uj@nLG31y;<`Ao6xds5vfAtWJudaHZ8wHqjjx5auYA8MeehR(CE1hy zY;}#-k#M>DUa5y(fkZ@BV%y*)TH@bylvZS@mifkRp8Ae#hs@Ip zSNlI!@8_9;ZZ1CH;f)QV!ephH94p236xoi8hCmXFSiar3oOkL|c7v-sOk*FBUBh=l z(MsPD?&ki)w}9`-j!7-1C}cmjm8r-4+A7tpJ{GN~Wr>vWy~XAw13#P!jy6<%0L9LA9eG4?*ASW=4-oNEcs>fJY7F>9Y0c)-;fw$b-g6`VX z?;ivdvJ|prRNCzAF?!urTx6ZHhiZr2i1KIld}7CvG+_< zC+UpVAwHpdPP(tbyIA(2#s}+JR1GnMELluTG!B**v`gej0Z##t{;rquu)<488-rb9MK$A-K=^$-=k_< zj??RGLzFZ)ughvwR{u*UhaF_YT$;2dtL_B8 z0VC;sB}}!g6aq#s!V;pjr+a_2QJ5H|iPe3HIHLvfo1{)J+zsc!_&ERiL4E{uH#z3! zvUau5sa&%pKUMd8+?zLJN!k)x*J<31;y3E<)tI#5?z>WhXwYv$@a7j)e=9P2-gBsw z`;=I#=>7@(x1G-Rg-abWOFeq~5!6-nY}!{Vc&v3Ef|^sl22?|9tC;S^dSZb(v7ot9 za?aa#7>!-0A%7AF*t5q@xW_96TiOT8=B;=*EUqZ{+)3I7xr$uh4mSHMGuHjOP%9E7 z&C$JV_jK|13PSeO>Yvy>XT!)AtmfMRpVdFPWYDIt#)ad|xbS%& ze_$7fnLrm?z--$LpKDo5gz7p7g4>=5Ax^AZqSoLD+vGITY!@6!f`xR}*j{_Wg?S^M z|HYFDwoO2O$3b=^g^-s<9ovyeT@>&8@Yt!*ml%<|2?6P+gd{I;nFDkN;|LiH2DL~pBm4-SgOqi*+Xj#89ut(jbve*cY9&)cf zn^&JI%V7gaYLU>tl`+(C@olNJtw0C7_Q?9QxNgnsHyh#M#L5%D^ct-y)}mqW8%CvT zyaGdO;jW^!Rm(vecD90~l`vbt0#fGEX7GPhar~vbL)oXiq8g^5$3gcZ4NY7sY##X^ zibboV4A9q;fv%pw^(Ydb(wYsE;n&&9Kga+tM;OrM-7J|S{~6qUQ3zQ=DC~0O^KRO} zJ$F0tz`**N#^G}?YD53W6TM|fYP6#QN_vi>b!t?F+}296Z=*)nH~q#rjBv(X%NGCP zxJ|U=ucf{Y1FE{V1Zq$Uzitn&wBbws?PL5qHHk!Hn?4@=qP zakH1~?b0mdYfW1Yu0g_JLJ(F^7Dxq$bVSV2X;#dF0Zsv9gI8uFihva$_jl$??qQK0!(0S$Izo4KqXY2Z2=%-LDGOYuTih?PYX z+|s=IFVtME_R3&c`+*P#5HM|#F4x`{6yZH*0dVh2xc5-$1>Z()!_+8&l9ye`Bzs-t z&i(8Ig$#$It~}xm6(WiGZjJr%tUYriqT?hR6d|)c5_io&h^>!BR>I29gDyFs<(5TR zoGE|4UZ*7@g08ikZeqKuWI;U2kSy~w2=ok0;~R0(CFwV;M~8tVIz0ZUN{V38AyT6i zHfRMX#VtB~?QI9?XCs2M5}*ai7Iwc{fsah1bmYVnMAR`Qcs&4hJGRLpBWorCwl<~T zz{d5|M0(LV?Mvm&YTE)e&polO>Ry~D6k_s`HcvcaS+?5xOIkwpFed}#GMbn#3v(-V zg%vmwQ&ni&)Z7MC&tSH6yeq&5b-j;p8^#8-za6(2Vl8l1mG9^?WUsJP6qF2SFby{&yw069R$zhSov zkc2xal$%At1@7@NJaU&>j?49SRsMd}-fWOJ@Xvjsb-D)|zW^J+qMX{?C8_gVPx7gm zx1<4VWkQ10PpD~REoicn<*p-RfD4N67aKLSj#KYg(CJe7=%>|~U7zt1uB-hbwC@r} z>^Zbv2>K|&?jPe2XG!8^N~2K$Ce8?(Z0O}gR_d>pP|3P4Z=+n)VT{z+juht# z9=Eag7QZ&LmK;W_d)aQ)k_sw=ubLzX?YJNq3r|(Gw=~%$5*8+%LQ~U$2hYzT7$zO) z0t=B!*_mKfYughX@hdY%yjq_vSuCd@Fmf7L#OIa*{0k;fj* zz(t!%p9+M%;*WVV!j_nGz+fhU#cE9UmvmEQ6J*)e#a?iYXSR0CjyqS*)gerBOI`4K z!VWpOa@J}=FtHoj8Q~ZtEJ_S%1p1LB|ZRovr4x^m5|SxE04qtEA|sMGX??sfkg zH~in4Oc4!81pnWd5!j8eSw1yc#Sb-41f{Z1f^f=i6&_dJ`=8YsF*Te8%%4zg${{|% zjz^zPmeN%yxow;PJKVj#izX;*cGC=EQ1TzFZrh#lcp@@oCgklf`(TiWwD(SK%QPJd z?vy01W=Q?&o}xCPwfk5#6rrESX7A=d2ca2!%y2pnm4#1`k!AUJmWIZxM!rCD_CQBM z>gqSy7LZ8nxNy4m38}HvhxnqNhwe&hLG!?i+q7Z|QQQ9`1bteWR#->z?v0gH1`$wXC@GPyaalf%!_}q z7*G%p1QgeO^w6jN4EanX`z$=2bylW(Rgw<_boVj-(}X(XNc!FhZoVvfIedz%|eAH6a-D z@f_ODDm`b1c4`;>l4kZRbSWoF->4I zKI-RLdhKGXb0jJJk1z5l%Mic~1cY<`<p1yZ+B5y z@yADN^iSQva2u+^hbyQn*n$_ff#>=`)uGN@R(?` zfoKHf&v3Wx+a0HlL?E6dA}KT}&|BIWqEO7(U3c;(fYga|`C40~sot?)eK{{;Fz6+M z{cQGVAx^NRSO?*o3zk$OR~8YcK)(6D38g!Y^s>1I+3^fi;6AP!4djK|n70No%6Js` z|I?fysF?u-g~-|;%L z=LG~-CAXjDxLt#IEpfV0hx9K*8+A|2KB>I7S?&w0G>Wf~R2l|g!>Gv?bT>GU+7BssaG)G!%yVG0Qm zdP{Oh2`K4rv&R-n@+PSEJc*q_7(g zTr3^H#2TXM+Gk)CP85^FarZi+y$2ql9dwF4I;l}L6btD_tAda&?bLz{IrxKD=jpo@ z38(YBHY>9rmT@zq!-lbgm7HFFaB)0G8~1N_6Yq@Abw^P`TcNj46tI?KCUHEu6BZo= zAJN2w(J$*#d+@+_>c0dv5u31v^*sG~sDTomVk92v&+T5v;!6_OxQIFZiq##T!zSD! zuG+dTlS&Jq=Eac(12b1FZkP7%sF;FxE`d}Ear5~Za~j3on5;5o*v>?83W=mk_7Jjt zMnkZ+H*MG>V@#C}5yZqV-4;$}Ife}-goBZRb<_=K*}&35D)LM}m>KE=T%%WP<1EMIQG1gu!IY~Q z66Ih#P$DM~M#Se~a|?;%_wXwZ{OF=QB@vFfsH>U?JxaQQFS#Wzr2t0gMXjO_5y3f2 zpW7Ov5nufz3cDNf%a$+94`dJ(Nl;A6>sEX)FLG zV0=@7c}Jza?uAXaqD2U<%h7O)i>(x}C|jJ0y?dK47ILCBmaPywZ}iT{NquDLZA>KH zRLwsRQwDOPbu$8Fj{oN|U44hc99aS-#CRwz`zww-IowqnNy?}a4|RudORVcw`b2A_ z-}KL&79@s%m1h?U^kzVc88iZj-jZ4y27s5xP182isMo||$TfN)3vs;5%f294FT4WK znowwjCzLgW;?uaioEJ|eZsK9JNn&150&6EjIiACZzZW}aPgFgi5Cb-5JBsXbYN2Q} zE9{(04x}l^WRESr)W%}K0%v2t@R4AsvGA3$m&5s zhcOxDRczZny@#vAq)H2qm$6af1azJUIwVVQxIL!jngvLCPsQAC0G6~Mg02|}9i55W zZdYF;bISjidll|cd=4ACPinjLKb|iT`yeiWv-@J)E*EUc%4?>ifz^?a6~>GSJmF>r zuu6G;ko8_y$Pep1!QbUQdQtW5#>^O%Q0KJy%tn%Rh{uhxlv$}FH)XBPdt(H%hr|$g zw_oIFBUHFo11c-d7RLEkpgfe$A(DV#MPQIAKI5Ul@xnZMe?ADi1-ajJ_vwwe;dAz{&mOpBmi!d6j(PzP}(knQ8#C&;N{j0G|fu9^h zG$Bz3%%*9)F&*hTc!yxUA=VpOH>=eHYIq?Br$NFq9UW8_ufp%WKfTBgJJR?Lxhn~s zYD?Kwy<+-;v})#I6+G0^ZDl=dL-R#|lS4RBE5;b&tqm(=!)|7HT3L z7dMq2cGnnZn+X3F9!h(mFFrcr4}y-Enjw9<4xAo>eOBc#aNjlF&ZIvgFntaZbJ7kp z$4YVndrIvNKSceae<-r%z^p(RVq8+7IDN4}T^wx7T)+h&^!821rh`FasVlb+gR zg;Bun)rWvTBK!}Xp_k(WmP<&Jem2XzbOGYV?CTiIRJbeGc9lYSJ&ZcVUEUfi^>77h zQ8~vU-!u~+ip#zzq}g*VYot>NdY&hDPeM5lXcrLT97x30V_9}2N$(kQ>t*L4Uc^ki z49{${dtZXZEek=-4(Dvy{xD9QslyR?U(#&DjX_puvExnV$(BFZNTl1{;YjkV7EsPc z&~F0Dvxs^+3rF&*epM{mX6P=m$<);TPF zY<(KRN<|t|s|;BkWQwqY9@zlTy8RR~`QqunP_r<5%2>cM7spwQhf6E-?P~#Bm^X{^ zd_Pio%^I7+q*Jg5&LRlYqB9{pDag{(keW?$HMD@X82&N|QeDE5h*9OB+C1fKsOZR< zHR&YvvES~KVy>OyY%~gqCGV7}pjyjN=&??^@}T`l?sNf!i#>CjcxRs!-N#O-&r>x@ z^{i?q2??KFQWHc24Qe688N1*OZ`H$iNBqsCRe3D_v0Nn)b$x zMv3^EAHe+`QL#vK8dV^~!tjr_4H3+OllSP=)_ke0DL{(>#Ag3LPu@#nAW@A*0ocK) zx8ylb+=-#nx<6~Ck3SgSB5~&8Ag_2b(gtwX;D&tt5i95*OE6BG9-8441y7?tUVh!d z)%p)~M}zi#lSx~Uxoq6WJdmZJ z)9Vsfn}ktIb?SXJ26-KG6JF~qrN`@3P1*nuP)%02o9h3@?pKp3^Q}VMP_n5j45#_@ z2^-(CJ^3KpbJpc8c4~}*b@_RVC3<>YwoA`;ruwyCh#61w+J>qcd2rMS?6yG&HRo$9)WuLt z3+-M6Hr_vbJ{@swX1@qo)C24D>KnYD4a>`E?fN|dq{L(5I3}5VeN_4p)MD(P(ocE}s!KR*4g{;k&+a0-vZiKemb%Do{}>V3=eesJ z5og0){jq7TA}V#y{%#8GPG(N9U=~RteCC`ec!&k_#y~0XN7KgvvI-h0*dUHx4!uxR zfUF;U;HX1%!K9+_m*-_^jjBt6l5E$S0uX?2Wzg^SfZpavvmU-_9+7LCM9}8w6J_=v zeapf(j}Dos%}DsA!xuu7H#{!WQP=jXu%k>)^B5q`MUl##FuqyWabmqBOR}v^akby% zYH-u<^%xTmpYbH#pjvj2{;Qz~rZejCmbnxUHF_!PKL5aERW~QtNEe;*U$UQKvlIjr z8r!p{_{!x4?@g^7QbKT^sx=f#am?S`__E(ISbmni`@xYB@BUgA9y6wrWh6)D1;Mbbv6K zsI>-7RS}YPIkJdOs|&s3r2@=V6`M$RDKx2kosxkRoSdmjHP(v5>BAK`!)hxMDfT6g z0zQ^cun9T)1jtJnD@SS*$FJVaWX3w?r1eW1L9km&+MW!>hByIud8^WD)n|Y~=ZV_Z zmxhE;Izyw4cCC;`H(S_U_~Qtm$&~Gw&-&S5B9fHd90nv4Cco=e2OrE}o&45@7(eQK zLsjo#BLmnx2KsYdSP%1;u++~nrU#Dmx9lG6v_&m$^~RY$HBkPOR<@9B5F5Kwiu8;u z>x}@p6@~li(ahjM>jym+BPNZ56J~!4y39DCoQjYq+()2x^;t7Dc zyr!`g*3nn$sWRB{2J0qd1Tz6Mi_Z?9GqQ!zj@`|t4>=~_(U8|#&*lN=!bZRpZS)ZY zK8t_4TBZZ3pDIHv02%feF{|{YHKvq@_Xe?paYw{z1}l7*XzXc;e3^@P+xPFqKKj9 zb{T+!IL#(_$HUI@JD4pXZ|~tVc+ozUYsGY6nL$tIY|^0mmGs~oVq>6?=YmaKw%Jaq z!Z0aA?{kVp?!;ZB=Su& z8^#k1qN;n{#31Gi85}5sfDpsE9&r;rQO;d#=lBW-J;N_^Mzims>vD?hqJ)Eu?fp{` za^fktx1+YhFiaZ3M#j%^+)TD?iG^OO(u+=2B{ef3fpiybuKXcGUa`NY)v>K$;VX&e z>1CajC=q4H@RBKLIYsbU3;QN?Dj|)zndzgsIQr>!*tA}V)07=!yc0GY>77$EGlFi2 z%X&&H;5T4-CWf;i|Ipr5^yOqJ~cCq$+b1-Ei93A1MTlIUVQIxv4* zxUjvE;Y;J{DbhclB<`@>L6povMvKjy9W3`An|JLal&G|khll5p&|1w_A(F+oVTzk79lVK3T z_>uE6hGHoGl#n}cYylL_3LuA-c)>1atm8H|!`z|75D`stCaCwKna@~mzxFyDon-^5 zre#^R?%WGJpPq}(W@x0$Sd~VW02I&bpra#3cI6nsaCUR{_MQrJ{J|kNF@(x( zmJ_YbxMpVZTIT?j;Oc|xFSnH>@1XXg+a!#S*Z=gfT=QJ84V#@}ermywqRsI`0_ERl zZJuzi$!C>#bAB_ufba!sbuCXunu+gJl!x!*vqjp}d3c9NE!5W3!yy<@YE?xg<3$T* z3yP-snGuM!Vp~4PhmZ=f=kaQDq}9+>ERu^iUdgo61*l9sX#Q2(I z@!o7GN^El=024U*abQEyUdn48UR>2XF&r65al)*k>ZRjM1pA}*NTLp(ga`$Bzn$!q z-=IdTaY3Q$;`hF;Ypf545}5U5A*^D_yBE+qt`9A$|48=sx(R)D}=`Hhn7pN|%(Iw!& z=tdE{44Cglsld(AWRTNC!p0ic0n#O|{T{i9TMZ_5@9$?d9cbWD>R&{3%b+-QOXWz{ zGP3~OysB0(!ndl>)~P5gUGn+coNpen+5djB2MyyEe`L%V>xH0)6&k>@PcBk)IFS z+zH!c2`~jA(TSOx(W-T$7aP#1+&L9E#_h8r6=q+mC(~A8FQw&LqjXo!{tF> z3k6xxb+RvWihY#3W>;td)Zx|ra{FUNAX(Yz#L2Gh5}e~Pu6ES@!In02iC%g|(Z}G8ytM>CyqfW(v393tdRkh2c_;*~BeG&T8q(aOV?c5Kx^& zhVPLTG%T{x#!&if zh)tQZJ=u?7S%&Ud^>5un{0QA6{;ovb=)cfF`*(ZSu?eu7kyI{lv5J$o!vKduVxFx_ z&wL|WE5O%Z#%7lAdH_>EfsReA z={}PU2tV8AQ01OqZsILlJUH!3%Kl**lQJd7Raw9Ge=7O=I~GsMJEu>rbkWkFQv$ap zThgVrkdRvT1ELEzmk@hV0j1?^Rh)vLs4&zTH#mG;A`EpEfLKz$ zQg8o*UZr2Kpg$^ofS)M1IRt+oU`nQe=UKeL%9SY|sU@-Pv=h4xN~FVOR2&ctU)6@7g^x+{jQRwIjtBO{1 zz&t&qr7>8oP}wOWEMu$^NDJ#}!S3Zb8mB2unqqepNPdwI%L@c}saEu5mj6{gF?d@B z2?gCK`qf14OvM>hA&ma$wwnoo*#luYlc;(ElS>?K24z9&gjMJ7hCH0c3L}8B|Bp2- zA@IX0@pCjA_%2NxbLcn?5j~MJzH9t0hkNkxBW|i_Ea^0K-r#*Kg)mnRug1?DD-VU&PdwB*Oc%+Mr_9jjev`YeYSeiPiSP#R96u(hGb?e3=XR0|A&V;=${q2#&gsp`;z%HJ zu3-$t<4_l@vA}mdSILf2X5PTu9AJmy=TJniC^3y*`G|~xX5>Vnj_z1#b`9y^T#~}l z^RJs0%KP;#Xxvf{W3~Q2$+4rMhcQr9$+_%wn3aNQ|Z%Lq(w^JCaba@K)121 zePwZ8vb^!jF=8zbA77D!9A^xTzyk^ngq!3jdiX@w&>sm0EGLaRJC~QfkQ;M8+};}z zmR70Ceg&7muNPN3=I)eNGnT0gq8I#IE!Y^;%YF*fLINme(=ic9naaLh)5nC5uN;10J6lBU!fhE*GHD z2$G104aAQo>-mkiJIzAcHdQ3O`2N9ElrBNfotr?_8LO%4o&9`Arloqx z4ZyKGPg_VFyKRZaUu9J`msP^oT8!0y!t1(sr>l^QZSwRTA+l}l0C=ZS)rk$cgeBW6 zNW9W=(!)xFJVmjLd(DIp#XchkEtb-;7~YpH5m3$UOSbp0B@Bmg3> zky!!w*J|IG(=~DWBhT%FWQF;ETD$zM@Pjk%WU^^wTM&fvh>YOpIu8#6pWX!Coo^pS zgFH4fB&y%R-~kh|gz!P}GyYy4|M-I%WHQNSlLN3Nl3oa(tC&qmLp+)olDznvnTz8c zpACH;?jQ3m(Ax78u#s2t;6jV=oZj6Bb~WkDDh19UE$2&eamHh|%gicEnhR~tPp#;C z`_&e>7Gt+G2Utt3-?(0;4qa+=XRWJjIh@E3xn^ECDrO*C<96p^xZ%u*noli~z2l#x zHC)g9T^B~FaS{-rNH)&)q@cbV$2hBTnWD^h*p^#*W6vT2=a^!vo4v;Fv>E@N8S0i% zU~CMGtvkF@Ey0HM4<&SrblRHp7QTVxME^_ie+B=9G({VIC%}Gj3a(s+c9?YD20N^# z4f!pR4ftcD(@WdA;=*77L)YN~%t8%vK1I^ZjoSMUSX&HLq@!cp9%8lv3%><%}9A#3L%T$<@=q4f~A+t#AkTl*2!# zdOw)b(ki^&5QYSPIroMSZ(PF76Nep_Si>zzLL%-~h4CmVGsIW|f1G$%Pu}iQ;C9eK zE3A(Y9Ga-aUt&;T%---$Wo`YaY5c$-&rca^&>89ZF#pmZVw%pj17VsjQG)tQ)@K6u zJuu&7!N0zU?ue>&?=4Up7CZRRkW|3+W1JPbs*aeaiaIA6Np__olbRqdmWT^2K-=!%q*n@=qWo{3+Lp1b>+yo%H#@78 z*9$ZvfRGX#?Rl}Pe4gRg12QI}xP49iRy(L9|0$BjuS8`Q=Iem2lYEx&^OHcpJ!V`^ zQDWW!|9}J-&OW?YloW5Pmh;s~;A+qZnYr-#CJCYBk<1x=k`fpSaLSXu{wA(iEt4V7 zAfm=?q!4I`EQk`ZVnP&f$3??;ofnd?2R7UtlFTbImwI#muWgO0(%KJ#_SJ=j(R4PU z)}gx=&y~88NQ1rnE5V06jC(&B1Ht@}U$WDs9@@Q=F69f`F9rs$CQk@*f>;QHpfzaI z%k*;1r(N&6TcY&k2qVIAo<}X-kvPIQZuOthITh0)cOUKib1b|`c!eqgM?`Er7paDzWZap{qi!;mC%Z7|ZTDHO! zpiWaBX!oS$2Sr+m9<(!HKKTT)Qdy3!=X*|?p$M$!bVu+~oGqk!NM$Mc#~ifGa(UAf z5pT+8`dBn3>}?Hst00AyuFLy@0D+YHqE%l2oMtpvO1@2c>xZM{A08yw@ZX)8R6&7U zC9J?hyGq$cTPX?erPF&pfeEfjKJnJ4_NSD{eYibtJ`yemPuzuO3?6XHrq|&n$mww6 zWaq&zhyT^TxZ$FuIUjUPe%VfCLFRhNKr6p5Frc`RDyD3bfa5|`IdXyWmS{HyZ$t6Dl}NTn=1ohMY)FglnyF9#@swe zY^oN(io<;XnW7YpTg?;x2R?@IhzBb%SCLw4j6Dm+8+f9-pHoe_%`EWfbUe&Bq=!Hs zyOF)n_8-No-wbMr%O%D9Wd2FBj-IjMWZD)v5QnKk!R0*#v4o+Gk|OLr^K49x!&M`r zjnQn1=r=fjM8rWgSk5f&^S>}RX(Nj5JH7=4)O)@BT|P&Hp%%1BEHn_@Z4n66VE^A) z%hHw-iM3YLL}4GzWwGr^StFdi0R1@?(i<}YU!Y=<86q5?_}evB&*Uw^)<>-gHR%Pm zK?QM`&Hi&N?|<&IEpe?Mzkx}xYbwurEKQH7tEO7oCB_57s6E#Hl^Maap4KZ$Nojf$ z^}6@6?W6Kksx#dZirrI*NT0$oo6rbjvjp(b#8N1{YR)*@(z3gkV@yvVYTMFeT;e+B@8b zHiTEwOV{%~cFA!Oq@^AP-eZX>J%DH@Sk=gYk9@C<{j{R3iD$JF=xLJgasznkN^iae znX+fb6k?dj*P}saDQK!86FuG|l_pkX9z>(Uh1XZuZh!{I$f^+c5y0uJs?+Ic<7Qz( zVD-Pz?1~O@(hFI=za6%y0SX%nZ}9kuTmY_uCg;iwGX2iyM>ghFu#BW~`z+@9z|B*3 zZ14$J`%Hl8X!wNfU$eEyhA1XF3Y_k~@(%8%XPxIh5XD@QNEBYLt4E?#cNE!y}`~dmD(S=mk=Tbku`)9!y z^txqxG~!lU*u@ZmHB^D;?C3LbG)02Lfj%ARV?>|0#i5yC@|rIcZ8#+*z2qvQ2<{1B zn+xeKuM~Z0(C&=RiK+Vz+tBlU=_ed7T%VDifK!!)i-P1`-!PKVdu-7jub*9z%fb!p z>3Z{apEsYX<$Y;TgL|Vv{pyqFAi7*1(KUd7D5ckHsVMA$wkddp%NxA|YtHZJ9i^@q z2@FilKkhR({3XEQm4SXL$~HXiG5D*tr+7;<@JReV3}<3;GYXcSKM?Tn zXJN%)*vmoqXuqH%Spi}o*zU)6UJz>Bvzg{ zh1a)j&7_^#lUWL{I!G6U^wu^V!umz&*b!U_q-X(awdyqIs;S1l;}6rYA@l#$w`PzI z(<+-CNeJcdNSioy%VTS-1cdISKQ=fr-4i%6{BSbpsbQGKuVCDaLscI(Pn6-W7BR`a z^#sS)Z!^ft3BBiHg)7Yx?=?-? z#)Xq19ELf^@Un3>Vi*rFhi-0+xQKKd;fYXXeSwC>D23ZBjhUY_HI`BD9eo1rr}A(` zFIWm-g||K{jbfm>!wN1Nl<+_{&nt;`<%N|f9Q(9oV+%DkgKo$`C?vU~ZkyQw-5B9< z&;vtraz5livxApHI!+tQ;7WXT~0MZ0VJooUO*+Cw{sqsldQFzc}&E z3!J6xnfGI{=ZvJWni(_NyBm8AIY0;J?U2a&smr^zQP;{CK`*a>SM=A&2I{T@?qzOKg? zoMTk{3@U^cBSg-6uIzW2n?|07Hk?c@S9PnG^_r`!62($v#>5#wVFsJ8p;9c?6OFyr z3|&*q&l@Plsd9EzA1^Vt=YY>E8$X;p+nL^IZ?UW$_O&XUHei@RYEOYA{(>Gb=m$3j zV^7drnGt4&+krT0qaybg+e9VE;zAir_(`ERaB_d#{&;DuT6va)>rFoDGj|(VR5xMQ zq*nnFDV69R{kmN*LJOQg{>#!jU7v`@^3sWQ*hP&(Ai_(h@dX5>*g899Z7abJ>sOB$6N9}AiSAcQ@QrdD z**(4tcKW@pjYwQQ^72_R^IZ*o&fPrktzTgy6g`=^vzwi?j}%HyC3 zD|@W=9ksL{KAqZ|vI&hWAxz85(@F%z?|+(EowE(ec?l2#ADBS`EL;(0?1838b5vEu z5H9`)G+PRPi(shj^J3_=9?=sv7M6%$kXS%NTFF!o7C4AsQ_Pz&;e{r*r`9W2?;)}Z zh^9-JAex%@?&da_e^i6LP~H`Z7vVX2RI2!t0<(qlg!PEvtG1(H0}k_<%rD7=%?SQ{ zLR_-jMW|ed?w}*h?@ts|4olQqc3igWy3u^BkC0I;WV9CONGr0T{}Sy`Y1)!FP;XH+ zBJv45J&6utY#fi0!{A;J2_vcWT%-4|>Ys0!&ye7WqyUw>Wgg%X7P{M>Mi9d|VVTZN zWF85a=y=mL;IR=bam%!(YIm|F3V@v?%!hXPeHG=b`jLkv%duGZf;|cNT=M0-9`6@4 z*O9ee@jDwEj`^}B>WLY8QeLDjj=x;%cVj@OU|LZe=#giD)63K3!(99xe5=DfX%$+aqxK?wplewR)9^Y5kePZm{e72JH+5|qo`8DK|bBq{>5g;fAi z3b{TGFStnWLNf{Z1&8sO1Hw)gFzbP!rXG)(n&3f#aB~!v9jA={WvO}6lGdhBOEpXI z34oeaO&>Je*QR^W&t+l4&SeQt)kv7N?VvOAWfMLWy1P;`S+<2yPL0*kLg75M?y5Y( zf?Ggw-HdLJp;1*{?JIK>k*ll*gngjXmtUA2!HsR^|IxL(`v*|wik)Q1?Ig=-hr-4o zbifjPsn&}Ik<3wE5(&1V)o=4L$ZDSed#*kw_&B_x#WcOdsxqoJcCKSmS>@^_!L4hWx_z)uJ4)F0zK-`#X1x_+Z9xl)(LT(xB_p-Q1X|BpzsgQQ z@^Hq3pyTPnoafii_Mv4=+7jX}(6&;058m`R^=%VbojXKBroLH@$bnh^c68y3-3JeT znM z7ZlV;qNY1%NH_M8Q%#(TLfJGjr}_ z>h2&t72r?Ip(dJ19xxiYoZM!AoDqrw&(L|UAdnumD|(2i$JT>D&@iDFl)-8wc3aJG zI*L+Pg>Wj#!Md5#@W){2zpR+Qf^$+KG|iRU37=PSuT7t-+K-yfleyY7BVOg^BO698 zuZ~=Ax@|N84j1Lh)vEdn4Wt6WrvFw)-rBDNaV|+P0}DYKD7jPP#78|wKK@~Wmf*N8 z8#5$+@dWc8*jUJ%^1t%(U5Mu6%ImWW>o@ouyytrw2CnvJM$QZANk6-;79Ae0Jx^sc%ys;7%`z@)GHu zDUnZ}FZ70CHnL9}*4erOQ;IBz0jU-OHaBhlHgRf8^W1sJhSo+=Mr9Z=wb%JAp1&nL zYT?+yPLNTwrK?_;NEmOdqSMgbtaXegl_PD4rbEUa)hJyh($?2DWH>Bi)iMien3{G= zNCJP;Y&0XB#|a!%ht+m-{E|i1HS-}|J_wKB5$~3X^4Xt6E74zwV31rr7%fMCA>{X; z+0~x?{reCz+>%VpwJg_s;`q~&&gSHi`~dK+ojClNgX?fz%8)p?K~z3{f)lX6Y_TP@ zmATPiQ`DwIjU5cu42G1zCNNe*j*D9odpma8>4(p8Uuzwrpfa660FG=AI|lOe8+B{6 zvu&|$|b(j+_h0-|=evZdc}`&{9kE#3Y!x<)s+J+G#sbDkd*!BLuF8y>k1hwU5H%9M>fVR1NE9QG$JA6s!YzV zrf2iO-(N%Paay@)olHz=Pfyt@?Ni{(_^)qiZMq9B!I(V2BD32FAOKFVTy$E`YEGnmGc^cC%yfA8RO1KBR@X}lTq_Hu95=Th%cbJ z(b2XzGUMV8G;!Uf0pc*=iK?w+peA!>5vno)(GxDI2vf~ac*TOQkU}D%c~Vx-bw4(O zED#bt69@3W>#TCXwEOKH*Bpl-Sx7>_-KTGQ>%yoXACkQiz4TSh>B4O)Xd6knJ^X!QI+*P4@ZmCT}ivC z%=xa^pP~zu!z&%qnXfzyRn!^csB>+j^v1MtwQB==OA3uUnmzMsPBy&BRn!W|FB&O( zRx{$NNz5bw>Zm#maQUeA2PRP!V)DquJ`}cG36$}TMvhiH>5F#JhWlHw<#KoliLFwA zXc$#{Hrjo}s)ALqiM_8Ygl#022#sxXEc_E8pwrj?xZP%mzW~*R3Y@X zgx@mBUpcThv43hE@&Z`$Q*=V7H!er{B}h4eh6svGxLZU{Lmx&&-Opp2;e zshFXlhn3O$@SC>mF)2dIEiW6WM#8&vi{gs_$?dEsNu$K4S@7TYt>1T?A{OL|1S~Ii z^FTk6&9kY+yrbfPo57&4oGHMPf`PLF!-Z9bu13#h%qgg24uEOOtY}<~ewjz;0^Z>^ zainrOsb7+f61p-aVrbim$@mUKa29cA;)FWa}4IfY0oj-8u$|e472ShabZpW zu8tPXEku1E%%`T9!_`1kd|;|T$ae}a6oeRxg-6{A+F`|p4n~I1fGsbtbj7_hyN9mZMpzubkZjD}!?X5N2yzKqDe32Fh+hHB zE|ZUFLE@m%hk-$wK$XD8Y7UxzdAHhwI$mo_H6!5CaVo>}FTon&dMz@sHfq033UzPW zwDzR-JPJqQhi(mVaL@cw*ZIgCK$2GixpIK}xF%gi9hyr{WicB$dPL2X(hiyR8y#Bs z*W>tr9?pHvfTCst;dAzhHg9!^{jfXEG|8sst8HZrA0Q}GwBlw{wja%>$klEJ)v|I! zaQq!J>hT@>?Xk^7`Z@_f7%F+PZIgBS7@2{#8LGMZQZKP9hIE6#!iJob@iwU_%E0mq zddKYz5p{f4$l*$N+Fte3*gh>JW7zWAyH^Z|5GE6p)3KRi@G8?ru2A_>WAlBz?Ye|? z6v71)>8S1QTxek0@2vhi^zmYwRD38xM=mn17Suqf>;WR)tKJdj@Lh8d>Be0}Y9Y%# zkdd-#IcT1|ErD?&>T}1J2_Qf|8SOCxs8M zTlA{0f^c9_bWJC_&5OCs<>aaW(~U+?#Q{KcIq`KeKoXZOJ4KA6%wi1Z&BS`Guk{8S z%VPZi@H)^zBfm*|B0HbH@9S^#@}|B;bles*ss(P$Y+Be|Kzxfw_ql8XkJBZClW5yd zfzUQ+e9ri9^>LM9veQM9kHdz8pLqCfW6N8p@tlv;fvf5y+AknHmBvirN3<~XvAw1c9tCGxe=K>wo^t><7HCDhC92meaeSf(} zlL=_pxT-t#U|maMeNy|C6^+V1&WX5~SfQ2!nR#NAR?na;yKJS}Ub4Jc_X>hs(72XO z+HTQebd;}}&1`xK9Q&z9;I>BSY`6?ht5YpRbDJCV;7)(Sf;^XV&3>4{^fUzvTz2R?zzU?Mmuo41c z`?tMlbB@2t}t}IZiokt{wqHcJ;%Mgj}dm(_c!`d7`Mk7i0~4j z;0$pC@XX<%TbGq)rs&(>rN&?a!P_4KnQTfgLjkPvu_Cb~g)9m-lBJG~pOK|;a~()i zMT4V(tA7{Pc%S|>Axtjr`^{3mh3e}9akA6rwe@1IIGf!3I&jfd>xX;;m77TS z7NRzX^k}(O74E>w&x|y?il+?7!x-Ah(}&VX%u`J)=X7_*>r$G$s(tEn0MWQ5roKLV z=v%p|;crgghUTh66&^+I6T#F5vWo%n6-`!RiB8_YvKO3YknZBAP?lgwdN@-~qNb(~ zI`m&RD}FP9PF#T&DRx8R=GjX^D8#mV92RCQGq*%1nDSb&Y%LPFOO2#asa85bw@Vmc zb-LKjmDSYzwIxbgSh$2By5WQ&loB5b;${rNT!vYDLI(jr4ku`KG&@>bFB#t_O})i? zY*ZHh6!f+_r-O2OSAh{|KFV|qKjvs}*z zeZHlmpeG?)EwpUZ{Imqvnhe;3W=ZZF$)ank&M#{)bc;hZFId7>pPJD4jeR3SDd*|| z?h;=M`)&*_pP`oL^K+Q`UXT^R2%_*a+x3{*%1TF+sOOQBm7l9VJ*mxIbi|v!4HlOB z3$7C}9Sp6Vj%9B0hbhITuwTIsWA#RhF^b@h;LqK6zt+30poUH{P)Pfg*Shr)knbBC z29Em(I38&nNolLq+;wIJ0&VsQW5*5oyMlVu+bE}Vq4tPe=Bo<5nO{yt{dt#9N&YH$ z7I=}-1>lfwVJ*vbeT$|P_|a`nQ}#~JyMbO_mb!R9%jP%dna08q+%IwzpQ00^0r*Ix z2Bs7=!+DNYxsh|_F{eBOz*z_Tak}u*)aLZGg{30K*$-8Jnv$Ye9Vq5RBSXXvrM}%4 z=!Bv?Kz8z8I4ekw9O5aN6L*trDAC~m4XQ)a&Wg%y9z>>i_~e2v#?%84d71l|&UKt; zPrS>S3Z~>~hqdroPdnb3WZ?ExBb-wE-M{!Y0xOvqdLP&0S@QeVsHGdbMj==$`PS+q z0nr*4TKy}49#&@P045y9_wY0GYwJp~9QV9+uOTJthoJ1$Xn-qV$GSyDrwr=};VK=A zZg~Z1G`kZcX6@9tpl9ig4sL~Wi%uiH2K#n8MI=xt{q4ubaD>z3)8x(p zvR+ILK9moAvW*W4i;;gmoE8lPd1z!m^zY-Df-;eCH#Elj;Fa!pAXs4q3XXGZzSJzg zlD|bj`TrP>M4cCgrt+q}-vVhk1ZSKx?cPBMH8$l~R-1_q`kaSIdPWayCj}XnzSW_( z{Wf&Nhh~^>!;;3!qD-Y7rR^!mM$7?Is_wBZXQL1KkvqW4Fhrg+-Y^BB$YKKjOp zP#D&(aDJDDm_JlMK9h45cp|{1-g_bZ=TU*x#6TY>2{M3r0>=JIbW;I(F+Z`yWd5+@R5dxEfWY4j$;C!W=7RHgwXR+c2 zKFH=SjrtxZPNh}hjK5B;&U;_#9c$^0dKFPl`V-|nzjiJYsT_pzQAHvTHAnEB(5;>_ zYAd-yYZ2nsnYMKZOFRwkhXp>R}`#-#z4HlE=@Xe}u31L`10hnGm44 zlWA(5lA&!E6+X|si#wL|;|R;lfQG^p84UKlva!|q78UnEG{IwD>2~@ZJMmN|A^)Tr z$wk1Vk5R`)-PDnMv?*GGcD$5kN4O9vIMz^ob0>6&!VH8K4buF@Q-quePcTlXa8b00 zb~AzQW#i>(t-N4^J&24Co&0MPZ9dvyQP!KZYUC2-v=47s9$k^ZYw4`s$Ulzcy|v)% zus3%De~oU<4rs0Ui+KZ||L8TG@4?;jl0t3~#ozUM3imqf*7T|2%>I#XSR-#`rVn%n z6Ec(;W%rPQGco&>r*g6x0_;+>kc8A_%JvSVV_#lpw@UN=`l+fpagf+)o>w=o=xRvi zYRMQjWu_DPwOi|mdGbtbpF;vHj`Y-J_6(yU^vS^m#^<~aNNp{@KZNsGkqv&_p=o9z zlyC&XQ>Rq=*tSJOAN#ILWnl(+X9NH3`KfESDI91)4dZSfb)LlMZb`a2xd?^Dy}Do~ zOy^Y>r(1j~cJG^(!#b!lKNp%z4;yn;he3O}9Afkg8ycC02oXq=>0^gJ5GC=U153eC zEhYI~wMV)#U8#YEp7F-j$u|-cy>k6;FXr>_DJSV_z|XIh{RB`{yA%tJ&AOr!W8Qnl zVs99M2`El_tH*Wg2X3@5aU4C{(aMeK2%k0r_kAPTFY*g*BoIb>hfaL}h7B7aywUUm z)&FOW3&z9F^M6WjQ6|0B<}n-fF3VmrMK3MxtKCT4){(aL^y@R3vW$QP+{r{Gv|sgh z43yF~^?0&lguq~NoTq0$@3CBfc3<6kbGUs6j>e&#sPfjukUHumB*8?c%oi7e8QgK+ ztA~`g+-?)vKCOhY_J{xMwN!1H z!|RdOY(T@1nH@!eLzc%czXgR7DSZV;CEf)q~h}v)9PVY9roq=3~WBAwo`1SLXZR+6Yla1$^+LJJWbs__m(&67>6QHrV zIp041lNn{>{IQVR>Q#Dd?eGc{35cqijZn7@-&+{0(Tx6|l^jcUyP)}vZAt+Ec($MH zothtv0KyI?1S0s+D$o6qBjwqbIHbBD)*F+8Tw4aqdl305pu8bs<4E2Lu1HT>5b^@D zbp9tZ`3#2D@sBPCbuA2wN=i{e*+Im3bQSD{L-4yx2=*?*W0Gz?MdHN%u2 zuMf95$yVw(hvE(u6X4X<8L9?jl7r5=^Rsf&hVK@C8?-XWJ1%1>93cMW*Z*PlE}u9X z5S=q>wEQ*Cdqn+#gt+ctr2xGl{B{bnV(=o*3*-C31^atTru}{592QgH$AwZpx{#{Z z%ew6mHXc1pikqirQg~@7z99El5V`6l27F8OG<`j2;*RV@EGl30y@nw{_!pa4_^c#O z59e2BPF+wpUB|OBg^VNMRK` zurtaK4Acia~~G&1FOn_GUvrI&!-Rj)YS z--i!^Y_vkX(n*y`m|``)9^X9rrrQ3K*$Rym^#GootQpUqNCd3ZifexX;?Xo2i=6m% zrF3PD#tFhSPp3rph%Yd zfPuQ)^rxspc;L7o4I2D`RMB~;!(b^P8f?eBr4if9tGZq+tgMrC^zw#WTZvGKIHj$R z(BuG2Ie_w;qV$fj!JbUpbGge`*H+h-i(Y-zNLvcQ!Uvvw)T$094CWWdVwLJpIu_gNO+!S5c2wWvS(5 zEeYGYOxql1_u#QSgX^ji55@B37jRK8m|A{cY1>aFbz;^Q2dMY{UGgB?s}$Vk(fj`5 zL2&ck$}Jh~Hv>R38}o%gpm#jaZNl*qjvb-%SW|?~>c?FC9klY=55ohaehuDbvOb~* z@W2&91v7tuAQFq%o!hs(#Az{)dK^&s1=h3O?iwoc-0|8*r6$-K2`s!v_4lEQB*^>} zVeb&1jR5pqxJCLy+cif+f+R?2SEI*2hijvqbM_y*h{5pf|wp5%4Uxo$2v~gsq)sH3y_dR#fDw*2w4Fs zB!jg`(J&tE0n58YrG0LQ+>=q;KfoEpe=p|rqRvQcGlEb{riAd5GRH8S4^UqvA6{k`@(Oy^CD~EwYZV!)kLI0t0fMDw;b8u_f=dONq;z4O>;OC`r)hYKJEHgWK=M+E>SQ=)AGSoGT z(rs>9tXcI*=#2!}J-?7d)s6CesvKHD=tJ9yj8E1y%|+N(Eo@^z+XL#(z{y%QCf*;+ z)dl}O?}?~Fw0U?I(1(daSb#cKCu_^)^uLM;o<5GLKw_=;8t&)?qPd3fkaAkM z-wBb*=nsE;*cF1ub&@pCz5JY0Ts*djT@EhFk>RjHtV&2Zw!8)A$+N`8b!75J4-Mb1 z{xy<3+Jpa&(v8$Sl$g`$2vTICIg${4;!chC{Fe>5Hs<)Z7lJi0lqgDAX?7s7)WZ)b zzEbx@`AfeGpkWfeG0F^v=$*pB$Wwbb(`{75IY_>}-~B#g`ggCSV)2`EF1^QZgKiGF zTf|nU!PR0d?s?^kjZVhV_p^FdTWCyaT5N*0Yx=6zi-CI&;nOsu_n2CL6feViFHw&< z|3&>O7u~aOwxL7|_^!%s2sQEei~o}LWZ-pL^wrQdbnAYz`9j#0`(;*SbUpzY+K5WP~Y2n{p_99I$I&oXq{ z&Inni%C*%e9ew!XD>?};RBpT1qQO^*2Gr?!5IPj^&r>pSWfeSEer7Bf8}(I7$tt=G zxvzBPu8#*g7L%Xk63Z2a9j;WjkX6j!ws^Jp`8Ww#gwd*UkWg|`AZzRVhwAVtmGxUK zwM!Pz1G?oeAm&++`vf7Kjzv6<-9Z$IjXqKgfyE9J+xI(;#choCEl4`3o;uwBONqn$ zS-u_v_^`>Rj`?(36Al`O=Hr9Wx*ALv+M|yI94P;<0J@Pt+-W?NSHS^U9acG-5xc(E zcUrI@p&*$9Db-(fBmdh+3nY8rd8wkq5w2=2(E(s9Y$b9EfulP_%BtAIxOXTROPPnc zFZ>n}M3aSC<^KG2w5ddkK)h+UGQwQxyx&)qcX)NS`!t__#=Tl>PA;~!6Gr6~rob>u3?@od^Hy`0~8u`Mmw`*0jVfkZfzVrrS zquq3~^Dj;lhpr@4&Nxhj4gBm*IzxVh+zY7&brCLHMokygA`5uQ+}F($5cd_`UTIM$ z0h;b!#;ssR0cFB~kClK{z|ba@r*5Vr8^g+?x}1th)6V=M;D4-Ylh4bo6?o=MR|hfm zvmN%XQjIJMFD#UShQZ^~r{l>clF;e1DnDU21b%z0a_|cLwv&zgCrudU=^TU_p%9g` z&slsn{z>>!tm6UJb<2*ag{aS5OS4bi)D2J0H3@Fv%iC2z8rk%nQT3&+eE6u5M2~0R zWhkgkpTp9UEEEK3B0^ipT$yRczRPsScX$riuW#8OM8|LlE*|AODrd|8r)azh4XjI- zq$RmJgH${ZAN>PhNLDC1EX&CBAyn_Q9O!x~#CM8z$1;fjQb9-WE>p`a^UShpQ))e{ z(Q)-(m)(`UyXxP8^c*v^?Msj+aTcGOuf5uv^#5O-bFSgX@>sV|d`Kw5*R&X-TcJ(& zr1EN-liYapmyIE2*_8p5$SKPw^dEJtoeKKsy)qRmNz=w7EG{SYnXrHCPtLL#PzGC0 zc@k!Uk=hh>T;wXE0H5gmz5JlC<2( zsxQJ|UCkMH`m5=D-K$LQ6T%j&%!^;cLeG1%CxK?}hmjcCylTXfz++@U+16@qA{d@I z`U>&d#!16Q1oIxbN&db~u-uf;lM8R&l^`_g^t$d#a?W=bxdFcI(5@!?m8PxKp+)Jj zHvlcdC3@^`9S{5{l~(G91|NtWM&)Hyb!Q{7!#PQV_t2)Chz-#1-2-Ih%gSHIYWDp(cf zc&y;`!Du;(NFfnPMdF|!hdGdO_MlGVL{Pn)Y4AsoeH1=TLV%Vpi&MVkX{2d)1ODw_ zTl`)<`;GpBYv9S?>r#r($8LFP2849C9T-y!Q_@#wp(@WPS{8{@!oun-c}dzL$p}1O zS3xby&|MoX400RkR3R${qV^z7&kW0w@&=8PTEHWy#m4E(~WtZP{8;3A{qcv zOrpnc_zlQvAj=K@*kV&OAcXhTYtL%(=}#oZe=I$_yjbKYLcyoyW*35;)$j#)B@ciL zNa}Yus0~`nno`!nKnBXxn>Qv?FgqO6kns$bU%|Q?#$s_`s6M9f;B+BaALeD=m67K>< zk##TXCs`m7eufdN$9R1 zWvI5q%ii4j2S(BgzvYjg^?>ax0z>%)U@3ekLNb5(K}pQ3ETP@-3D>qY_hB7%GGbk- zWFUz@#i1e1!i=TQf4hQUG#~Oc2dWdg;=6pSZ`i3Evx7b1eibwX41}6@(=icIJ<+s9 z%onnJ?_7athPi0On4XOY>Of?~Yva%rzdi6reYbRVV`JngANAYf_<`NS&9{b5lIYoj z-XQ4*7fA2nv+a-!dthV2k0KTHtwZiM5as&SM>y1 zqEg+X#Wp3$Is{lL_G%O&Lla-FOkj)akI1_>;%UT2$ghwsHC5T_dnp4Curx|l#ad8) z4V|6NJwMe<0=lk5y#j5f6l~R1HDe)wsS)&LFi&aS{tbt5{xy%X6yL3nyNj|Pmf;A% z8Xq*;9(drHF0y+RR&HL3yruFeP*QA;DF=#4`0&*INC|6?LPN}DyR(v<2xcGGaVr0+ z=RHs{s1$Yw2v;?&7TM5k&5L_lReP$~c!(N1qCer0OBy&n=dxV_P)x(s_sH6}Tqy78 zmeg?I&3wV6@ipAvw=Xln9*c|i_hh6VP&eqo;K;oP-CYO0OB=)aTSZ`-X5Y7_{5Sd9u3o+CO(~l?~*eZ^nUp)ZSrSH*-r*k8ym5(sM2p4 z1NL0K2uJGJKQ^Lf%j6PvL!626WO7*k2aDk==bJJOZkEppsqiW#>3Kv~(2qv5+S)qC zezjoSX3{Jh_4FVmTg z+cv$qBn9gXj4cyT9F@Ff@74D%^wL*D`N&O(Nb2vbc?|zf@u+`KA0PYm6n%5gTn~5- z#8tkW*FcncNMHBo+(@PXWe@1nWRPk*bftEf+A&ksLqO82xrXooqlCWTuQgDx0rn}MTd6v@Ll?(`d+M#?>u(MWex^@b07QUekCYfbOgG0FDMva{`CCXf2W ztd#sFI%mH_M#U&avaSwGV+{jU?}k;f>Xz!y%n})u_Md_>_}!9sLj_WE%G>l*l^C8GK62uVwEDI{)HD#p@?yhRMJpy|feIp-5?)CKo$v^SGt_APNCwvO@ zCeYc4q?hf)$qsqvGWOz>pD63m@at2qyX41A?$Me>N6EU&&^qK(ONRBl3!20f0Rz}~ zz^Tx;kcu>v(e$R2^mR%chbZ0+peYV30qMw8A$Y!b%xksiR8LjFK&?lY74u@3+!^m@ zbqK2p+OBw|Lh3=i&%E>bR2w=plY_fdsc7SW+7#cjDyV{jBQnnOSZs(y)$yvthwwTcsJG6D4wTAS)r>R%bfq=aDcxb(8G2 zR-Dsd`~k$arMr&rkkZjpF)O#M=g+mUGuzkx9?e1T zHYka{q5RF58%w8Q;Q6SnO}xpEtlAnB4Y0X=iR@3-=RPcHHiBv|p6D8Yo98Wn?bLO! zzV|w=euu*4D|XD+ebplSZJ}beu!cl+?ZuzdsTB*oUv%({hGtooKrR;yky>;YuakaI z2SJ|V%u-EAN-&x&#qu&#XkIsCdl)DW&J|l7@4>nLZF%nOh>P{6NP`tAT2G^!5XHlU zemrr^F)o|ZTupq=h~RpBBVGHDqb@pH7qr~R0^jX_jixLU2A)=nYm_GoQ2%$pUNg`6 z882PkYlt`~j|c)O?W|ALjpq>a#s())6mXG^%GgcGt}?CO;{Q`zm+m=0%x1)C`dPqm zf>9e`&A6v{#UdSi>&wl?oPBM_$(NewQz7v8QW3gf*h*}{Z=2b5)cF4FFppLKtF%-W z;!j=3Kx4{ln|dH_(tTZGKj2JrKv8{fn>}_LmV9r+-{dTZ21IKij5M17b~M&eaktmj^lxD`4-yh7)I0FFY7zEXg?F(5v%#_2dmJCc4M%U|(Hpzj z-+G}Y@hllnhX#LACiYSv;pa-k8{l)k)uzq%?o;0EGi|aLt-{)3GdZ4Pm${ET9QJf< z^Gf`+5Ew;D>zq@*Mm@(_lBOfW-4*6;D@WkJL{({$X1C4b;mF7&}{x%LIj=yNM02Wmb0!rW@;^07i$I0sS{Y zrh)Kmg^aI&q`3VK+cb+5QV+Av8wq)&xXJcdl0)rt?QEOb25WDE!6qbW0H$0?)ffq# z*3^de=v*Id)O8wIGy2y9iL63IY-9R-J6Qwz<;|z2Drd_LfaxEC4B3B{GI219K%x2! zqASW7EWOHz{>lM-B)nFFS6GhjnC9QGf^DRJH3)VLLX@wmbn29eRZ(wrGyiVC3LJOW zirl&SgOx~e=YoQWhyE$d`NcC4<2b3bYUrX~zG6=u_18jwD1)~xP89z5tPMB7v}H4v zieF2iSKFL}cu7yjeGG%&SZ~S0}6VKZdW`0zHMRB+~tYknM_L?#MY(szJXpLK+#lG?o&1OReLfsmQ;|1zhBV^OE8FzE>EE)ReQ@f zHXRHH{D7N+S{~+MeBS+E6iYCOu$P|SqF4yE;)EHdeSldY7U=Kdf_Tqt_;W|L!e;7) zsYfBnRa4*pACI~4zM?ty$Pf{P?m;TvkKqlG2n!I!3Tu*Xc3OjBjhae9HY=sgG;5&5 zgY~+loPjx2zw>lN2LbulTDhC>HzsRKFw;Ahr;pUg@(O$k4Cpt+fgfu|)`OU&H!Vc( z&6F?HS>q-Aoa5)cs^M&Q)7QH+yBnNAi{OPKKt0P{U22Q>i<^Lpk?{O-ymFB#^^&_a zCtCx)MFL0GP5=pQc1!)Pr+PtIa|+pdiokZ80kd6$FRqMeF`>MlFL~6&E@Vsv(~?{f z!Fu0Xmb#|Hq$%05&%Dw|Yt{h!5G}((kPvll`i2c1T+3B+t36SuV6Kg@D{KFJy!O`E zI!BS5bBs_rHZZ7_;)NziuuYq67YXvu?#$QxQkk7f>Vb#qz6U;p(x)TIEPhs=$>^n- z(Fh+3@8(ynD){Ijhl=$|>)*CpOPQ|gOec$w$Z0MTl}4YteNFL}W=nV* zYNGwfa9$aUV$sj5V^aOg3~jJKmsy)i8S?Gp%)8ZL(4fhCfAFFyh{i#|TYy;jwXQ*k z&-1U*PPNt5YC-7;%vvuec>j&_9s)FxT!^|4jiX#AmRAc~Wn9$XYJL$i;;H%#)b!&& zL)!T=tm25lY>tG`jN{sL(_Q!8?==Q7i_f&cvp?A`dmOPtz!bE&cao{PxjH|$-t_$l9Y=bB#_4yV z+nK@0q)$2B9X=ZpwY-Sdh0HVdpM&JhO40Q=0B@9J@?yb!hAc&JV|D^pr(IOD!!^h+VPq9?x|M z#a3DOsy9HxB7eH8w3u4PlPDM-ovT=-8~ve(@Accki~U>?NA8ZZ*}&;ohO)RIaX4t8 ziMujB7Uq)ZhN9Led!$1xv@KSFV;9=HA6!k^ukOg!X6EcxVOR6AL-9_7DcDEUv!@=*J_j>pRA#M=WviFvMQY z8Qcj(H+%e~Ke*DFq&tlvPT@VG{&tp8q+R*TpGT=F%;7j)rr;h+sO4k7ojMB{#HHsn zJ@UaLT}J&I+S2gwJGT1d3o0=KvAng*J`=A&utu<*Db+O^qdPE5L^<98x%jWS9Q4X& z0h$(iTN!hb9T$G#r3S;bdF#9ca8aAF&Xn0BQU`B)y*wM0ldGB z`xwfE(PU8@`)pD{Xby5HA28N$kragJ{^oyXMce%}o1oMmSrjv>#<@1xjL8 z^j7`_wDXi!#C+t!sVELOoo%t9Mc55P}Alm;ZiHGJ0EO^%>Q{=YWgjgbWd7 zC4i$XD1G3mU1T<(C;mD`f@S05zuN8U%7OH&51yX$78J*rW8j!VX1g2tZZQ_aBK<+; zVAZa#kC0dm5BhufM22dl;s?*lZ<`;zogUQOG(|t)v`A$R9h7XF=venh^DKir%?^x& zh!g|p;493sRHM(X&J}VHVpE>F7w=$t>WqOa+9Nx2uFkm&8s*#sd;;Rb?DkfQ9ynk@ z(Q}ZtKcpvvhw!*1ghWoBHHhqo&OPDYyN(X~^;d6jCHB`k#u(~$lolbI!F8QQ(_V73 zDl30)dsWwkzn49(@)>@jM@Y6a)4`~>qHks_-`BiE!E4uWTK&T+m>(&(HwD|YAFpD<$e zzk8PT$Xwy8LRx=xXGQ3z(d2}hh6A8Us)F<9i)kwBN3s~*R$>^w&w zL(f|9q+Ch-DFLf{t=BczmTZp#^o_#_9d_h(`JGQh&mlQNh!#)VrzLimb1?f8DL?P}61&;m1?Y2N1ue=V5sR-SW6y5;`3M z6R!*i?61&;QR;CSWyGYS#UZ;Q7Ih}!l^UzbRnYQ8m}6!)d4nUcGEw&U;r8{j_v>KW zFc!3P7(?IAKp+V%Sej5Su38-2ujrx70nc3rz{4D|+@DLyliSkWIU1HvLf|k5lv@@gb}m_RPS4&_qcF{ z-;US&0B!?rcj= zRX^TV0Rt?VC*x|x=D-x>U|+PK&*spm80j^hO|k(PGM)V>pa zz#Ak?$;>lu2`P&gEysa$QaT+<=~}6JKU+1^_I`iJ`+hQaeU!GXdCQY4;#NJDJ2ME1 zVvW2}*0hehCUbA!v_JgsajB(5p3%|4couU>kuNd-9`X4rK4?Eo5WbNw+)n zioD~QyZBgOrVOE8VL*d3B{LvXcaaRXX%;=eqTraPjpC2Ty`&AI(-8jEf6?!dXq^y$ zR4~G1IFCIS-Sd{s<`QJD84I4@TGdN0Zi@-1*K*7}Y4#x&FVXidA#|1t3yP46zkO!N zxs5od;&qzk<}#9Cl?Y!w6~G73Ym)WDDLHakaGrDMu9oB^VZEbj(7zv=t(9+fU@!O_ zeSX@2v8kf+Mubr`l&Egs05xvTpcgAQLMo*5@Qk*Ir*az&HF&;5DPH#}AA$(dOKttv zxm_CPVH}ybAkIr{pO5-0=yPDK@O$P%IA9ntalvxZLU0C>$d9;lQD>?ytZ$)8pW@$U z%OcoZnjt$iobyvCt)n%|*9yE@= 12 chars, upper, lower, digit, symbol). +FIXTURE_PASSWORD = 'Import1Model!Fixture' + +# Deterministic filler so a regeneration differs only in the salt and the export timestamp. +FILLER_SEED = 20260825 + + +def _load_producer_module(repo: Path, name: str) -> types.ModuleType: + """Import one producer module by path, without importing the Streamlit app.""" + path = repo / 'app' / 'src' / 'operations' / 'model_export' / f'{name}.py' + if not path.is_file(): + raise SystemExit(f'producer module not found: {path}') + spec = importlib.util.spec_from_file_location(f'producer_{name}', path) + if spec is None or spec.loader is None: + raise SystemExit(f'cannot load producer module: {path}') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _producer_commit(repo: Path) -> str: + result = subprocess.run( # noqa: S603 - fixed argv, no shell + ['git', '-C', str(repo), 'rev-parse', 'HEAD'], # noqa: S607 - git from PATH is intended + capture_output=True, + text=True, + check=False, + ) + return result.stdout.strip() or 'unknown' + + +def _write_model_dir(root: Path, flavor: str, pickle_bytes: bytes) -> None: + """Write the files an MLflow 2.x model directory carries for this platform.""" + root.mkdir(parents=True, exist_ok=True) + (root / 'MLmodel').write_text( + f'artifact_path: {flavor}\n' + 'flavors:\n' + ' python_function:\n' + ' loader_module: mlflow.sklearn\n' + ' model_path: model.pkl\n' + ' sklearn:\n' + ' pickled_model: model.pkl\n' + ' serialization_format: cloudpickle\n' + ' sklearn_version: 1.4.2\n' + 'mlflow_version: 2.10.1\n' + 'model_uuid: 0f9c1c1f9c1f4c0f9c1c1f9c1f4c0f9c\n' + 'utc_time_created: "2026-08-25 12:00:00.000000"\n', + encoding='utf-8', + ) + (root / 'model.pkl').write_bytes(pickle_bytes) + (root / 'conda.yaml').write_text( + 'channels:\n- conda-forge\ndependencies:\n- python=3.11\n- pip\n- pip:\n' + ' - mlflow==2.10.1\n - scikit-learn==1.4.2\nname: mlflow-env\n', + encoding='utf-8', + ) + (root / 'python_env.yaml').write_text( + 'python: 3.11.15\nbuild_dependencies:\n- pip==24.0\ndependencies:\n- -r requirements.txt\n', + encoding='utf-8', + ) + (root / 'requirements.txt').write_text( + 'mlflow==2.10.1\nscikit-learn==1.4.2\ncloudpickle==3.0.0\n', encoding='utf-8' + ) + + +def _write_csv(path: Path, rows: int, rng: random.Random) -> None: + lines = ['datetime,feature_a,feature_b,SE'] + for i in range(rows): + lines.append( + f'2026-08-{(i % 27) + 1:02d}T00:00:00,' + f'{rng.uniform(0, 100):.6f},{rng.uniform(0, 100):.6f},{rng.uniform(0, 100):.6f}' + ) + path.write_text('\n'.join(lines) + '\n', encoding='utf-8') + + +def _build_artifact_tree(root: Path) -> None: + rng = random.Random(FILLER_SEED) # noqa: S311 - test filler, not key material + # Incompressible filler keeps the zip above the 64 KiB chunk size, so the golden fixture + # exercises multi-chunk `MESSAGE`/`FINAL` framing with real producer bytes. + big_pickle = bytes(rng.getrandbits(8) for _ in range(90 * 1024)) + small_pickle = bytes(rng.getrandbits(8) for _ in range(8 * 1024)) + _write_model_dir(root / 'prediction_model', 'prediction_model', big_pickle) + _write_model_dir(root / 'data_model', 'data_model', small_pickle) + (root / 'model_card').mkdir(parents=True, exist_ok=True) + (root / 'model_card' / 'model_card.json').write_text( + json.dumps( + { + 'model_name': 'fixture_model', + 'target': 'SE', + 'created_by': 'fixture', + 'creation_method': 'time_series_template', + } + ), + encoding='utf-8', + ) + _write_csv(root / 'train_data.csv', 120, rng) + _write_csv(root / 'test_data.csv', 40, rng) + _write_csv(root / 'raw_data.csv', 200, rng) + + +RUN_PARAMETERS = { + 'model_name': 'fixture_model', + 'target_variable': 'SE', + 'input_variables': "['feature_a', 'feature_b']", + 'models_params': "{'alpha': 0.1}", + 'nan_treatment': 'drop', + 'scaler_name': 'StandardScaler', + 'train_size': '0.8', + 'test_size': '0.2', + 'include_ar': 'False', + 'retrain': 'False', + 'start_date': '2026-01-01 00:00:00', + 'end_date': '2026-08-01 00:00:00', +} + +METADATA_FIELDS = { + 'model_name': 'fixture_model', + 'experiment_name': 'fixture_experiment', + 'run_id': '8f14e45fceea167a5a36dedd4bea2543', + 'model_version': '3', + 'model_project': 'fixture_project', +} + + +def main() -> int: + repo = Path(os.getenv('PRODUCER_REPO', str(DEFAULT_PRODUCER_REPO))).resolve() + bundle = _load_producer_module(repo, 'bundle') + crypto = _load_producer_module(repo, 'crypto') + + crypto.validate_password_strength(FIXTURE_PASSWORD) + + staging = Path(tempfile.mkdtemp(prefix='sientia_fixture_')) + try: + _build_artifact_tree(staging) + metadata = bundle.build_metadata_json(run_parameters=RUN_PARAMETERS, **METADATA_FIELDS) + zip_bytes = bundle.build_zip(staging, metadata) + sientia_bytes = crypto.encrypt_archive(zip_bytes, FIXTURE_PASSWORD) + finally: + shutil.rmtree(staging, ignore_errors=True) + + target = FIXTURE_DIR / 'golden_model_v3.sientia' + target.write_bytes(sientia_bytes) + (FIXTURE_DIR / 'golden_model_v3.metadata.json').write_text( + json.dumps(metadata, indent=2) + '\n', encoding='utf-8' + ) + print(f'wrote {target} ({len(sientia_bytes)} bytes, zip {len(zip_bytes)} bytes)') + print(f'producer commit: {_producer_commit(repo)}') + print(f'password: {FIXTURE_PASSWORD}') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/helpers/bundle_factory.py b/tests/helpers/bundle_factory.py new file mode 100644 index 0000000..db54be5 --- /dev/null +++ b/tests/helpers/bundle_factory.py @@ -0,0 +1,272 @@ +"""Test-only `.sientia` writer, mirrored field for field from the producer. + +There is no production writer in this repository (change `import-model-workflow`, design D1): the +producer is the Streamlit app in `sientia-projects-templates`. This factory exists for one reason — +building the **hostile** variants the gate tests need, which the producer cannot emit. Its output is +pinned against the golden fixture by `test_format.py::test_factory_header_matches_fixture_layout`, +so a drift in either direction shows up as a failing test. + +Mirrored from: +- `app/src/operations/model_export/crypto.py` — header layout, KDF, `secretstream` framing +- `app/src/operations/model_export/bundle.py` — zip layout and the two-top-level-key JSON + +Deliberate deviation: the default KDF cost is a fraction of the producer's (8 MiB / 1 pass instead +of 256 MiB / 3), because the reader honours whatever the header declares and a test suite that +derives thirty 256 MiB Argon2id keys is a test suite nobody runs. The golden fixture is what covers +the production cost parameters. +""" + +from __future__ import annotations + +import base64 +import hashlib +import io +import json +import os +import struct +import unicodedata +import zipfile +from dataclasses import dataclass, field +from typing import Any + +import nacl.bindings as sodium +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +# --- mirrored from the producer's crypto.py --- +MAGIC = b'SIENTIA1' +FORMAT_VERSION = 1 +AEAD_ID = 1 +KDF_ID = 1 +SALT_SIZE = 16 +RESERVED_SIGNATURE_SIZE = 64 +CHUNK_SIZE = 64 * 1024 +HEADER_STRUCT = struct.Struct('>8sBBB16sQQB32s64s') +HEADER_SIZE = HEADER_STRUCT.size +PRODUCER_KDF_MEMLIMIT_BYTES = 256 * 1024 * 1024 +PRODUCER_KDF_OPSLIMIT = 3 + +# Cheap-but-valid cost parameters for the suite; see the module docstring. +TEST_KDF_MEMLIMIT_BYTES = 8 * 1024 * 1024 +TEST_KDF_OPSLIMIT = 1 + +DEFAULT_PASSWORD = 'Factory1Model!Test' +DEFAULT_ENVELOPE_KEY = bytes(range(32)) + +MODEL_DIRS = ('prediction_model', 'data_model') +MODEL_FILES = ('MLmodel', 'model.pkl', 'conda.yaml', 'python_env.yaml', 'requirements.txt') +CSV_FILES = ('train_data.csv', 'test_data.csv', 'raw_data.csv') + +DEFAULT_METADATA = { + 'model_name': 'factory_model', + 'experiment_name': 'factory_experiment', + 'run_id': 'aa11bb22cc33dd44ee55ff6677889900', + 'model_version': '2', + 'model_project': 'factory_project', + 'export_timestamp': '2026-08-25T12:00:00+00:00', +} + +DEFAULT_PARAMETERS = { + 'model_name': 'factory_model', + 'target_variable': 'SE', + 'input_variables': "['feature_a', 'feature_b']", + 'retrain': 'False', +} + + +def default_tree() -> dict[str, bytes]: + """The artifact tree a well-formed bundle carries, as archive-name -> bytes.""" + files: dict[str, bytes] = {} + for directory in MODEL_DIRS: + for name in MODEL_FILES: + files[f'artifacts/{directory}/{name}'] = f'{directory}/{name}\n'.encode() + files['artifacts/model_card/model_card.json'] = json.dumps({'target': 'SE'}).encode() + for name in CSV_FILES: + files[f'artifacts/{name}'] = b'datetime,feature_a,SE\n2026-08-25T00:00:00,1.0,2.0\n' + return files + + +def build_metadata_document( + *, + metadata: dict[str, Any] | None = None, + parameters: dict[str, Any] | None = None, + drop_metadata_field: str | None = None, + drop_top_level_key: str | None = None, + drop_parameter: str | None = None, +) -> dict[str, Any]: + """Build `metadata.json`, with hooks for the malformed variants gate 6 must reject.""" + meta = {**DEFAULT_METADATA, **(metadata or {})} + params = {**DEFAULT_PARAMETERS, **(parameters or {})} + if drop_metadata_field: + meta.pop(drop_metadata_field, None) + if drop_parameter: + params.pop(drop_parameter, None) + document: dict[str, Any] = {'parameters': params, 'metadata': meta} + if drop_top_level_key: + document.pop(drop_top_level_key, None) + return document + + +@dataclass +class ZipSpec: + """How to lay out the zip inside the bundle.""" + + files: dict[str, bytes] = field(default_factory=default_tree) + document: dict[str, Any] | None = None + include_metadata: bool = True + symlinks: dict[str, str] = field(default_factory=dict) + absolute_entry: str | None = None + traversal_entry: str | None = None + bomb_bytes: int = 0 + extra_files: dict[str, bytes] = field(default_factory=dict) + + +def build_zip(spec: ZipSpec | None = None) -> bytes: + """Zip `artifacts/` plus `metadata.json`, mirroring the producer's `build_zip`. + + `metadata.json` is written first with `json.dumps` and no indent, entries use `/` separators, + and no directory entries are stored — all three matter to the reader. + """ + spec = spec or ZipSpec() + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED) as archive: + if spec.include_metadata: + document = spec.document if spec.document is not None else build_metadata_document() + archive.writestr('metadata.json', json.dumps(document)) + for name, content in {**spec.files, **spec.extra_files}.items(): + archive.writestr(_regular_file_info(name), content) + for name, target in spec.symlinks.items(): + info = zipfile.ZipInfo(name) + info.create_system = 3 + info.external_attr = (0o120777 << 16) | 0o200000 + archive.writestr(info, target) + if spec.absolute_entry: + archive.writestr(_regular_file_info(spec.absolute_entry), b'absolute\n') + if spec.traversal_entry: + archive.writestr(_regular_file_info(spec.traversal_entry), b'traversal\n') + if spec.bomb_bytes: + archive.writestr(_regular_file_info('artifacts/bomb.csv'), b'\0' * spec.bomb_bytes) + return buffer.getvalue() + + +def _regular_file_info(name: str) -> zipfile.ZipInfo: + """A zip entry carrying a Unix regular-file mode, as `ZipFile.write` would store it.""" + info = zipfile.ZipInfo(name) + info.create_system = 3 + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o100644 << 16 + return info + + +def derive_key(password: str, salt: bytes, opslimit: int, memlimit: int) -> bytes: + """Argon2id over the NFC-normalised password — the producer's `_derive_key`.""" + normalized = unicodedata.normalize('NFC', password) + return sodium.crypto_pwhash_alg( + sodium.crypto_secretstream_xchacha20poly1305_KEYBYTES, + normalized.encode('utf-8'), + salt, + opslimit, + memlimit, + sodium.crypto_pwhash_ALG_ARGON2ID13, + ) + + +def encrypt_archive( # noqa: PLR0913 - every knob here exists to build one hostile variant + plaintext: bytes, + password: str = DEFAULT_PASSWORD, + *, + format_version: int = FORMAT_VERSION, + aead_id: int = AEAD_ID, + kdf_id: int = KDF_ID, + memlimit: int = TEST_KDF_MEMLIMIT_BYTES, + opslimit: int = TEST_KDF_OPSLIMIT, + declared_memlimit: int | None = None, + declared_opslimit: int | None = None, + parallelism: int = 1, + reserved_signature: bytes | None = None, + magic: bytes = MAGIC, + digest_override: bytes | None = None, + salt: bytes | None = None, + drop_final_tag: bool = False, +) -> bytes: + """Produce a full `.sientia` byte stream, mirroring the producer's `encrypt_archive`. + + The `declared_*` arguments write cost parameters into the header that differ from the ones + actually used, which is how a "512 MiB header" is built without allocating 512 MiB. + `drop_final_tag` pushes every chunk as `MESSAGE`, so the stream is framed as if more were + coming — the truncation the digest cannot see. + """ + salt = salt if salt is not None else os.urandom(SALT_SIZE) + key = derive_key(password, salt, opslimit, memlimit) + + state = sodium.crypto_secretstream_xchacha20poly1305_state() + stream_header = sodium.crypto_secretstream_xchacha20poly1305_init_push(state, key) + + chunks = [] + offset = 0 + total = len(plaintext) + while offset < total: + chunk = plaintext[offset : offset + CHUNK_SIZE] + offset += len(chunk) + is_last = offset >= total and not drop_final_tag + tag = ( + sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL + if is_last + else sodium.crypto_secretstream_xchacha20poly1305_TAG_MESSAGE + ) + chunks.append(sodium.crypto_secretstream_xchacha20poly1305_push(state, chunk, None, tag)) + ciphertext = b''.join(chunks) + + digest = digest_override if digest_override is not None else hashlib.sha256(ciphertext).digest() + header = HEADER_STRUCT.pack( + magic, + format_version, + aead_id, + kdf_id, + salt, + declared_memlimit if declared_memlimit is not None else memlimit, + declared_opslimit if declared_opslimit is not None else opslimit, + parallelism, + digest, + reserved_signature if reserved_signature is not None else bytes(RESERVED_SIGNATURE_SIZE), + ) + return header + stream_header + ciphertext + + +def make_bundle( + zip_spec: ZipSpec | None = None, + password: str = DEFAULT_PASSWORD, + **header_kwargs: Any, +) -> bytes: + """The common case: a well-formed tree, encrypted with the test cost parameters.""" + return encrypt_archive(build_zip(zip_spec), password, **header_kwargs) + + +def tamper_ciphertext(bundle: bytes, offset_from_stream: int = 8) -> bytes: + """Flip one byte inside the chunk stream **and** repair the header digest. + + Repairing the digest is what makes this a gate-3 test rather than a gate-2 one: the file is + self-consistent and only the AEAD can tell it was altered. + """ + position = HEADER_SIZE + 24 + offset_from_stream + altered = bytearray(bundle) + altered[position] ^= 0xFF + repaired = hashlib.sha256(bytes(altered[HEADER_SIZE + 24 :])).digest() + altered[44:76] = repaired + return bytes(altered) + + +def truncate_bundle(bundle: bytes, drop_bytes: int = 32) -> bytes: + """Cut the tail off a bundle, leaving the header's digest describing bytes that are gone.""" + return bundle[:-drop_bytes] + + +def build_envelope(password: str = DEFAULT_PASSWORD, key: bytes = DEFAULT_ENVELOPE_KEY) -> str: + """`base64(nonce || ciphertext || tag)` under AES-256-GCM, as the frontend will send it.""" + nonce = os.urandom(12) + sealed = AESGCM(key).encrypt(nonce, password.encode('utf-8'), None) + return base64.b64encode(nonce + sealed).decode('ascii') + + +def envelope_key_b64(key: bytes = DEFAULT_ENVELOPE_KEY) -> str: + """The key as `IMPORT_PASSWORD_KEY` carries it.""" + return base64.b64encode(key).decode('ascii') diff --git a/tests/laborious/utils/bundle/__init__.py b/tests/laborious/utils/bundle/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/laborious/utils/bundle/conftest.py b/tests/laborious/utils/bundle/conftest.py new file mode 100644 index 0000000..ffc41f6 --- /dev/null +++ b/tests/laborious/utils/bundle/conftest.py @@ -0,0 +1,34 @@ +"""Shared fixtures for the `.sientia` bundle tests.""" + +from pathlib import Path + +import pytest + +FIXTURE_DIR = Path(__file__).resolve().parents[3] / 'fixtures' / 'bundle' +GOLDEN_BUNDLE = FIXTURE_DIR / 'golden_model_v3.sientia' +GOLDEN_METADATA = FIXTURE_DIR / 'golden_model_v3.metadata.json' +# Recorded in tests/fixtures/bundle/README.md; the fixture protects nothing. +GOLDEN_PASSWORD = 'Import1Model!Fixture' + + +@pytest.fixture +def golden_bundle() -> Path: + """The `.sientia` file the real producer emitted.""" + assert GOLDEN_BUNDLE.is_file(), ( + f'golden fixture missing: {GOLDEN_BUNDLE} — regenerate it with ' + 'tests/fixtures/bundle/regenerate.py' + ) + return GOLDEN_BUNDLE + + +@pytest.fixture +def golden_password() -> str: + return GOLDEN_PASSWORD + + +@pytest.fixture +def work_dir(tmp_path: Path) -> Path: + """A per-test stand-in for `IMPORT_WORK_DIR/`.""" + root = tmp_path / 'workflow-id' + root.mkdir() + return root diff --git a/tests/laborious/utils/bundle/test_format.py b/tests/laborious/utils/bundle/test_format.py new file mode 100644 index 0000000..e8c0f6b --- /dev/null +++ b/tests/laborious/utils/bundle/test_format.py @@ -0,0 +1,227 @@ +"""The wire format: header parsing, ceilings, digest coverage and the fixture contract.""" + +import ast +import hashlib +from pathlib import Path + +import nacl.bindings as sodium +import pytest + +from laborious.utils.bundle import format as fmt +from tests.helpers import bundle_factory as factory + + +def _write(tmp_path: Path, payload: bytes, name: str = 'bundle.sientia') -> Path: + target = tmp_path / name + target.write_bytes(payload) + return target + + +# --------------------------------------------------------------- constants + + +def test_header_size_is_140(): + assert fmt.HEADER_SIZE == 140 + assert fmt.HEADER_STRUCT.format == '>8sBBB16sQQB32s64s' + + +def test_libsodium_literals_match_the_binding(): + """The literals in `format.py` keep the module import-free; these assertions keep them true.""" + assert fmt.STREAM_HEADER_SIZE == sodium.crypto_secretstream_xchacha20poly1305_HEADERBYTES + assert fmt.ABYTES == sodium.crypto_secretstream_xchacha20poly1305_ABYTES + assert fmt.KEY_SIZE == sodium.crypto_secretstream_xchacha20poly1305_KEYBYTES + + +def test_producer_constants_match(): + assert fmt.MAGIC == b'SIENTIA1' + assert fmt.CHUNK_SIZE == 65536 + assert fmt.MAX_KDF_MEMLIMIT == 268435456 + assert fmt.MAX_KDF_OPSLIMIT == 3 + assert fmt.SALT_SIZE == 16 + assert fmt.DIGEST_SIZE == 32 + assert fmt.RESERVED_SIGNATURE_SIZE == 64 + + +def test_prefix_size_is_65717(): + """The pre-upload prefix is computed from constants, never read from a file.""" + assert fmt.PREFIX_SIZE == 140 + 24 + (65536 + 17) + assert fmt.PREFIX_SIZE == 65717 + + +def test_digest_offset_skips_both_headers(): + assert fmt.DIGEST_OFFSET == 164 + + +# ------------------------------------------------------------------- parse + + +def test_parses_a_producer_emitted_header(golden_bundle): + header = fmt.read_header(golden_bundle) + + assert header.format_version == 1 + assert header.aead_id == fmt.AEAD_ID_XCHACHA20POLY1305_SECRETSTREAM + assert header.kdf_id == fmt.KDF_ID_ARGON2ID + assert len(header.salt) == 16 + assert header.kdf_memlimit_bytes == 268435456 + assert header.kdf_opslimit == 3 + assert header.kdf_parallelism == 1 + assert len(header.ciphertext_digest) == 32 + assert header.reserved_signature == bytes(64) + + +def test_wrong_magic_rejected(tmp_path): + path = _write(tmp_path, factory.make_bundle(magic=b'NOTSIENT')) + + with pytest.raises(fmt.BundleFormatError, match='expected bundle marker'): + fmt.read_header(path) + + +def test_unknown_format_version_rejected(tmp_path): + path = _write(tmp_path, factory.make_bundle(format_version=2)) + + with pytest.raises(fmt.BundleFormatError, match='format version'): + fmt.read_header(path) + + +def test_unknown_aead_id_rejected(tmp_path): + path = _write(tmp_path, factory.make_bundle(aead_id=7)) + + with pytest.raises(fmt.BundleFormatError, match='encryption identifier'): + fmt.read_header(path) + + +def test_unknown_kdf_id_rejected(tmp_path): + path = _write(tmp_path, factory.make_bundle(kdf_id=9)) + + with pytest.raises(fmt.BundleFormatError, match='key derivation identifier'): + fmt.read_header(path) + + +def test_memlimit_above_the_ceiling_rejected(tmp_path): + path = _write(tmp_path, factory.make_bundle(declared_memlimit=512 * 1024 * 1024)) + + with pytest.raises(fmt.BundleFormatError, match='memory'): + fmt.read_header(path) + + +def test_opslimit_above_the_ceiling_rejected(tmp_path): + path = _write(tmp_path, factory.make_bundle(declared_opslimit=4)) + + with pytest.raises(fmt.BundleFormatError, match='passes'): + fmt.read_header(path) + + +def test_zero_cost_parameters_rejected(tmp_path): + path = _write(tmp_path, factory.make_bundle(declared_opslimit=0)) + + with pytest.raises(fmt.BundleFormatError, match='positive'): + fmt.read_header(path) + + +def test_reserved_parallelism_must_be_one(tmp_path): + path = _write(tmp_path, factory.make_bundle(parallelism=2)) + + with pytest.raises(fmt.BundleFormatError, match='parallelism'): + fmt.read_header(path) + + +def test_non_zero_reserved_signature_rejected(tmp_path): + path = _write(tmp_path, factory.make_bundle(reserved_signature=b'\x01' + bytes(63))) + + with pytest.raises(fmt.BundleFormatError, match='reserved signature'): + fmt.read_header(path) + + +def test_file_shorter_than_header_plus_stream_header_rejected(tmp_path): + path = _write(tmp_path, b'SIENTIA1' + bytes(100)) + + with pytest.raises(fmt.BundleFormatError, match='smaller than'): + fmt.read_header(path) + + +def test_parse_header_needs_the_whole_header(): + with pytest.raises(fmt.BundleFormatError, match='shorter than'): + fmt.parse_header(b'SIENTIA1') + + +# ------------------------------------------------------------------ digest + + +def test_digest_covers_the_chunk_stream_not_the_stream_header(golden_bundle): + """The producer hashes the chunks before prepending either header — offset 164, not 140.""" + header = fmt.read_header(golden_bundle) + payload = golden_bundle.read_bytes() + + assert hashlib.sha256(payload[164:]).digest() == header.ciphertext_digest + assert hashlib.sha256(payload[140:]).digest() != header.ciphertext_digest + assert fmt.compute_ciphertext_digest(golden_bundle) == header.ciphertext_digest + + +def test_verify_ciphertext_digest_accepts_the_fixture(golden_bundle): + fmt.verify_ciphertext_digest(golden_bundle, fmt.read_header(golden_bundle)) + + +def test_digest_mismatch_rejected(tmp_path): + path = _write(tmp_path, factory.make_bundle(digest_override=bytes(32))) + header = fmt.read_header(path) + + with pytest.raises(fmt.BundleFormatError, match='does not match the digest'): + fmt.verify_ciphertext_digest(path, header) + + +def test_truncated_file_fails_the_digest(tmp_path): + path = _write(tmp_path, factory.truncate_bundle(factory.make_bundle())) + header = fmt.read_header(path) + + with pytest.raises(fmt.BundleFormatError, match='does not match the digest'): + fmt.verify_ciphertext_digest(path, header) + + +def test_file_digest_is_the_whole_object(tmp_path): + payload = factory.make_bundle() + path = _write(tmp_path, payload) + + assert fmt.file_digest(path) == hashlib.sha256(payload).hexdigest() + + +# ------------------------------------------------------- factory conformance + + +def test_factory_header_matches_fixture_layout(golden_bundle, tmp_path): + """The test-only writer must be structurally identical to the producer's output.""" + produced = fmt.read_header(_write(tmp_path, factory.make_bundle())) + golden = fmt.read_header(golden_bundle) + + assert (produced.format_version, produced.aead_id, produced.kdf_id) == ( + golden.format_version, + golden.aead_id, + golden.kdf_id, + ) + assert produced.kdf_parallelism == golden.kdf_parallelism + assert produced.reserved_signature == golden.reserved_signature + assert len(produced.salt) == len(golden.salt) + assert len(produced.ciphertext_digest) == len(golden.ciphertext_digest) + assert factory.HEADER_STRUCT.format == fmt.HEADER_STRUCT.format + assert factory.PRODUCER_KDF_MEMLIMIT_BYTES == golden.kdf_memlimit_bytes + assert factory.PRODUCER_KDF_OPSLIMIT == golden.kdf_opslimit + + +# --------------------------------------------------------------- liftability + +FORBIDDEN_IMPORT_ROOTS = frozenset( + {'laborious', 'temporalio', 'mlflow', 'nacl', 'cryptography', 'sqlalchemy', 'pymongo'} +) + + +def test_format_module_imports_nothing_platform_specific(): + """`format.py` stays liftable into a shared package: stdlib only, no platform imports.""" + tree = ast.parse(Path(fmt.__file__).read_text(encoding='utf-8')) + roots = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + roots.update(alias.name.split('.')[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + roots.add(node.module.split('.')[0]) + + assert roots.isdisjoint(FORBIDDEN_IMPORT_ROOTS), roots & FORBIDDEN_IMPORT_ROOTS + assert roots == {'__future__', 'hashlib', 'struct', 'dataclasses', 'pathlib', 'typing'} diff --git a/tests/laborious/utils/bundle/test_reader.py b/tests/laborious/utils/bundle/test_reader.py new file mode 100644 index 0000000..9430173 --- /dev/null +++ b/tests/laborious/utils/bundle/test_reader.py @@ -0,0 +1,460 @@ +"""The seven gates: what each one rejects, in what order, and what it leaves on disk.""" + +import json +import random +from pathlib import Path + +import nacl.bindings as sodium +import pytest + +from laborious.utils.bundle import format as fmt +from laborious.utils.bundle.reader import BundleGateError, BundleLimits, BundleReader +from laborious.utils.bundle.steps import ImportStep +from tests.helpers import bundle_factory as factory + +OBJECT_KEY = 'imported_models/factory_model_v2.sientia' + + +@pytest.fixture +def reader() -> BundleReader: + return BundleReader() + + +def _bundle_file(work_dir: Path, payload: bytes) -> Path: + target = work_dir / 'download.sientia' + target.write_bytes(payload) + return target + + +def _open(reader: BundleReader, work_dir: Path, payload: bytes, password: str | None = None): + return reader.open( + _bundle_file(work_dir, payload), + password if password is not None else factory.DEFAULT_PASSWORD, + work_dir / 'extract', + work_dir, + ) + + +def _extracted_files(root: Path) -> list[str]: + if not root.exists(): + return [] + return sorted(str(p.relative_to(root)) for p in root.rglob('*') if p.is_file()) + + +# ------------------------------------------------------------------- gate 1 + + +def test_gate_1_accepts_a_well_shaped_object(reader): + reader.check_object_limits(size=1024, key=OBJECT_KEY) + + +def test_gate_1_rejects_an_oversized_object(reader): + limited = BundleReader(BundleLimits(max_object_bytes=1000)) + + with pytest.raises(BundleGateError) as error: + limited.check_object_limits(size=1001, key=OBJECT_KEY) + + assert error.value.gate == 1 + assert error.value.step is ImportStep.DOWNLOAD + + +def test_gate_1_rejects_an_empty_object(reader): + with pytest.raises(BundleGateError, match='empty'): + reader.check_object_limits(size=0, key=OBJECT_KEY) + + +def test_gate_1_rejects_a_key_outside_the_import_prefix(reader): + with pytest.raises(BundleGateError, match='import prefix'): + reader.check_object_limits(size=10, key='other_place/model.sientia') + + +def test_gate_1_rejects_a_key_with_the_wrong_suffix(reader): + with pytest.raises(BundleGateError, match='.sientia'): + reader.check_object_limits(size=10, key='imported_models/model.zip') + + +# ------------------------------------------------------------- golden bundle + + +def test_opens_the_producer_fixture(golden_bundle, golden_password, work_dir, reader): + """The producer's own bytes, at the producer's own KDF cost.""" + opened = reader.open(golden_bundle, golden_password, work_dir / 'extract', work_dir) + + assert opened.metadata['model_name'] == 'fixture_model' + assert opened.metadata['experiment_name'] == 'fixture_experiment' + assert set(opened.metadata) == { + 'model_name', + 'experiment_name', + 'run_id', + 'model_version', + 'model_project', + 'export_timestamp', + } + assert opened.parameters['target_variable'] == 'SE' + assert opened.header is not None + assert opened.digest == opened.header.ciphertext_digest.hex() + assert _extracted_files(opened.extracted_dir) == [ + 'artifacts/data_model/MLmodel', + 'artifacts/data_model/conda.yaml', + 'artifacts/data_model/model.pkl', + 'artifacts/data_model/python_env.yaml', + 'artifacts/data_model/requirements.txt', + 'artifacts/model_card/model_card.json', + 'artifacts/prediction_model/MLmodel', + 'artifacts/prediction_model/conda.yaml', + 'artifacts/prediction_model/model.pkl', + 'artifacts/prediction_model/python_env.yaml', + 'artifacts/prediction_model/requirements.txt', + 'artifacts/raw_data.csv', + 'artifacts/test_data.csv', + 'artifacts/train_data.csv', + 'metadata.json', + ] + assert list(work_dir.glob('*.zip')) == [] + + +def test_fixture_metadata_matches_its_committed_plaintext_copy( + golden_bundle, golden_password, work_dir, reader +): + from tests.laborious.utils.bundle.conftest import GOLDEN_METADATA + + opened = reader.open(golden_bundle, golden_password, work_dir / 'extract', work_dir) + committed = json.loads(GOLDEN_METADATA.read_text(encoding='utf-8')) + + assert opened.metadata == committed['metadata'] + assert opened.parameters == committed['parameters'] + + +# ------------------------------------------------------------------- gate 2 + + +def test_gate_2_rejects_a_wrong_magic_before_deriving(work_dir, reader, monkeypatch): + payload = factory.make_bundle(magic=b'NOTSIENT') + monkeypatch.setattr( + sodium, 'crypto_pwhash_alg', _never_called('key derivation must not run at gate 2') + ) + + with pytest.raises(BundleGateError) as error: + _open(reader, work_dir, payload) + + assert error.value.gate == 2 + assert error.value.step is ImportStep.DOWNLOAD + + +def test_gate_2_rejects_kdf_parameters_above_the_ceiling_without_allocating( + work_dir, reader, monkeypatch +): + payload = factory.make_bundle(declared_memlimit=512 * 1024 * 1024) + monkeypatch.setattr( + sodium, 'crypto_pwhash_alg', _never_called('the ceiling must be enforced before deriving') + ) + + with pytest.raises(BundleGateError) as error: + _open(reader, work_dir, payload) + + assert error.value.gate == 2 + + +def test_gate_2_rejects_a_digest_mismatch(work_dir, reader): + with pytest.raises(BundleGateError) as error: + _open(reader, work_dir, factory.make_bundle(digest_override=bytes(32))) + + assert error.value.gate == 2 + + +def test_gate_2_rejects_a_truncated_transfer(work_dir, reader): + with pytest.raises(BundleGateError) as error: + _open(reader, work_dir, factory.truncate_bundle(factory.make_bundle())) + + assert error.value.gate == 2 + + +def _never_called(message: str): + def _raise(*_args, **_kwargs): + raise AssertionError(message) + + return _raise + + +# ------------------------------------------------------------------- gate 3 + +AMBIGUOUS_REASON = 'wrong password, or the file is damaged or was altered' + + +def test_gate_3_wrong_password(work_dir, reader): + with pytest.raises(BundleGateError) as error: + _open(reader, work_dir, factory.make_bundle(), password='Wrong1Password!Here') + + assert error.value.gate == 3 + assert error.value.step is ImportStep.DECRYPTION + assert AMBIGUOUS_REASON in str(error.value) + + +def test_gate_3_altered_ciphertext_reads_the_same_as_a_wrong_password(work_dir, reader): + tampered = factory.tamper_ciphertext(factory.make_bundle()) + + with pytest.raises(BundleGateError) as altered: + _open(reader, work_dir, tampered) + with pytest.raises(BundleGateError) as wrong_password: + _open(reader, work_dir, factory.make_bundle(), password='Wrong1Password!Here') + + assert str(altered.value) == str(wrong_password.value) + assert altered.value.gate == wrong_password.value.gate == 3 + + +def test_gate_3_stream_without_a_final_tag_is_rejected(work_dir, reader): + with pytest.raises(BundleGateError) as error: + _open(reader, work_dir, factory.make_bundle(drop_final_tag=True)) + + assert error.value.gate == 3 + assert AMBIGUOUS_REASON in str(error.value) + + +def test_gate_3_leaves_no_decrypted_zip_behind(work_dir, reader): + with pytest.raises(BundleGateError): + _open(reader, work_dir, factory.make_bundle(), password='Wrong1Password!Here') + + assert list(work_dir.glob('*.zip')) == [] + + +def test_gate_3_normalises_the_password_to_nfc(work_dir, reader): + composed = 'Café1Model!Test' + decomposed = 'Café1Model!Test' + payload = factory.make_bundle(password=composed) + + opened = _open(reader, work_dir, payload, password=decomposed) + + assert opened.metadata['model_name'] == 'factory_model' + + +def test_gate_3_does_not_trim_the_password(work_dir, reader): + payload = factory.make_bundle(password='Trailing1Space!Test ') + + with pytest.raises(BundleGateError): + _open(reader, work_dir, payload, password='Trailing1Space!Test') + + +def test_multi_chunk_plaintext_round_trips(work_dir, reader): + """More than 64 KiB of plaintext, so `MESSAGE` and `FINAL` framing are both exercised.""" + spec = factory.ZipSpec() + # Incompressible on purpose: a compressible filler would deflate below one chunk and the test + # would silently stop covering multi-chunk framing. + spec.files['artifacts/prediction_model/model.pkl'] = random.Random( # noqa: S311 - filler + 20260825 + ).randbytes(200 * 1024) + payload = factory.make_bundle(spec) + assert len(payload) > fmt.CHUNK_SIZE + fmt.ABYTES + fmt.DIGEST_OFFSET + + opened = _open(reader, work_dir, payload) + + extracted = opened.extracted_dir / 'artifacts' / 'prediction_model' / 'model.pkl' + assert extracted.read_bytes() == spec.files['artifacts/prediction_model/model.pkl'] + + +# ------------------------------------------------------------------- gate 4 + + +@pytest.mark.parametrize( + ('spec_kwargs', 'match'), + [ + ({'bomb_bytes': 40 * 1024 * 1024}, 'compression ratio'), + ({'traversal_entry': 'artifacts/../../etc/passwd'}, 'escaping'), + ({'absolute_entry': '/etc/passwd'}, 'absolute'), + ({'symlinks': {'artifacts/link': '/etc/passwd'}}, 'symbolic link'), + ], +) +def test_gate_4_rejects_hostile_central_directories(work_dir, reader, spec_kwargs, match): + payload = factory.make_bundle(factory.ZipSpec(**spec_kwargs)) + + with pytest.raises(BundleGateError, match=match) as error: + _open(reader, work_dir, payload) + + assert error.value.gate == 4 + assert error.value.step is ImportStep.ARCHIVE_INSPECTION + assert _extracted_files(work_dir / 'extract') == [] + assert not (work_dir / 'extract').exists() + + +def test_gate_4_rejects_too_many_entries(work_dir): + limited = BundleReader(BundleLimits(max_entries=3)) + + with pytest.raises(BundleGateError, match='entries') as error: + _open(limited, work_dir, factory.make_bundle()) + + assert error.value.gate == 4 + + +def test_gate_4_rejects_an_oversized_uncompressed_total(work_dir): + limited = BundleReader(BundleLimits(max_uncompressed_bytes=64)) + + with pytest.raises(BundleGateError, match='uncompressed') as error: + _open(limited, work_dir, factory.make_bundle()) + + assert error.value.gate == 4 + + +def test_gate_4_rejects_content_that_is_not_an_archive(work_dir, reader): + payload = factory.encrypt_archive(b'not a zip at all') + + with pytest.raises(BundleGateError, match='readable archive') as error: + _open(reader, work_dir, payload) + + assert error.value.gate == 4 + + +# ------------------------------------------------------------------- gate 5 + + +def test_gate_5_refuses_a_pre_existing_extraction_root(work_dir, reader): + (work_dir / 'extract').mkdir() + + with pytest.raises(BundleGateError, match='already exists') as error: + _open(reader, work_dir, factory.make_bundle()) + + assert error.value.gate == 5 + assert error.value.step is ImportStep.EXTRACTION + + +def test_gate_5_extraction_writes_plain_files(work_dir, reader): + opened = _open(reader, work_dir, factory.make_bundle()) + + for path in opened.extracted_dir.rglob('*'): + if path.is_file(): + assert path.stat().st_mode & 0o111 == 0, path + + +# ------------------------------------------------------------------- gate 6 + + +@pytest.mark.parametrize( + ('spec', 'match'), + [ + (factory.ZipSpec(include_metadata=False), 'metadata document'), + ( + factory.ZipSpec( + document=factory.build_metadata_document(drop_top_level_key='parameters') + ), + 'parameters and metadata blocks', + ), + ( + factory.ZipSpec( + document=factory.build_metadata_document(drop_metadata_field='experiment_name') + ), + 'experiment_name', + ), + ( + factory.ZipSpec( + document=factory.build_metadata_document(drop_parameter='target_variable') + ), + 'target_variable', + ), + ( + factory.ZipSpec( + document=factory.build_metadata_document(metadata={'experiment_name': 'x' * 51}) + ), + 'experiment name', + ), + ( + factory.ZipSpec( + document=factory.build_metadata_document(metadata={'experiment_name': 'ab'}) + ), + 'experiment name', + ), + ], +) +def test_gate_6_rejects_a_malformed_metadata_document(work_dir, reader, spec, match): + with pytest.raises(BundleGateError, match=match) as error: + _open(reader, work_dir, factory.make_bundle(spec)) + + assert error.value.gate == 6 + assert error.value.step is ImportStep.STRUCTURE_VALIDATION + assert not (work_dir / 'extract').exists() + + +def test_gate_6_rejects_unreadable_metadata_json(work_dir, reader): + spec = factory.ZipSpec(include_metadata=False) + spec.extra_files['metadata.json'] = b'{not json' + + with pytest.raises(BundleGateError, match='readable JSON') as error: + _open(reader, work_dir, factory.make_bundle(spec)) + + assert error.value.gate == 6 + + +def test_gate_6_rejects_a_missing_artifacts_directory(work_dir, reader): + spec = factory.ZipSpec(files={}) + + with pytest.raises(BundleGateError, match='artifacts directory') as error: + _open(reader, work_dir, factory.make_bundle(spec)) + + assert error.value.gate == 6 + + +@pytest.mark.parametrize( + ('dropped', 'match'), + [ + ('artifacts/data_model/MLmodel', 'data_model artifacts are incomplete'), + ('artifacts/prediction_model/model.pkl', 'prediction_model artifacts are incomplete'), + ], +) +def test_gate_6_rejects_an_incomplete_model_directory(work_dir, reader, dropped, match): + spec = factory.ZipSpec() + spec.files.pop(dropped) + + with pytest.raises(BundleGateError, match=match) as error: + _open(reader, work_dir, factory.make_bundle(spec)) + + assert error.value.gate == 6 + + +def test_gate_6_rejects_a_bundle_without_data_model(work_dir, reader): + spec = factory.ZipSpec() + for name in list(spec.files): + if name.startswith('artifacts/data_model/'): + spec.files.pop(name) + + with pytest.raises(BundleGateError, match='no data_model artifacts') as error: + _open(reader, work_dir, factory.make_bundle(spec)) + + assert error.value.gate == 6 + + +# ------------------------------------------------------------------- gate 7 + + +@pytest.mark.parametrize('name', ['artifacts/run.sh', 'artifacts/tool.bin', 'artifacts/mod.so']) +def test_gate_7_rejects_content_a_model_export_should_not_carry(work_dir, reader, name): + spec = factory.ZipSpec() + spec.extra_files[name] = b'#!/bin/sh\n' + + with pytest.raises(BundleGateError, match='should not carry') as error: + _open(reader, work_dir, factory.make_bundle(spec)) + + assert error.value.gate == 7 + assert error.value.step is ImportStep.CONTENT_POLICY + assert not (work_dir / 'extract').exists() + + +# ------------------------------------------------------------------ ordering + + +def test_the_earlier_gate_wins(work_dir, reader): + """A bundle that would fail gate 4 and gate 6 must report gate 4, having extracted nothing.""" + spec = factory.ZipSpec( + traversal_entry='artifacts/../escape.csv', + document=factory.build_metadata_document(drop_parameter='target_variable'), + ) + + with pytest.raises(BundleGateError) as error: + _open(reader, work_dir, factory.make_bundle(spec)) + + assert error.value.gate == 4 + assert _extracted_files(work_dir / 'extract') == [] + + +def test_header_is_checked_before_the_password(work_dir, reader): + """A bundle with a bad header and a wrong password reports the header, not the password.""" + with pytest.raises(BundleGateError) as error: + _open(reader, work_dir, factory.make_bundle(magic=b'NOTSIENT'), password='Nope1!Nope1!') + + assert error.value.gate == 2 diff --git a/tests/laborious/utils/test_connectors_config.py b/tests/laborious/utils/test_connectors_config.py index b94bb8d..8dd82f5 100644 --- a/tests/laborious/utils/test_connectors_config.py +++ b/tests/laborious/utils/test_connectors_config.py @@ -1,11 +1,42 @@ -from os import environ +import tempfile +from os import environ, path + +import pytest from laborious.utils.connectors_config import ( + build_import_config, + build_import_status_config, build_minio_config, build_mlflow_config, build_opc_config, ) +IMPORT_ENV_VARS = ( + 'IMPORT_PASSWORD_KEY', + 'IMPORT_BUNDLE_BUCKET', + 'IMPORT_BUNDLE_PREFIX', + 'IMPORT_WORK_DIR', + 'IMPORT_MAX_OBJECT_BYTES', + 'IMPORT_MAX_ARCHIVE_ENTRIES', + 'IMPORT_MAX_UNCOMPRESSED_BYTES', + 'IMPORT_MAX_COMPRESSION_RATIO', + 'IMPORT_MODELS_COLLECTION', + 'IMPORT_BUNDLE_RETENTION_DAYS', + 'IMPORT_STATUS_DB_HOST', + 'IMPORT_STATUS_DB_PORT', + 'IMPORT_STATUS_DB_NAME', + 'IMPORT_STATUS_DB_USER', + 'IMPORT_STATUS_DB_PASSWORD', +) + + +@pytest.fixture +def clean_import_env(monkeypatch): + """Import configuration is read from a known-empty environment.""" + for name in IMPORT_ENV_VARS: + monkeypatch.delenv(name, raising=False) + return monkeypatch + def test_build_mlflow_config_with_env_vars(): # Arrange @@ -120,3 +151,83 @@ def test_build_minio_config_with_defaults(): 'retention_hours': 24, 'secure': False, } + + +def test_build_import_config_with_defaults(clean_import_env): + assert build_import_config() == { + 'password_key': None, + 'bucket': 'sientia', + 'prefix': 'imported_models/', + 'work_dir': path.join(tempfile.gettempdir(), 'sientia-import'), + 'max_object_bytes': 1024 * 1024 * 1024, + 'max_entries': 5000, + 'max_uncompressed_bytes': 4 * 1024 * 1024 * 1024, + 'max_compression_ratio': 200.0, + 'models_collection': 'models', + 'retention_days': 7, + } + + +def test_build_import_config_reads_the_environment(clean_import_env): + clean_import_env.setenv('IMPORT_PASSWORD_KEY', 'a2V5') + clean_import_env.setenv('IMPORT_BUNDLE_BUCKET', 'other-bucket') + clean_import_env.setenv('IMPORT_BUNDLE_PREFIX', 'incoming/models') + clean_import_env.setenv('IMPORT_WORK_DIR', '/var/lib/import') + clean_import_env.setenv('IMPORT_MAX_OBJECT_BYTES', '2048') + clean_import_env.setenv('IMPORT_MAX_ARCHIVE_ENTRIES', '12') + clean_import_env.setenv('IMPORT_MAX_UNCOMPRESSED_BYTES', '4096') + clean_import_env.setenv('IMPORT_MAX_COMPRESSION_RATIO', '12.5') + clean_import_env.setenv('IMPORT_MODELS_COLLECTION', 'models_v2') + clean_import_env.setenv('IMPORT_BUNDLE_RETENTION_DAYS', '14') + + config = build_import_config() + + assert config['password_key'] == 'a2V5' + assert config['bucket'] == 'other-bucket' + # A prefix without its trailing slash would make `imported_models_backup/x` pass the gate-1 + # prefix check, so the separator is added rather than trusted. + assert config['prefix'] == 'incoming/models/' + assert config['work_dir'] == '/var/lib/import' + assert config['max_object_bytes'] == 2048 + assert config['max_entries'] == 12 + assert config['max_uncompressed_bytes'] == 4096 + assert config['max_compression_ratio'] == 12.5 + assert config['models_collection'] == 'models_v2' + assert config['retention_days'] == 14 + + +def test_build_import_config_does_not_validate_the_password_key(clean_import_env): + """A missing or malformed key must fail the import at decryption, never at worker start.""" + clean_import_env.setenv('IMPORT_PASSWORD_KEY', 'not base64 at all') + + assert build_import_config()['password_key'] == 'not base64 at all' + + +def test_build_import_status_config_with_defaults(clean_import_env): + assert build_import_status_config() == { + 'host': 'localhost', + 'port': 5432, + 'dbname': 'sientia-core-mlops-bff', + 'user': 'mlops_bff_user', + 'password': '', + 'schema': 'public', + } + + +def test_build_import_status_config_reads_the_environment(clean_import_env): + clean_import_env.setenv('IMPORT_STATUS_DB_HOST', 'bff-postgres') + clean_import_env.setenv('IMPORT_STATUS_DB_PORT', '6543') + clean_import_env.setenv('IMPORT_STATUS_DB_NAME', 'other-db') + clean_import_env.setenv('IMPORT_STATUS_DB_USER', 'other-user') + clean_import_env.setenv('IMPORT_STATUS_DB_PASSWORD', 'secret') + + config = build_import_status_config() + + assert config == { + 'host': 'bff-postgres', + 'port': 6543, + 'dbname': 'other-db', + 'user': 'other-user', + 'password': 'secret', + 'schema': 'public', + } diff --git a/tests/laborious/utils/test_import_password.py b/tests/laborious/utils/test_import_password.py new file mode 100644 index 0000000..5d01876 --- /dev/null +++ b/tests/laborious/utils/test_import_password.py @@ -0,0 +1,99 @@ +"""The password envelope: AES-256-GCM in, plaintext password out, nothing in between.""" + +import base64 + +import pytest + +from laborious.utils.import_password import ( + EnvelopeError, + decrypt_password_envelope, + load_envelope_key, +) +from tests.helpers import bundle_factory as factory + +KEY = factory.DEFAULT_ENVELOPE_KEY +OTHER_KEY = bytes(range(1, 33)) + + +def test_recovers_the_password_used_at_export(): + envelope = factory.build_envelope('Correct1Horse!Battery', KEY) + + assert decrypt_password_envelope(envelope, KEY) == 'Correct1Horse!Battery' + + +def test_wrong_key_fails_authentication(): + envelope = factory.build_envelope('Correct1Horse!Battery', KEY) + + with pytest.raises(EnvelopeError, match='authenticated'): + decrypt_password_envelope(envelope, OTHER_KEY) + + +def test_tampered_envelope_fails_authentication(): + payload = bytearray(base64.b64decode(factory.build_envelope('Correct1Horse!Battery', KEY))) + payload[-1] ^= 0xFF + + with pytest.raises(EnvelopeError, match='authenticated'): + decrypt_password_envelope(base64.b64encode(bytes(payload)).decode(), KEY) + + +def test_malformed_base64_rejected(): + with pytest.raises(EnvelopeError, match='base64'): + decrypt_password_envelope('this is not base64!!', KEY) + + +def test_envelope_too_short_rejected(): + with pytest.raises(EnvelopeError, match='too short'): + decrypt_password_envelope(base64.b64encode(bytes(20)).decode(), KEY) + + +def test_empty_envelope_rejected(): + with pytest.raises(EnvelopeError, match='no password envelope'): + decrypt_password_envelope('', KEY) + + +def test_password_is_not_trimmed(): + """A trailing space is part of the secret; the KDF must see it.""" + envelope = factory.build_envelope('Trailing1Space!Test ', KEY) + + assert decrypt_password_envelope(envelope, KEY) == 'Trailing1Space!Test ' + + +def test_password_keeps_its_unicode_form(): + """Normalisation happens at key derivation, not here — the envelope round-trips verbatim.""" + decomposed = 'Café1Model!Test' + envelope = factory.build_envelope(decomposed, KEY) + + assert decrypt_password_envelope(envelope, KEY) == decomposed + + +def test_loads_a_valid_key(): + assert load_envelope_key(factory.envelope_key_b64(KEY)) == KEY + + +@pytest.mark.parametrize( + ('raw_key', 'match'), + [ + (None, 'not configured'), + ('', 'not configured'), + ('not base64 at all', 'valid base64'), + (base64.b64encode(bytes(16)).decode(), '32 bytes'), + (base64.b64encode(bytes(64)).decode(), '32 bytes'), + ], +) +def test_rejects_an_unusable_key(raw_key, match): + """There is no fallback: an unusable key never means the envelope is plaintext.""" + with pytest.raises(EnvelopeError, match=match): + load_envelope_key(raw_key) + + +def test_no_secret_appears_in_the_error_text(): + password = 'Correct1Horse!Battery' + envelope = factory.build_envelope(password, KEY) + + with pytest.raises(EnvelopeError) as error: + decrypt_password_envelope(envelope, OTHER_KEY) + + message = str(error.value) + assert password not in message + assert envelope not in message + assert factory.envelope_key_b64(OTHER_KEY) not in message From dac4f1718f1756d2ba9afa2abba958681d73e805 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 25 Aug 2026 17:07:42 -0300 Subject: [PATCH 26/64] QTZPOC-20 QTZPOC-20: Implement model import workflow for .sientia bundles Introduces the import_model workflow and its associated activities to enable importing .sientia bundles into MLflow and MongoDB. This includes: * New core workflow and activity logic for handling bundle download, decryption, validation, MLflow provisioning (experiments, runs, registered versions), and MongoDB model document creation. * Comprehensive E2E and unit test suites, including a dedicated test harness and a manual run script for development. * Updated .env.example and README.md with new configurations and usage instructions. * Ensures correct MLflow model source resolution for legacy loader compatibility and robust MongoDB document management. * Refactors existing utility scripts into a scripts/ directory. --- .env.example | 42 +- README.md | 150 ++- docs/model-import.md | 349 +++++++ docs/sientia-bundle-format.md | 196 ++++ e2e/conftest.py | 11 +- e2e/fixtures/experiment_run.sql | 22 +- e2e/fixtures/experiment_run_v12.sql | 36 + e2e/import_harness.py | 950 ++++++++++++++++++ e2e/test_import_harness_smoke.py | 196 ++++ e2e/test_import_model_digest.py | 214 ++++ e2e/test_import_model_gates.py | 675 +++++++++++++ e2e/test_import_model_happy_path.py | 314 ++++++ ...test_import_model_provisioning_failures.py | 364 +++++++ encode.sh | 13 - laborious/activities/activities.py | 7 +- laborious/activities/model_import.py | 13 +- laborious/worker/worker.py | 15 +- laborious/workflows/import_model.py | 18 +- pyproject.toml | 9 + requirements-dev.txt | 3 +- requirements-local.txt | 2 +- requirements.txt | 2 +- scripts/encode.sh | 20 + encrypt.py => scripts/encrypt.py | 62 +- scripts/import_model_manual_run.py | 497 +++++++++ inter_arrival.py => scripts/inter_arrival.py | 4 +- .../model_convert.ipynb | 55 +- run_coverage.sh => scripts/run_coverage.sh | 0 run_local.sh => scripts/run_local.sh | 0 tests.ipynb | 938 ----------------- tests/helpers/status_engine.py | 174 ++++ tests/laborious/activities/test_activities.py | 115 +++ .../activities/test_model_import_bundle.py | 565 +++++++++++ .../activities/test_model_import_errors.py | 410 ++++++++ .../test_model_import_provisioning.py | 656 ++++++++++++ .../activities/test_model_import_status.py | 754 ++++++++++++++ tests/laborious/test_import_absence.py | 264 +++++ .../worker/test_runtime_task_queues.py | 100 ++ .../laborious/workflows/test_import_model.py | 781 ++++++++++++++ tests/scripts/__init__.py | 0 tests/scripts/test_import_model_manual_run.py | 217 ++++ 41 files changed, 8168 insertions(+), 1045 deletions(-) create mode 100644 docs/model-import.md create mode 100644 docs/sientia-bundle-format.md create mode 100644 e2e/fixtures/experiment_run_v12.sql create mode 100644 e2e/import_harness.py create mode 100644 e2e/test_import_harness_smoke.py create mode 100644 e2e/test_import_model_digest.py create mode 100644 e2e/test_import_model_gates.py create mode 100644 e2e/test_import_model_happy_path.py create mode 100644 e2e/test_import_model_provisioning_failures.py delete mode 100755 encode.sh create mode 100755 scripts/encode.sh rename encrypt.py => scripts/encrypt.py (61%) create mode 100644 scripts/import_model_manual_run.py rename inter_arrival.py => scripts/inter_arrival.py (92%) rename model_convert.ipynb => scripts/model_convert.ipynb (54%) rename run_coverage.sh => scripts/run_coverage.sh (100%) rename run_local.sh => scripts/run_local.sh (100%) delete mode 100644 tests.ipynb create mode 100644 tests/helpers/status_engine.py create mode 100644 tests/laborious/activities/test_model_import_bundle.py create mode 100644 tests/laborious/activities/test_model_import_errors.py create mode 100644 tests/laborious/activities/test_model_import_provisioning.py create mode 100644 tests/laborious/activities/test_model_import_status.py create mode 100644 tests/laborious/test_import_absence.py create mode 100644 tests/laborious/workflows/test_import_model.py create mode 100644 tests/scripts/__init__.py create mode 100644 tests/scripts/test_import_model_manual_run.py diff --git a/.env.example b/.env.example index 6e621e9..d13ef47 100644 --- a/.env.example +++ b/.env.example @@ -32,4 +32,44 @@ MINIO_ENDPOINT_URL="http://localhost:9000" MINIO_ACCESS_KEY="sientia" MINIO_SECRET_KEY="sientia" MINIO_REGION_NAME="sa-east-1" -MINIO_DEFAULT_BUCKET="sientia" \ No newline at end of file +MINIO_DEFAULT_BUCKET="sientia" + +# Suffix for every Temporal task queue name; required, non-empty. +# The import worker listens on "import_model-${RUNTIME}-queue". +RUNTIME="local" + +# --- Model import (.sientia bundles) ----------------------------------------- +# The import worker is registered only when IMPORT_MODEL_ENABLED is truthy +# (1/true/yes/on). Do not turn it on before the V12 migration is applied to +# public.experiment_run in the BFF database. +IMPORT_MODEL_ENABLED="false" + +# base64 of the 32-byte AES-256-GCM key that wraps bundle passwords. +# Generate a throwaway one with: openssl rand -base64 32 +IMPORT_PASSWORD_KEY="replace-with-base64-of-32-random-bytes" + +# Where the frontend uploads the bundle. The object key must sit under the prefix. +IMPORT_BUNDLE_BUCKET="sientia" +IMPORT_BUNDLE_PREFIX="imported_models/" + +# MongoDB collection holding the model listing the import writes its document into. +IMPORT_MODELS_COLLECTION="models" + +# The import log lives in the Spring Boot BFF database, a second Postgres +# connection independent of POSTGRES_* above. The schema (public) and the table +# (experiment_run) are constants of the code, not configuration. +IMPORT_STATUS_DB_HOST="localhost" +IMPORT_STATUS_DB_PORT="5432" +IMPORT_STATUS_DB_NAME="sientia-core-mlops-bff" +IMPORT_STATUS_DB_USER="mlops_bff_user" +IMPORT_STATUS_DB_PASSWORD="bff_password" + +# --- scripts/import_model_manual_run.py only --------------------------------- +# Inputs for the manual run that stands in for the frontend. The script writes +# one object and one row to whatever the variables above name. +IMPORT_SCRIPT_BUNDLE_PATH="tests/fixtures/bundle/golden_model_v3.sientia" +IMPORT_SCRIPT_BUNDLE_PASSWORD="the-password-the-bundle-was-written-with" +IMPORT_SCRIPT_USERNAME="manual.run@example.com" +# Cell 2's guard: restate IMPORT_STATUS_DB_HOST here (or in the cell) after reading what cell 1 +# printed. Left empty on purpose — the script refuses to write until it matches. +IMPORT_SCRIPT_CONFIRM_TARGET="" \ No newline at end of file diff --git a/README.md b/README.md index 0b5c844..c818329 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ A comprehensive, Temporal-based ML orchestration system for industrial data proc - [Minimal Retrain Workflow](#4-minimal-retrain-workflow-minimal_retrainpy) - [Drift Workflow](#5-drift-workflow-driftpy) - [Simple Metrics Workflow](#6-simple-metrics-workflow-simple_metricspy) + - [Import Model Workflow](#7-import-model-workflow-import_modelpy) - [Installation & Setup](#installation--setup) - [Prerequisites](#prerequisites) - [Environment Setup](#environment-setup) @@ -31,6 +32,7 @@ A comprehensive, Temporal-based ML orchestration system for industrial data proc - [Running Tests and Coverage](#running-tests-and-coverage) - [Manual Test Execution](#manual-test-execution) - [Manual Application Execution](#manual-application-execution) + - [Running One Model Import by Hand](#running-one-model-import-by-hand) - [Code Quality & Validation](#code-quality--validation) - [Overview](#overview) - [Validation Tools](#validation-tools) @@ -126,7 +128,8 @@ Laborious uses a Temporal-based architecture with strong separation of concerns ### Key Components #### **Worker (`laborious/worker/worker.py`)** -- Temporal client setup, four workers via `sientia_do.temporal.worker.prepare_worker` +- Temporal client setup, four workers via `sientia_do.temporal.worker.prepare_worker`, plus a + fifth registered only when `IMPORT_MODEL_ENABLED` is truthy (`1`, `true`, `yes`, `on`) - Runtime-scoped task queues: `{workflow}-{RUNTIME}-queue` for all workflows - Metrics server initialization, notification handler setup - Graceful shutdown and autoscaling-friendly behavior @@ -135,6 +138,16 @@ Laborious uses a Temporal-based architecture with strong separation of concerns `simple_metrics-queue`. Use `drift-{RUNTIME}-queue` and `simple_metrics-{RUNTIME}-queue` matching the worker pod `RUNTIME` env (same as `predictions_batch` / `minimal_retrain`). +Task queues, one per worker: + +| Queue | Worker | Registered | +|---|---|---| +| `predictions_batch-{RUNTIME}-queue` | PredictionsBatch (+ its two sub-workflows) | always | +| `minimal_retrain-{RUNTIME}-queue` | MinimalRetrain | always | +| `drift-{RUNTIME}-queue` | Drift | always | +| `simple_metrics-{RUNTIME}-queue` | SimpleMetrics | always | +| `import_model-{RUNTIME}-queue` | ImportModel | only when `IMPORT_MODEL_ENABLED` is truthy | + #### **Workflows (`laborious/workflows/`)** - `predictions_batch.py`: Batch prediction entry point - `sub_workflows/prediction_process.py`: Core prediction pipeline @@ -142,6 +155,8 @@ matching the worker pod `RUNTIME` env (same as `predictions_batch` / `minimal_re - `minimal_retrain.py`: Automated model retraining and production update - `drift.py`: Data drift detection and monitoring - `simple_metrics.py`: Regression metrics calculation (RMSE, MSE, MAE, R²) +- `import_model.py`: `.sientia` bundle import — registered model + model document, behind + `IMPORT_MODEL_ENABLED` #### **Activities (`laborious/activities/`)** - `gates.py`: Data quality validation, filtering, and data formatting operations @@ -509,6 +524,55 @@ flowchart LR } ``` +### 7. Import Model Workflow (`import_model.py`) + +The **ImportModel** workflow turns an encrypted `.sientia` bundle already uploaded to MinIO into a +registered MLflow model plus one document in the MongoDB `models` collection — or into a durable +`ERROR` row carrying a sentence a person can read. It is registered **only** when +`IMPORT_MODEL_ENABLED` is set, so the importer can live in its own runtime with its own credentials. + +Full contract for the frontend and the BFF: [`docs/model-import.md`](docs/model-import.md). +The file format: [`docs/sientia-bundle-format.md`](docs/sientia-bundle-format.md). + +#### Execution Flow +1. **Claim**: takes ownership of the `public.experiment_run` row the BFF inserted (it never inserts one) +2. **Download**: object limits, then the mandatory `expected_digest` comparison +3. **Open**: seven ordered gates — header, decryption, archive inspection, extraction, structure, content policy +4. **Provision**: experiment, a new run with the recorded name, the artifact tree, a registered version (stage untouched) +5. **List**: the MongoDB model document, `active: false` +6. **Finish**: cleanup, then one terminal write classifying the outcome once + +#### Architecture Diagram +```mermaid +flowchart LR + A[1. claim_import_status] --> B[2. download_import_bundle] --> C[3. open_import_bundle] + C --> D[4. create_import_experiment] --> E[5. upload_import_artifacts] + E --> F[6. register_import_model_version] --> G[7. write_import_model_document] + G --> H[8. cleanup_import_files] --> I[9. record_import_terminal_status] + + A -.-> BFF[(sientia-core-mlops-bff)] + B -.-> MinIO[(MinIO)] + F -.-> MLflow[(MLflow)] + G -.-> Mongo[(MongoDB)] + I -.-> BFF +``` + +#### Input Parameters +```json +{ + "import_run_id": 41, + "object_key": "imported_models/sales_forecast_v3.sientia", + "expected_digest": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + "password_envelope": "Base64OfNonceCiphertextTag==", + "bucket": "sientia" +} +``` + +A successful import leaves the registered version **unstaged** and the model document **inactive**: +promoting and activating are manual operator steps (QTZPOC-21). The `V12` migration of +`sientia-core-mlops-bff` must be applied before enabling the worker — see +[`docs/model-import.md`](docs/model-import.md) § 7. + ## 📋 Prerequisites - Python 3.11+ @@ -589,10 +653,10 @@ Use the provided script to run the application locally: ```bash # Make script executable (first time only) -chmod +x run_local.sh +chmod +x scripts/run_local.sh -# Run the application -./run_local.sh +# Run the application (from the repository root) +./scripts/run_local.sh ``` The script will: @@ -606,10 +670,10 @@ Use the provided script to run tests with coverage: ```bash # Make script executable (first time only) -chmod +x run_coverage.sh +chmod +x scripts/run_coverage.sh -# Run tests with coverage -./run_coverage.sh +# Run tests with coverage (from the repository root) +./scripts/run_coverage.sh ``` The script will: @@ -654,6 +718,62 @@ fi python -m laborious.worker.worker ``` +### Running One Model Import by Hand + +[`scripts/import_model_manual_run.py`](scripts/import_model_manual_run.py) stands in for the frontend +that does not exist yet. The Angular screen plus the Spring Boot BFF will, between them, hash a +`.sientia` file, wrap the bundle password, upload the object and insert one `PENDING` row before +starting `import_model`; this script does exactly those four things against whatever `.env` points +at, then reads the outcome back and can undo itself. + +It is a **development tool**. It writes to whatever `.env` names — a real object in a real bucket and +a real row in a real `public.experiment_run` — and a `.env` copied from a cluster deployment can +perfectly well name production. It is not for production use. + +**Prerequisites** + +- A reachable environment: the BFF Postgres, MinIO, MongoDB, MLflow and Temporal that `.env` names. +- The `V12` migration applied to `public.experiment_run` in the BFF database. Without it the import + provisions the model and then cannot record the outcome — see + [`docs/model-import.md`](docs/model-import.md) § 7. +- A worker running with `IMPORT_MODEL_ENABLED` truthy and the same `RUNTIME`, if you want cell 8 (or + the Temporal UI) to actually execute the run. +- The import variables from [`.env.example`](.env.example) filled in, including `IMPORT_PASSWORD_KEY` + and the script's own `IMPORT_SCRIPT_BUNDLE_PATH`, `IMPORT_SCRIPT_BUNDLE_PASSWORD` and + `IMPORT_SCRIPT_USERNAME`. + +**Running it.** The file is split into `# %%` cells: run them one at a time in the VS Code +interactive window, or run the module top to bottom. + +```bash +source ./venv/bin/activate + +# top to bottom (cells 5 and 6 write; cell 2 must pass first) +python scripts/import_model_manual_run.py +``` + +| Cell | Does | Writes? | +|---|---|---| +| 1 | Resolves and prints every target — Postgres, MinIO, Mongo, MLflow, Temporal, the task queue | no | +| 2 | **Confirmation guard** | no | +| 3 | Picks the bundle, computes `expected_digest`, prints the parsed header | no | +| 4 | Builds the AES-256-GCM password envelope | no | +| 5 | Uploads the object, re-reads it and checks the digest still matches | **yes** | +| 6 | Inserts the `PENDING` / `run_type='IMPORT'` row, prints `import_run_id` | **yes** | +| 7 | Prints the workflow input as JSON, ready for the Temporal UI | no | +| 8 | Optionally starts `import_model` and waits (`START_THE_WORKFLOW`) | starts a run | +| 9 | Reads the outcome back: the row, the MLflow experiment/run/version, the model document | no | +| 10 | Undo: deletes the object and the row always; the provisioning behind `DELETE_PROVISIONING` | deletes | + +**The guard.** Cells 5 and 6 are the only cells that write, and neither runs until cell 2 passes. +Cell 2 compares a confirmation value against the Postgres host cell 1 resolved: set +`IMPORT_SCRIPT_CONFIRM_TARGET` in `.env`, or paste the host into the cell. It is a read-then-restate +rather than a boolean on purpose — a flag can be flipped without ever looking at what it points at. + +No credential is written in the script and none is printed: it prints hosts, ports, database, bucket +and object names, the row id and the workflow input, and never the bundle password, the envelope key +or the envelope's plaintext. + ## Code Quality & Validation ### Overview @@ -819,6 +939,22 @@ See [OPC UA Communication](#opc-ua-communication) for semantics, concurrency, an | `MLFLOW_PORT` | MLFlow server port | `5080` | Yes | | `MLFLOW_USERNAME` | MLFlow username | `aignosi` | Yes | | `MLFLOW_PASSWORD` | MLFlow password | `aignosi` | Yes | +| `IMPORT_MODEL_ENABLED` | Register the import worker on `import_model-{RUNTIME}-queue` | _(unset)_ | No | +| `IMPORT_PASSWORD_KEY` | base64 of the 32-byte AES-256-GCM key wrapping bundle passwords | _(none)_ | If importing | +| `IMPORT_BUNDLE_BUCKET` | Bucket holding uploaded `.sientia` objects | `sientia` | No | +| `IMPORT_BUNDLE_PREFIX` | Key prefix the uploads must sit under | `imported_models/` | No | +| `IMPORT_BUNDLE_RETENTION_DAYS` | Lifecycle expiry applied to that prefix | `7` | No | +| `IMPORT_WORK_DIR` | Worker scratch directory for bundles | `/sientia-import` | No | +| `IMPORT_MAX_OBJECT_BYTES` | Gate 1 object size ceiling | `1073741824` | No | +| `IMPORT_MAX_ARCHIVE_ENTRIES` | Gate 4 entry count ceiling | `5000` | No | +| `IMPORT_MAX_UNCOMPRESSED_BYTES` | Gate 4 uncompressed size ceiling | `4294967296` | No | +| `IMPORT_MAX_COMPRESSION_RATIO` | Gate 4 compression ratio ceiling | `200` | No | +| `IMPORT_MODELS_COLLECTION` | MongoDB collection holding the model listing | `models` | No | +| `IMPORT_STATUS_DB_HOST` | Import log database host (the BFF's) | `localhost` | If importing | +| `IMPORT_STATUS_DB_PORT` | Import log database port | `5432` | No | +| `IMPORT_STATUS_DB_NAME` | Import log database name | `sientia-core-mlops-bff` | No | +| `IMPORT_STATUS_DB_USER` | Import log database user | `mlops_bff_user` | If importing | +| `IMPORT_STATUS_DB_PASSWORD` | Import log database password | _(empty)_ | If importing | | `OPC_CONFIG` | OPC server configuration (JSON) | `{}` | No | | `OPC_ID` | OPC server identifier | `1` | No | | `OPC_URL` | OPC server URL | `opc.tcp://localhost:4840` | No | diff --git a/docs/model-import.md b/docs/model-import.md new file mode 100644 index 0000000..125f36a --- /dev/null +++ b/docs/model-import.md @@ -0,0 +1,349 @@ +# Importing a model (`import_model`) + +The `import_model` workflow turns an encrypted `.sientia` object in MinIO into a registered MLflow +model plus one document in the MongoDB `models` collection — or into a row that says, in a sentence a +person can read, why it did not. + +This document is the contract for the two sides that talk to it: the **frontend**, which uploads the +object and builds the request, and the **BFF**, which owns the import log row. Everything here is +enforced by code in this repository; where a statement is about a column or a constraint, the +authority is the migration named in §7. + +Format of the file itself: [`sientia-bundle-format.md`](sientia-bundle-format.md). + +--- + +## 1. What the caller does, in order + +1. **Upload** the `.sientia` object to the import bucket, under the configured prefix (default + `imported_models/`). The key must end in `.sientia`. +2. **Compute `expected_digest`** — the SHA-256, lowercase hex, of the **exact bytes that were + uploaded**. Not of a re-read, not of a re-serialisation: of what went over the wire. +3. **Encrypt the bundle password** into a `password_envelope` (§3). +4. **Insert the import log row** (§5) and take its `id`. +5. **Start the workflow** on `import_model-{RUNTIME}-queue` with the input of §2. + +The workflow claims the row, opens the bundle, provisions MLflow, writes the model document, cleans +up and records one terminal verdict. It never inserts a row, never promotes a version and never +activates a model. + +## 2. The workflow input + +```json +{ + "import_run_id": 41, + "object_key": "imported_models/sales_forecast_v3.sientia", + "expected_digest": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + "password_envelope": "Base64OfNonceCiphertextTag==", + "bucket": "sientia" +} +``` + +| Field | Required | Meaning | +|---|---|---| +| `import_run_id` | yes | primary key of the `public.experiment_run` row the BFF inserted. Must be a positive integer | +| `object_key` | yes | the uploaded object's key. Must match the row's `file_name` when the row carries one | +| `expected_digest` | **yes** | SHA-256 of the uploaded bytes, 64 lowercase hex characters. There is no path that skips the comparison: a missing value fails at `received`, a mismatching one at `download`, before the header is parsed | +| `password_envelope` | yes | the bundle password, encrypted (§3). A plaintext `password` field is **refused** | +| `bucket` | no | defaults to `IMPORT_BUNDLE_BUCKET` | + +**Fields that are refused rather than ignored**: `project_id`, `project_name`, +`provisioning_target`, `schema`, `table_name`, `status_table`, `password`. An input carrying any of +them fails at `received`, so no caller can believe it recorded something the import does not record +(see §10). + +## 3. The password envelope + +AES-256-GCM, and the ciphertext layout is what the WebCrypto API returns natively: + +``` +envelope = base64( nonce(12 bytes) || ciphertext || tag(16 bytes) ) +``` + +`crypto.subtle.encrypt({name: 'AES-GCM', iv: nonce}, key, plaintext)` returns `ciphertext || tag` +already concatenated, so the browser side is: generate 12 random bytes, encrypt, concatenate the +nonce in front, base64 the result. + +The key is a 32-byte AES-256 key shared with the worker through `IMPORT_PASSWORD_KEY` (base64). The +worker decrypts the envelope **inside the activity that uses the password**, so no plaintext password +is ever an activity input, an activity result or a workflow variable — and therefore never appears in +the Temporal event history. A test asserts that for every failure path. + +A failure to open the envelope is recorded as a `decryption` failure, distinct in the logs from a +wrong *bundle* password but sharing its step, because the step says where the import stopped, not who +is at fault. + +## 4. Environment + +| Variable | Default | What it is | +|---|---|---| +| `IMPORT_MODEL_ENABLED` | unset | registers the fifth (import) worker. Truthy values: `1`, `true`, `yes`, `on` | +| `IMPORT_PASSWORD_KEY` | — | base64 of the 32-byte AES-256-GCM key that wraps bundle passwords | +| `IMPORT_BUNDLE_BUCKET` | `sientia` | bucket holding the uploaded bundles | +| `IMPORT_BUNDLE_PREFIX` | `imported_models/` | key prefix the uploads must sit under | +| `IMPORT_BUNDLE_RETENTION_DAYS` | `7` | lifecycle expiry the importer applies to that prefix | +| `IMPORT_WORK_DIR` | `/sientia-import` | worker scratch directory | +| `IMPORT_MAX_OBJECT_BYTES` | 1 GiB | gate 1 | +| `IMPORT_MAX_ARCHIVE_ENTRIES` | `5000` | gate 4 | +| `IMPORT_MAX_UNCOMPRESSED_BYTES` | 4 GiB | gate 4 | +| `IMPORT_MAX_COMPRESSION_RATIO` | `200` | gate 4 | +| `IMPORT_MODELS_COLLECTION` | `models` | the MongoDB collection that *is* the model listing | +| `IMPORT_STATUS_DB_HOST` | `localhost` | the BFF database holding the import log | +| `IMPORT_STATUS_DB_PORT` | `5432` | | +| `IMPORT_STATUS_DB_NAME` | `sientia-core-mlops-bff` | | +| `IMPORT_STATUS_DB_USER` | `mlops_bff_user` | | +| `IMPORT_STATUS_DB_PASSWORD` | empty | | + +`IMPORT_STATUS_DB_*` is a **second** Postgres connection, independent of the one Laborious already +holds for predictions. The import writes to that database and no other, and to one table in it. + +A missing or malformed `IMPORT_PASSWORD_KEY` is an import failure at the decryption step, not a boot +failure: the worker starts either way. + +## 5. The import log row — the BFF's half + +The row is `public.experiment_run` in `sientia-core-mlops-bff`. **The workflow never inserts it.** +The inserting side writes: + +| Column | Value | +|---|---| +| `run_type` | `'IMPORT'` | +| `status` | `'PENDING'` | +| `username` | whoever asked for the import | +| `bucket_name` | the bucket the object was uploaded to | +| `file_name` | the object key | +| `request_data` | the request payload. If it carries a digest under `expected_digest`, `digest`, `sha256` or `file_digest`, the workflow checks it against the input's | + +Then it passes the returned `id` as `import_run_id`. + +The workflow's first act is to **claim** that row: + +```sql +UPDATE public.experiment_run + SET status = 'ORCHESTRATOR_WAITING_PROC', updated_at = :now, + orchestrator_response_data = :detail + WHERE id = :import_run_id + AND run_type = 'IMPORT' + AND (status = 'PENDING' + OR (status = 'ORCHESTRATOR_WAITING_PROC' + AND orchestrator_response_data -> 'import' ->> 'workflow_id' = :workflow_id)) +``` + +The second disjunct makes a re-execution of the same workflow idempotent instead of a conflict. Every +later write keys on `id` alone, because ownership was settled here. + +**Columns the workflow never writes**: `username`, `request_data`, `bucket_name`, `file_name`, +`run_type`, `created_at`. **Columns it does write**: `status`, `error_message`, `updated_at`, +`orchestrator_response_data`, and — as soon as the bundle is open, before anything exists in MLflow — +`experiment_name` (the bundle's own, verbatim) and `run_name` (`import-{model_name}-v{model_version}`, +truncated inside the name segment to fit 50 characters). Those two are **overwritten** by the +workflow; whatever the inserting side put there is replaced. + +### When the row cannot be claimed + +If the claim matches nothing — no such id, a `run_type='TRAIN'` row, or a row already running under +another workflow — the import **writes nothing anywhere**, because the row belongs to someone else. +Instead it: + +- fails the Temporal run with error type `ImportLogRowNotClaimableError` and code + `IMPORT_LOG_ROW_NOT_CLAIMABLE`, non-retryable; +- increments the import error metric and fires a notification; +- logs the id and the observed state (which of the three causes it was). + +So a row left `PENDING` with no `workflow_id` in its detail JSON means the workflow never owned it. +**Closing that row out belongs to whichever side inserted it** — by timeout, by a sweeper, by a person +— however that side chooses. The importer deliberately does not guess. + +## 6. Status vocabulary + +| Status | Written when | New in `V12`? | +|---|---|---| +| `ORCHESTRATOR_WAITING_PROC` | claimed, and while running | no — **reused** from the training flow | +| `ORCHESTRATOR_VALIDATION_ERROR` | failure at `received` … `content_policy` — the user can act on it | no — **reused** | +| `IMPORT_SUCCESS` | every step succeeded | yes | +| `IMPORT_ERROR` | failure at `experiment_creation` … `model_document`, or unclassified — only support can act on it | yes | + +The two reused values describe a *stage*, not a flow: what says the row is an import is `run_type`, +not the status. There is deliberately no `IMPORT_RUNNING`; it would carry no information `run_type` +does not already carry, and it would widen both the pairing `CHECK` and the frontend's switch for +nothing. + +`error_message` is the user-facing sentence, taken from the catalog in §8. On success it is written as +`NULL`, never `''` — the column carries a `LENGTH(error_message) >= 3` check. + +## 7. The migration this depends on + +`sientia-dataops-database-migrations`, +`postgres/sientia-core-mlops-bff/migrations/V12__add_import_statuses_to_experiment_run.sql`: + +- adds `IMPORT_SUCCESS` and `IMPORT_ERROR` to `chk_experiment_run_status`; +- adds `chk_experiment_run_terminal_status_run_type`, pairing only the **terminal** values with their + `run_type` (`IMPORT_*` on `'IMPORT'`, `TRAINING_*` on `'TRAIN'`) and leaving the shared + `ORCHESTRATOR_*` values out of the rule; +- **adds no columns at all.** + +`V12` must be merged and applied before `IMPORT_MODEL_ENABLED` is set anywhere. Without it, an import +provisions the model and then cannot record its verdict: the claim still succeeds (no new columns are +needed for it), the old `CHECK` refuses `IMPORT_SUCCESS`, and the failure surfaces as +`IMPORT_STATUS_NOT_RECORDED` on the first attempt — no retry wait — leaving the row in +`ORCHESTRATOR_WAITING_PROC` with a fully provisioned model behind it. + +## 8. `orchestrator_response_data`, and the error catalog + +There is no column for the error code and none for the workflow identifier (decision D21). Both live +at fixed keys in the JSON column the workflow already owns, and the whole `import` object is rewritten +on every write, so the identifier the claim wrote is never dropped: + +```json +{ + "import": { + "workflow_id": "import_model-41-01HW…", + "code": "IMPORT_BUNDLE_INCOMPLETE", + "step": "structure_validation", + "gate": 6, + "source": { "model_name": "sales_forecast", "experiment_name": "…", "…": "…" }, + "cleanup_failed": false + } +} +``` + +- `workflow_id` — what a user quotes to support, and what the claim's idempotency disjunct reads. +- `code` — the frontend's i18n key. **Absent on success.** +- `step` — where the import stopped (the values of §9). +- `gate` — which of the seven gates rejected it, when one did. +- `source` — the bundle's own metadata block, once it is known. +- `cleanup_failed` — temporary files were left on the worker. Present on a **successful** import too: + it never changes the status, because the user's model was imported. + +### Code → reason + +`error_message` is exactly the sentence in this table; nothing is interpolated into it, and the +exception's text — paths, table names, exception classes, tracebacks — reaches only the logs. + +| Code | Step | Sentence written to `error_message` | +|---|---|---| +| `IMPORT_REQUEST_INVALID` | `received` | The import could not be started because the request was incomplete or contradictory. Nothing was created. Submit the import again. | +| `IMPORT_FILE_UNREADABLE` | `download` | The uploaded file could not be read, or it is not the file that was sent. Upload it again. | +| `IMPORT_BUNDLE_CANNOT_BE_OPENED` | `decryption` | This file could not be opened: the password is wrong, or the file is damaged or was altered. Confirm the password with whoever exported the model, then try again. | +| `IMPORT_BUNDLE_REJECTED` | `archive_inspection` | The file's contents did not pass the platform's safety checks, so it was not opened. Export the model again from the origin platform. | +| `IMPORT_BUNDLE_UNPACK_FAILED` | `extraction` | The file could not be unpacked safely and nothing from it was kept. Export the model again from the origin platform. | +| `IMPORT_BUNDLE_INCOMPLETE` | `structure_validation` | This file is not a complete model export — part of what the platform needs is missing from it. Export the model again from the origin platform. | +| `IMPORT_BUNDLE_UNEXPECTED_CONTENT` | `content_policy` | The file contains something a model export should not contain, so it was rejected. Export the model again from the origin platform. | +| `IMPORT_STORAGE_PREPARATION_FAILED` | `experiment_creation` | The platform could not prepare a place to keep this model. The import stopped and no model was created. Contact support with this import's identifier. | +| `IMPORT_MODEL_FILES_NOT_STORED` | `artifact_upload` | The model's files could not be stored on this platform. The import stopped and the model is not available. Contact support with this import's identifier. | +| `IMPORT_MODEL_NOT_REGISTERED` | `registration` | The model's files were stored, but the model itself could not be registered, so it cannot be used. Contact support with this import's identifier. | +| `IMPORT_MODEL_SETTINGS_NOT_SAVED` | `model_document` | The model was registered, but the settings that tell the platform how to run it could not be saved. Contact support with this import's identifier. | +| `IMPORT_UNEXPECTED_ERROR` | — | The import stopped for an unexpected reason and no model was created. Contact support with this import's identifier. | + +Three codes never reach a row's `error_message`, because there is no row to write or no failure to +report to a user. They exist in the logs, the metric and the notification only: + +| Code | When | +|---|---| +| `IMPORT_LOG_ROW_NOT_CLAIMABLE` | the row is not this workflow's (§5) | +| `IMPORT_STATUS_NOT_RECORDED` | the verdict itself could not be written; the log line carries the verdict it could not record | +| `IMPORT_CLEANUP_INCOMPLETE` | temporary files were left behind; also flagged as `cleanup_failed` in the detail | + +## 9. Steps + +`received`, `download`, `decryption`, `archive_inspection`, `extraction`, `structure_validation`, +`content_policy`, `experiment_creation`, `artifact_upload`, `registration`, `model_document`, +`cleanup`. The first seven are the gates (see the format document); a failure in any of them is a +`ORCHESTRATOR_VALIDATION_ERROR`. The next four are provisioning; a failure there is an +`IMPORT_ERROR`. `cleanup` is never a reported failure. + +## 10. The import knows nothing about projects + +The model listing **is** the MongoDB `models` document. No migrated relational database in this +platform holds a project table or a model table, nothing in this repository reads a project-to-model +link, and the workflow input carries no project target — which is why naming one is refused rather +than accepted and ignored. Whatever the import screen does with projects is the frontend's business. + +The bundle's own `metadata.model_project` is logged as an MLflow parameter (`source.model_project`), +as a record of where the model came from. It is not a link. + +## 11. What MLflow ends up holding + +| Thing | Value | +|---|---| +| Experiment | `metadata.experiment_name` from the bundle, created if absent, reused if present | +| Run | always **new**, created with the `run_name` already written on the record | +| Artifacts | the bundle's whole `artifacts/` tree, with `prediction_model` and `data_model` at the run's artifact root | +| Parameters | the bundle's `parameters`, verbatim, plus the origin metadata as `source.*` (including `source.run_id`, which is never dereferenced) | +| Registered version | created on demand; source is the run's **resolved artifact URI** with `/prediction_model` appended, e.g. `mlflow-artifacts:/17//artifacts/prediction_model` | +| Stage | **untouched** | + +The source shape matters and is easy to get wrong: the legacy loader extracts the run id as +`source.split('/')[2]`, which is the run id in the resolved artifact URI but the *artifact path* in a +literal `runs://prediction_model`. Registering with the unresolved URI breaks loading +silently — nothing raises, the loader just returns the wrong string. This is the same value +`mlflow.register_model` stores for the training flow, because it resolves the `runs:/` URI before +writing it; `MlflowClient.create_model_version` does not, so the import resolves it itself. + +## 12. The model document + +Written **last**, into the collection named by `IMPORT_MODELS_COLLECTION` (default `models`): + +```json +{ + "id": "31", + "name": "sales_forecast", + "active": false, + "model_config": { + "transform_flavor": "sklearn", + "predict_flavor": "joblib", + "retention_minutes": 60, + "target": "SE" + } +} +``` + +- `id` is `str(max(int(id) for id in collection) + 1)` — computed in Python, from the ids that are + numbers. Never a `count`, never a lexicographic sort (which would answer `"9" > "30"`). Ids that + are not decimal numbers are ignored for the maximum and counted in the log. +- `name` is `metadata.model_name` from the bundle, verbatim. +- `active` is `false`, always. No code path in the import writes or updates a document with + `active: true`. +- `target` comes from `parameters.target_variable`; nothing else supplies it. +- The `models` collection carries a **unique sparse index on `id`**, created by the MongoDB baseline + migration. A duplicate-key insert is reported as a `model_document` failure — no retry, no id + recomputation. + +**A name already in the collection is left exactly as it is.** The import writes nothing, modifies +nothing, logs the skip and succeeds: `active`, `target` and the flavors of a model already on the +platform are operator-owned. + +## 13. After a successful import — the manual steps + +A `COMPLETED` import is **not** a servable model. Stated plainly: + +- the registered version is **unstaged**, and `load_predict_model` only sees a version once it is in + stage `Production`; +- the model document is **inactive** (`active: false`). + +So an operator must promote the version to `Production` and set `active: true`. This workflow never +promotes and never activates — that runbook is QTZPOC-21's. + +One more thing to know before serving an imported model: `predict_flavor: 'joblib'` is a **read-only** +flavor. It is what the artifact actually is, and it is enough to load and predict, but nothing in this +platform writes that flavor — so an imported model cannot be retrained until a write flavor is +configured for it. + +## 14. Retry, and what does not retry + +Only the status writes retry, and only because `UPDATE ... SET WHERE id` leaves the row +the same however many times it runs: + +| Call | Attempts | Why | +|---|---|---| +| claim, terminal write | 10, 1 s → 30 s backoff (≈151 s envelope) | the verdict must land; the database being briefly unavailable is exactly what a retry fixes | +| progress writes, names | 3, capped at 2 s | a hint nobody reads — the terminal write rewrites it — must not stall a working import | +| everything else | 1 | a repeated registry call or Mongo insert can write twice | + +`ImportLogRowNotClaimableError` and `ImportStatusRejectedError` are non-retryable: the first would +read the same rows and reach the same conclusion, and the second means the statement itself was +refused (a `CHECK` violation, an undefined column, a value too long), which a second attempt cannot +change. + +There is no rollback, no compensation, no resume and no reconciliation. A failed import stays failed; +trying again is a new upload and a new row. diff --git a/docs/sientia-bundle-format.md b/docs/sientia-bundle-format.md new file mode 100644 index 0000000..79dbc73 --- /dev/null +++ b/docs/sientia-bundle-format.md @@ -0,0 +1,196 @@ +# The `.sientia` bundle format + +A `.sientia` file is one exported MLflow model: its whole artifact tree, its run parameters and six +fields of origin metadata, zipped and then encrypted under a password the exporter chose. It is +written by the Streamlit platform (`sientia-projects-templates`) and read by this repository's +importer. Two programs in two repositories agree on these bytes, so the constants below are +duplicated by necessity — the drift is held in place by tests on both sides, not by a shared library. + +This document covers both directions: **§1–§3 how a bundle is generated**, so the producer can be +audited against it, and **§4–§7 how it is read**, so a rejection can be explained. + +Normative implementations: + +| Side | Where | +|---|---| +| Writer | `sientia-projects-templates`, `app/src/operations/model_export/{crypto,bundle,exporter}.py` | +| Reader | this repository, `laborious/utils/bundle/{format,reader}.py` | +| Test writer | this repository, `tests/helpers/bundle_factory.py` — mirrors the producer field for field, for building hostile variants | +| Golden fixture | `tests/fixtures/bundle/` — a real producer artefact, with its password and producer commit recorded in that directory's `README.md` | + +--- + +## 1. Generation, step by step + +Entry point: `exporter.export_model_bundle(*, client, run_id, model_name, experiment_name, +model_version, model_project, password) -> bytes`. + +1. **Password strength** — `crypto.validate_password_strength`: at least 12 characters, one + uppercase, one lowercase, one digit, one symbol. **Export-side only.** The importer cannot + re-check it and does not try: by the time a bundle exists, the password is whatever it was. +2. **Exportability** — `bundle.validate_run_is_exportable`: the run's top-level artifacts must + include **both** `data_model` and `prediction_model`, checked with `client.list_artifacts(run_id)` + before anything is downloaded. +3. **Whole-tree download** — `client.download_artifacts(run_id, '', staging)` into a + `tempfile.mkdtemp(prefix='sientia_export_')` directory. The entire artifact tree, not a subset. +4. **Run parameters** — `dict(run.data.params)`, copied **verbatim**. Whatever the run logged is + what the bundle carries, including `target_variable` (see §3). +5. **Metadata document** — `bundle.build_metadata_json`, exactly two top-level keys: + - `parameters`: the run parameters, verbatim; + - `metadata`: `model_name`, `experiment_name`, `run_id`, `model_version`, `model_project` and + `export_timestamp = datetime.now(UTC).isoformat()`. Six fields, no others. +6. **Zip** — `bundle.build_zip`, `zipfile.ZIP_DEFLATED`. `metadata.json` is written **first**, with + `json.dumps(...)` and no indent. Every file under the staging directory follows as + `artifacts/`, with `\` replaced by `/`. **No directory entries are stored**, so a reader + must derive directories from file paths. +7. **Encryption** — `crypto.encrypt_archive`: a fresh 16-byte `os.urandom` salt, Argon2id key + derivation, then a libsodium `secretstream` push in 64 KiB plaintext chunks, the last chunk tagged + `FINAL` and the rest `MESSAGE`. +8. **Download name** — `{model_name}_v{model_version}.sientia`. The staging directory is removed in a + `finally`, and the password is popped from the Streamlit session state. + +Producer pins: `pynacl==1.5.0`, `mlflow==2.10.1`. + +## 2. Constants + +Copied from `crypto.py` unless noted. The reader's names are the ones in +`laborious/utils/bundle/format.py`. + +| Producer | Value | Reader | +|---|---|---| +| `MAGIC` | `b'SIENTIA1'` | `MAGIC` | +| `_HEADER_STRUCT` | `struct.Struct('>8sBBB16sQQB32s64s')` | `HEADER_STRUCT` | +| `HEADER_STRUCT_SIZE` | `140` | `HEADER_SIZE` | +| `FORMAT_VERSION` | `1` | `FORMAT_VERSION` | +| `AEAD_ID_XCHACHA20POLY1305_SECRETSTREAM` | `1` | same name | +| `KDF_ID_ARGON2ID` | `1` | same name | +| `SALT_SIZE` | `16` | `SALT_SIZE` | +| `KDF_MEMLIMIT_BYTES` | `268435456` (256 MiB) | `MAX_KDF_MEMLIMIT` — a **ceiling** on the reader's side | +| `KDF_OPSLIMIT` | `3` | `MAX_KDF_OPSLIMIT` — likewise a ceiling | +| `KDF_PARALLELISM_RESERVED` | `1` | `PARALLELISM` | +| `DIGEST_SIZE` | `32` | `DIGEST_SIZE` | +| `RESERVED_SIGNATURE_SIZE` | `64` | `RESERVED_SIGNATURE_SIZE` | +| `CHUNK_SIZE` | `65536` (64 KiB) | `CHUNK_SIZE` | +| KDF algorithm | `crypto_pwhash_ALG_ARGON2ID13` | `KDF_ALG` | +| Key length | `crypto_secretstream_xchacha20poly1305_KEYBYTES` (32) | `KEY_SIZE` | + +Fixed by libsodium rather than by the producer, but needed to frame the stream: + +- `crypto_secretstream_xchacha20poly1305_HEADERBYTES == 24` → `STREAM_HEADER_SIZE` +- `crypto_secretstream_xchacha20poly1305_ABYTES == 17` → `ABYTES` +- pre-upload prefix `140 + 24 + (65536 + 17) = 65717` → `PREFIX_SIZE`, the smallest number of leading + bytes from which a header and one full chunk can be checked + +**Password normalisation.** `_derive_key` applies `unicodedata.normalize('NFC', password)` and +encodes UTF-8 — no trimming, no case folding, no padding. The reader normalises identically, so a +password typed in decomposed form opens a bundle written from the composed form. + +## 3. `target_variable`, and where it does not come from + +Gate 6 requires `parameters.target_variable`: it is the only source of `model_config.target` in the +model document, and every workflow that serves a model indexes that key directly. On the producer +side: + +| Export path | Logs `target_variable`? | Evidence | +|---|---|---| +| Time-series training template | yes | `app/pages/template_01.py` L1212-1214 | +| Model manager, "Save Experiment" | yes, if the session carried it | `app/pages/model_manager.py` L791-796 re-logs `st.session_state.parameters` verbatim | +| Model ensemble | same condition | `app/pages/model_ensemble.py` L695-700 | +| Pipeline / timeseries template | **no** | `app/pages/timeseries_template.py` L1761-1762, L1811-1812 log `run_metadata` only | +| AutoML | **no** | `automl_logging_operations.py` L454-491; the target appears as `"target"` inside the model card JSON, never as a run parameter | + +So a gate-6 rejection naming `target_variable` means the origin run came from the pipeline or AutoML +path. The remedy is to re-export from a run trained by the time-series template, which is what the +importer's `IMPORT_BUNDLE_INCOMPLETE` sentence already asks for. + +--- + +## 4. The file on disk + +``` +offset width field constraint the reader enforces +------ ----- -------------------- ------------------------------------------------------ + 0 8 magic == b'SIENTIA1' + 8 1 format_version == 1 + 9 1 aead_id == 1 (XChaCha20-Poly1305 secretstream) + 10 1 kdf_id == 1 (Argon2id) + 11 16 salt passed to the KDF as-is + 27 8 kdf_memlimit_bytes <= 268435456 + 35 8 kdf_opslimit <= 3 + 43 1 kdf_parallelism == 1 + 44 32 ciphertext_digest == sha256(file[164:]) + 76 64 reserved_signature all zero +------ ----- + 140 24 secretstream header libsodium HEADERBYTES + 164 ... chunk stream 64 KiB plaintext chunks, each +17 bytes of AEAD tag +``` + +All integers are big-endian; the format string starts with `>`, so there is no alignment padding and +the offsets above are exact. + +**The digest boundary.** The producer computes `hashlib.sha256(ciphertext)` where `ciphertext` is the +concatenation of the *pushed chunks only*, and then writes `header + stream_header + ciphertext`. On +disk, therefore, the digest covers everything after offset **164**, not after 140: + +> `ciphertext_digest == sha256(file[HEADER_SIZE + STREAM_HEADER_SIZE:])` + +A reader that hashes from 140 will reject every valid bundle. The fixture test that recomputes both +values and asserts the first matches while the second does not is what pins this, and is the reason +the golden fixture exists. + +**Version policy.** `format_version` is checked for equality, not for a range: a version this reader +does not know is refused at gate 2 rather than parsed optimistically. `reserved_signature` is +reserved for a future signature and must be all zero today, so a signed bundle is a new format +version and not a silent change of meaning. + +## 5. The seven gates + +`BundleReader.open()` runs them in this order, in one function, so the order cannot be changed by a +caller. Each raises `BundleGateError(gate=n)`, which the activity layer maps to an import step. + +| Gate | Step recorded | What it checks | Written to disk | +|---|---|---|---| +| 1 | `download` | object size ceiling, key prefix, `.sientia` suffix — from `stat_object`, before the body is fetched; then the SHA-256 of the downloaded bytes against the digest the uploader computed | the downloaded object only | +| 2 | `download` | the whole header table of §4, including the chunk-stream digest | nothing | +| 3 | `decryption` | Argon2id derivation with the header's parameters, then `secretstream` pull. The AEAD is the integrity check: a wrong password and an altered file fail identically, and the message says so | the decrypted zip, in the work directory | +| 4 | `archive_inspection` | central directory only: entry count, total uncompressed size, compression ratio, absolute names, `..` components, symlink and non-regular modes | nothing | +| 5 | `extraction` | extraction into a **fresh** directory (a pre-existing one is a failure), every member's `realpath` contained in it, no overwrite of a pre-existing file; a rejection removes its own partial tree | the extracted tree | +| 6 | `structure_validation` | `metadata.json` + `artifacts/` layout, exactly two top-level keys, all six metadata fields present and non-blank, `experiment_name` between 3 and 50 characters, `parameters.target_variable` present and non-blank, and `prediction_model`/`data_model` each holding `MLmodel` and `model.pkl` | nothing | +| 7 | `content_policy` | every file is either a known name (`MLmodel`, `metadata.json`, `model.pkl`, `conda.yaml`, `python_env.yaml`, `requirements.txt`, `model_card.json`, `model_card.svg`) or a known suffix (`.csv .json .yaml .yml .txt .pkl .svg .md`) | nothing | + +Gates 1 and 2 share the `download` step deliberately: an oversized object, a wrong magic, an +unsupported version and a digest that disagrees all say *these are not the bytes that were sent*, and +none of them may borrow the `decryption` sentence, which implicates the password. + +The `experiment_name` bound in gate 6 is not cosmetic: the import record's `experiment_name` column is +`VARCHAR(50)`, and the name is written onto the record before provisioning starts. Enforcing the +column's limit at the gate is what keeps it from surfacing three activities later as a constraint +violation. + +## 6. Ceilings and where they come from + +Gate 1 and gate 4 read their limits from `build_import_config()`, so a deployment can tighten them: + +| Variable | Default | Gate | +|---|---|---| +| `IMPORT_MAX_OBJECT_BYTES` | 1 GiB | 1 | +| `IMPORT_MAX_ARCHIVE_ENTRIES` | 5000 | 4 | +| `IMPORT_MAX_UNCOMPRESSED_BYTES` | 4 GiB | 4 | +| `IMPORT_MAX_COMPRESSION_RATIO` | 200 | 4 | +| `IMPORT_BUNDLE_PREFIX` | `imported_models/` | 1 | + +The KDF ceilings are **not** configurable: they are the producer's own cost parameters, and a header +declaring more is a resource-exhaustion attempt, refused before any memory is allocated. + +## 7. Fixtures + +- `tests/fixtures/bundle/golden_model_v3.sientia` — emitted by the producer's own code in its venv, + with the password and producer commit recorded in `tests/fixtures/bundle/README.md`. Regenerate it + with `tests/fixtures/bundle/regenerate.py`. It pays the producer's real Argon2id cost (256 MiB, + 3 passes), so it is used where the real thing matters and not in every test. +- `tests/helpers/bundle_factory.py` — the test-only writer. Same header fields at the same offsets, + but 8 MiB and 1 pass, and with a knob for every hostile variant: wrong magic, unknown version, KDF + parameters above the ceiling, digest mismatch, tampered ciphertext, truncated final chunk, non-zero + reserved signature, zip bomb, `..` entry, absolute entry, symlink, missing `data_model`, missing + `MLmodel`, missing `model.pkl`, malformed metadata, missing `target_variable`, extra `.sh` file. A + test asserts its header bytes are structurally identical to the golden fixture's. diff --git a/e2e/conftest.py b/e2e/conftest.py index c9c4504..878a114 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -33,6 +33,11 @@ from laborious.workflows.predictions_batch import PredictionsBatch from laborious.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction from laborious.workflows.sub_workflows.prediction_process import PredictionProcess +# The `.sientia` import fixtures live in their own module rather than here: four containers' worth +# of setup, an MLflow server subprocess and an ImportModel worker would double this file. Every +# fixture it adds is `import_`-prefixed, so nothing below changes meaning. +pytest_plugins = ['e2e.import_harness'] + # Test constants TEST_MONGODB_CONNECTION_STRING = 'mongodb://localhost:27017' TEST_DATABASE_NAME = 'test_db' @@ -60,8 +65,12 @@ def postgres_container(): This fixture creates a real PostgreSQL database in a Docker container that will be used for all tests in the session. + + `log_statement=all` is the one thing the import suite needs from this fixture: proving an import + issues no statement at all against the predictions database requires a log that would have + shown one. It changes what the server writes to its own log and nothing a test observes. """ - postgres = PostgresContainer('postgres:15') + postgres = PostgresContainer('postgres:15').with_command('postgres -c log_statement=all') postgres.start() yield postgres postgres.stop() diff --git a/e2e/fixtures/experiment_run.sql b/e2e/fixtures/experiment_run.sql index d43cfd1..524f392 100644 --- a/e2e/fixtures/experiment_run.sql +++ b/e2e/fixtures/experiment_run.sql @@ -1,4 +1,4 @@ --- public.experiment_run as migrations V1 … V12 leave it, for the e2e Postgres container. +-- public.experiment_run as migrations V1 … V11 leave it, for the e2e Postgres container. -- -- Source of truth: sientia-dataops-database-migrations, -- postgres/sientia-core-mlops-bff/migrations/ — this file mirrors the *final state* of: @@ -14,7 +14,11 @@ -- V9 remove_file_upload_statuses (status vocabulary) -- V10 remove_unused_statuses (status vocabulary) -- V11 add_run_type_to_experiment_run (run_type + reindexed) --- V12 add_import_statuses_to_experiment_run (IMPORT_SUCCESS / IMPORT_ERROR + pairing) +-- +-- V12 (add_import_statuses_to_experiment_run) lives in `experiment_run_v12.sql`, applied on top of +-- this file. It is separate because one e2e test — the evidence for why the migration gates +-- enabling the import worker — has to run against a table that does *not* have it, and a second +-- whole-schema copy with V12 removed would rot the moment either copy were edited. -- -- The same drift trade already accepted for the bundle fixtures: this is a copy, and the tests -- that read it are what make the copy auditable. If a migration lands there, update this file and @@ -48,19 +52,13 @@ CREATE TABLE IF NOT EXISTS public.experiment_run ( CONSTRAINT chk_experiment_run_file_name_pattern CHECK (file_name ~* '^[a-zA-Z0-9 /._-]+$'), - -- V12: the two shared stage values plus the training and import terminals. + -- The V1…V11 status vocabulary: the two shared stage values plus the training terminals. + -- V12 adds IMPORT_SUCCESS and IMPORT_ERROR to this list. CONSTRAINT chk_experiment_run_status CHECK (status IN ('PENDING', 'ORCHESTRATOR_REQUEST_ERROR', 'ORCHESTRATOR_WAITING_PROC', 'ORCHESTRATOR_VALIDATION_ERROR', - 'TRAINING_SUCCESS', 'TRAINING_ERROR', - 'IMPORT_SUCCESS', 'IMPORT_ERROR')), - CONSTRAINT chk_experiment_run_run_type CHECK (run_type IN ('TRAIN', 'IMPORT')), - - -- V12: each pair of terminal values belongs to its own run_type. The shared values stay out. - CONSTRAINT chk_experiment_run_terminal_status_run_type CHECK ( - (run_type = 'IMPORT' OR status NOT IN ('IMPORT_SUCCESS', 'IMPORT_ERROR')) - AND (run_type = 'TRAIN' OR status NOT IN ('TRAINING_SUCCESS', 'TRAINING_ERROR')) - ) + 'TRAINING_SUCCESS', 'TRAINING_ERROR')), + CONSTRAINT chk_experiment_run_run_type CHECK (run_type IN ('TRAIN', 'IMPORT')) ); CREATE INDEX IF NOT EXISTS idx_experiment_run_user_exp_status_date_type diff --git a/e2e/fixtures/experiment_run_v12.sql b/e2e/fixtures/experiment_run_v12.sql new file mode 100644 index 0000000..ee224e5 --- /dev/null +++ b/e2e/fixtures/experiment_run_v12.sql @@ -0,0 +1,36 @@ +-- V12 add_import_statuses_to_experiment_run, as a delta applied on top of `experiment_run.sql`. +-- +-- Source of truth: sientia-dataops-database-migrations, +-- postgres/sientia-core-mlops-bff/migrations/V12__add_import_statuses_to_experiment_run.sql +-- +-- Two things, and nothing else: +-- +-- 1. the status vocabulary gains IMPORT_SUCCESS and IMPORT_ERROR; +-- 2. chk_experiment_run_terminal_status_run_type pairs each terminal value with its own +-- run_type, so a TRAIN row can never end IMPORT_SUCCESS and an IMPORT row can never end +-- TRAINING_ERROR. The two shared stage values (PENDING, ORCHESTRATOR_WAITING_PROC) stay out +-- of the pairing — both run types legitimately pass through them. +-- +-- This file is applied separately because one e2e test runs an import against a table *without* +-- it: that failure shape — provisioned, then unrecordable, refused on the first attempt — is the +-- evidence for why this migration has to land before IMPORT_MODEL_ENABLED is turned on anywhere. + +ALTER TABLE public.experiment_run + DROP CONSTRAINT IF EXISTS chk_experiment_run_status; + +ALTER TABLE public.experiment_run + ADD CONSTRAINT chk_experiment_run_status CHECK (status IN ('PENDING', + 'ORCHESTRATOR_REQUEST_ERROR', + 'ORCHESTRATOR_WAITING_PROC', + 'ORCHESTRATOR_VALIDATION_ERROR', + 'TRAINING_SUCCESS', 'TRAINING_ERROR', + 'IMPORT_SUCCESS', 'IMPORT_ERROR')); + +ALTER TABLE public.experiment_run + DROP CONSTRAINT IF EXISTS chk_experiment_run_terminal_status_run_type; + +ALTER TABLE public.experiment_run + ADD CONSTRAINT chk_experiment_run_terminal_status_run_type CHECK ( + (run_type = 'IMPORT' OR status NOT IN ('IMPORT_SUCCESS', 'IMPORT_ERROR')) + AND (run_type = 'TRAIN' OR status NOT IN ('TRAINING_SUCCESS', 'TRAINING_ERROR')) + ); diff --git a/e2e/import_harness.py b/e2e/import_harness.py new file mode 100644 index 0000000..4589317 --- /dev/null +++ b/e2e/import_harness.py @@ -0,0 +1,950 @@ +"""Fixtures for the `.sientia` model-import e2e suite. Registered from `e2e/conftest.py`. + +**Why this is a module of its own.** `e2e/conftest.py` is shared by every existing e2e suite and +long enough already; four containers' worth of setup, an MLflow server subprocess, the import +environment variables and an `ImportModel` worker do not belong in it. Every fixture here is +prefixed `import_`, so nothing collides with the predictions harness, and no existing fixture is +edited — the one exception being `log_statement=all` on the shared `postgres_container`, which the +absence assertions need and which changes no test's behaviour. + +**The fidelity contract, which is the whole point of this file.** On the import path *nothing is +mocked*: + +| Dependency | Here | Why real | +|---|---|---| +| MinIO | the shared `minio_container`, plus an import bucket | gate 1 reads a real object | +| BFF Postgres | a container of its own, `sientia-core-mlops-bff` | the real `CHECK` constraints are half of what is under test, and two tests stop the server | +| MongoDB | a container, `models` seeded with gapped ids and a unique index on `id` | `max(int(id)) + 1` against a real unique index | +| MLflow | `mlflow server` from *this* venv, sqlite-backed, artifacts proxied | experiment/run/artifact/registry semantics are what the provisioning activities are tested for | +| the crypto | the production reader, real Argon2id, real `secretstream` | a bundle the producer wrote must open | + +What stays stubbed, deliberately: `CoreNotificationHandler`'s Mongo client keeps the existing +`mock_mongo_client`. Notifications are a side channel this suite counts, not a dependency it tests, +and a real notification collection would add a second write path to every absence assertion for +nothing. + +The import `Activities` is **never** composed with `patch_mlflow`, `patch_minio_repository` or +`patch_create_engine`. `mlflow.set_tracking_uri` is process-global, so a leak would silently turn a +real assertion into an assertion about a `MagicMock`; `import_activities` asserts the live tracking +URI names the test server before it hands the object over. + +**Bundles come from the unit suite.** `tests/helpers/bundle_factory.py` writes them and +`tests/fixtures/bundle/golden_model_v3.sientia` is the producer-emitted one. No third mirrored copy +of the producer's crypto is introduced here. + +**Isolation** (the containers are session-scoped): per test the status table is rebuilt, the `models` +collection is dropped and reseeded, and the import prefix is emptied. MLflow state is *not* reset — +experiment and model names are made unique per test instead, because deleted registered models +leave tombstones `get_registered_model` still finds, which would make "second import of the same +model" unreadable. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import json +import os +import signal +import socket +import subprocess +import sys +import time +import uuid +from collections.abc import Callable, Iterator +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import mlflow +import pytest +import pytest_asyncio +import requests +from pymongo import MongoClient +from sqlalchemy import Engine, create_engine, text +from temporalio.client import WorkflowHandle +from temporalio.worker import Worker +from testcontainers.mongodb import MongoDbContainer +from testcontainers.postgres import PostgresContainer + +from laborious.activities.activities import Activities +from laborious.workflows.import_model import ImportModel +from tests.helpers.bundle_factory import ( + DEFAULT_ENVELOPE_KEY, + DEFAULT_PASSWORD, + ZipSpec, + build_envelope, + build_metadata_document, + default_tree, + envelope_key_b64, + make_bundle, +) + +# --- constants ----------------------------------------------------------------------------------- + +REPO_ROOT = Path(__file__).resolve().parent.parent +FIXTURES_DIR = Path(__file__).resolve().parent / 'fixtures' +STATUS_TABLE_DDL = FIXTURES_DIR / 'experiment_run.sql' +STATUS_TABLE_DDL_V12 = FIXTURES_DIR / 'experiment_run_v12.sql' +GOLDEN_BUNDLE = REPO_ROOT / 'tests' / 'fixtures' / 'bundle' / 'golden_model_v3.sientia' +GOLDEN_BUNDLE_PASSWORD = 'Import1Model!Fixture' + +BFF_DATABASE = 'sientia-core-mlops-bff' +BFF_USER = 'bff' +BFF_PASSWORD = 'bff' + +# Lowercase and hyphens only: `chk_experiment_run_bucket_name_pattern` on the real table. +IMPORT_BUCKET = 'sientia-import' +IMPORT_PREFIX = 'imported_models/' +MODELS_COLLECTION = 'models' +MONGO_DATABASE = 'sientia' +IMPORT_TASK_QUEUE = 'import-model-e2e-queue' + +# Deliberately gapped, and deliberately strings: the baseline migration stores `id` as a string and +# `max(int(id)) + 1` must not be confused with a count or with a lexicographic maximum ("9" > "30"). +SEEDED_MODEL_IDS = ('1', '2', '7') + +MLFLOW_HEALTH_TIMEOUT_SECONDS = 90.0 +MLFLOW_HEALTH_POLL_SECONDS = 0.5 + +IMPORT_ACTIVITY_NAMES = ( + 'claim_import_status', + 'record_import_status', + 'record_import_names', + 'record_import_terminal_status', + 'report_import_status_write_failure', + 'download_import_bundle', + 'open_import_bundle', + 'create_import_experiment', + 'upload_import_artifacts', + 'register_import_model_version', + 'write_import_model_document', + 'cleanup_import_files', + 'ensure_import_bundle_retention', +) + + +# --- the BFF Postgres container ------------------------------------------------------------------ + + +@pytest.fixture(scope='session') +def import_bff_container() -> Iterator[PostgresContainer]: + """The import log database, on a server of its own. + + Separate from the shared `postgres_container` for two independent reasons, either sufficient: + the retry-recovery tests **stop and start this server**, which must not disturb the predictions + database every other e2e suite shares session scope with; and "the import touches no Postgres + database other than `sientia-core-mlops-bff`" is a stronger statement when the other database + lives on a different server. + + `log_statement=all` is what makes the absence assertions possible: the server log is the only + vantage point that sees every client, including the psycopg pool `Storage` holds. + """ + container = PostgresContainer( + 'postgres:15', + username=BFF_USER, + password=BFF_PASSWORD, + dbname=BFF_DATABASE, + ).with_command('postgres -c log_statement=all') + container.start() + try: + yield container + finally: + container.stop() + + +@pytest.fixture(scope='session') +def import_bff_engine(import_bff_container: PostgresContainer) -> Iterator[Engine]: + """A SQLAlchemy engine on the BFF container. + + `pool_pre_ping` matters here rather than being habit: two tests stop the container underneath + this engine, and a stale pooled connection would fail the *fixture* instead of the assertion. + """ + engine = create_engine(import_bff_container.get_connection_url(), pool_pre_ping=True) + try: + yield engine + finally: + engine.dispose() + + +def _apply_ddl(engine: Engine, files: tuple[Path, ...]) -> None: + """Drop `public.experiment_run` and rebuild it from `files`, in order.""" + with engine.begin() as connection: + connection.execute(text('DROP TABLE IF EXISTS public.experiment_run CASCADE')) + for path in files: + # One transaction per file, because each file is one migration's worth of statements. + with engine.begin() as connection: + connection.execute(text(path.read_text(encoding='utf-8'))) + + +@pytest.fixture +def import_status_table(import_bff_engine: Engine) -> Engine: + """`public.experiment_run` as `V1`…`V12` leave it, rebuilt for this test. + + Rebuilt rather than truncated: the pre-`V12` variant below has to be able to alternate with + this one inside a single session, and `TRUNCATE` cannot take a constraint away. The effect a + test sees is the one the spec asks for — the table holds only the row that test inserted. + """ + _apply_ddl(import_bff_engine, (STATUS_TABLE_DDL, STATUS_TABLE_DDL_V12)) + return import_bff_engine + + +@pytest.fixture +def import_status_table_pre_v12(import_bff_engine: Engine) -> Engine: + """The same table without `V12`: `run_type` present, the import statuses and pairing absent. + + There is no second copy of the base schema — this applies the same file, and simply does not + apply the delta. + """ + _apply_ddl(import_bff_engine, (STATUS_TABLE_DDL,)) + return import_bff_engine + + +InsertStatusRow = Callable[..., int] + + +@pytest.fixture +def import_insert_status_row(import_bff_engine: Engine) -> InsertStatusRow: + """Insert one import log row the way the BFF will, and return its `id`. + + Defaults mirror what the frontend sends: `PENDING`, `run_type='IMPORT'`, the object key in + `file_name`, and the uploader's digest inside `request_data` — which is where + `claim_import_status` looks for it. + """ + + def insert( + *, + object_key: str, + expected_digest: str | None = None, + username: str = 'import.e2e@example.com', + experiment_name: str = 'import-e2e-placeholder', + status: str = 'PENDING', + run_type: str = 'IMPORT', + bucket_name: str = IMPORT_BUCKET, + request_data: dict[str, Any] | None = None, + orchestrator_response_data: dict[str, Any] | None = None, + ) -> int: + payload = request_data if request_data is not None else {} + if expected_digest is not None and request_data is None: + payload = {'expected_digest': expected_digest} + now = datetime.now(UTC).replace(tzinfo=None) + statement = text( + 'INSERT INTO public.experiment_run ' + '(experiment_name, username, status, created_at, updated_at, bucket_name, file_name, ' + ' request_data, orchestrator_response_data, run_type) ' + 'VALUES (:experiment_name, :username, :status, :created_at, :updated_at, ' + ' :bucket_name, :file_name, CAST(:request_data AS JSONB), ' + ' CAST(:detail AS JSONB), :run_type) ' + 'RETURNING id' + ) + with import_bff_engine.begin() as connection: + return int( + connection.execute( + statement, + { + 'experiment_name': experiment_name, + 'username': username, + 'status': status, + 'created_at': now, + 'updated_at': now, + 'bucket_name': bucket_name, + 'file_name': object_key, + 'request_data': json.dumps(payload), + 'detail': ( + json.dumps(orchestrator_response_data) + if orchestrator_response_data is not None + else None + ), + 'run_type': run_type, + }, + ).scalar_one() + ) + + return insert + + +ReadStatusRow = Callable[[int], dict[str, Any] | None] + + +@pytest.fixture +def import_read_status_row(import_bff_engine: Engine) -> ReadStatusRow: + """Read one import log row as a plain mapping, or `None` when it is gone.""" + + def read(import_run_id: int) -> dict[str, Any] | None: + with import_bff_engine.begin() as connection: + row = ( + connection.execute( + text('SELECT * FROM public.experiment_run WHERE id = :id'), + {'id': import_run_id}, + ) + .mappings() + .one_or_none() + ) + return dict(row) if row is not None else None + + return read + + +# --- MongoDB ------------------------------------------------------------------------------------ + + +@pytest.fixture(scope='session') +def import_mongo_container() -> Iterator[MongoDbContainer]: + """A real MongoDB for the `models` collection the import writes its document into.""" + container = MongoDbContainer('mongo:7') + container.start() + try: + yield container + finally: + container.stop() + + +@pytest.fixture(scope='session') +def import_mongo_client(import_mongo_container: MongoDbContainer) -> Iterator[MongoClient]: + """A client on the Mongo container, for seeding and for reading assertions back.""" + client: MongoClient = MongoClient(import_mongo_container.get_connection_url()) + try: + yield client + finally: + client.close() + + +@pytest.fixture +def import_models_collection(import_mongo_client: MongoClient) -> Any: + """The `models` collection, dropped and reseeded with gapped ids and a unique index on `id`. + + The index is the assertion's teeth: without it "the id continues the collection" and "a name + already present is left alone" would both pass against a collection that quietly took a + duplicate. + """ + collection = import_mongo_client[MONGO_DATABASE][MODELS_COLLECTION] + collection.drop() + collection.insert_many( + [ + { + 'id': model_id, + 'name': f'seeded_model_{model_id}', + 'active': True, + 'model_config': { + 'transform_flavor': 'sklearn', + 'predict_flavor': 'joblib', + 'retention_minutes': 60, + 'target': 'SE', + }, + } + for model_id in SEEDED_MODEL_IDS + ] + ) + collection.create_index('id', unique=True) + return collection + + +# --- MinIO -------------------------------------------------------------------------------------- + + +@pytest.fixture +def import_bucket(minio_container) -> Iterator[Any]: + """The import bucket in the shared MinIO container, with its prefix emptied for this test. + + The bucket is kept rather than recreated — a bucket carries the retention lifecycle rule the + import sets, and dropping it between tests would make that rule untestable. + """ + client = minio_container.get_client() + if not client.bucket_exists(IMPORT_BUCKET): + client.make_bucket(IMPORT_BUCKET) + _empty_prefix(client, IMPORT_BUCKET, IMPORT_PREFIX) + yield client + _empty_prefix(client, IMPORT_BUCKET, IMPORT_PREFIX) + + +def _empty_prefix(client: Any, bucket: str, prefix: str) -> None: + """Remove every object under `prefix`, so a leftover cannot satisfy the next test.""" + for obj in client.list_objects(bucket, prefix=prefix, recursive=True): + client.remove_object(bucket, obj.object_name) + + +UploadBundle = Callable[..., tuple[str, str]] + + +@pytest.fixture +def import_upload_bundle(import_bucket: Any) -> UploadBundle: + """Upload bundle bytes under the import prefix; return `(object_key, hex_digest)`. + + The digest is computed over the bytes handed in, which is what the frontend promises and what + gate 1 holds it to. A test that wants the digest to *disagree* passes its own. + """ + import io + + def upload(payload: bytes, name: str | None = None) -> tuple[str, str]: + object_name = name or f'e2e_{uuid.uuid4().hex}.sientia' + object_key = f'{IMPORT_PREFIX}{object_name}' + import_bucket.put_object( + IMPORT_BUCKET, + object_key, + io.BytesIO(payload), + length=len(payload), + content_type='application/octet-stream', + ) + return object_key, hashlib.sha256(payload).hexdigest() + + return upload + + +# --- building bundles ---------------------------------------------------------------------------- + + +@pytest.fixture +def import_envelope() -> Callable[[str], str]: + """Wrap a bundle password the way the frontend will, under the key the worker is configured with. + + The key is `bundle_factory.DEFAULT_ENVELOPE_KEY`, which `import_activities` puts in + `IMPORT_PASSWORD_KEY` — so an envelope built here is one the worker can actually open, and an + envelope built under any other key is a decryption-step failure, which is a test of its own. + """ + + def build(password: str = DEFAULT_PASSWORD) -> str: + return build_envelope(password, DEFAULT_ENVELOPE_KEY) + + return build + + +@pytest.fixture +def import_bundle_factory(import_names: dict[str, str]) -> Callable[..., bytes]: + """Build a `.sientia` bundle carrying *this test's* unique model and experiment names. + + Every knob the gate tests need is passed straight through to the shared factory — there is no + second implementation of the producer's header, KDF or framing here, which is the whole point of + reusing `tests/helpers/bundle_factory.py`. + + The names default to `import_names` so two tests provisioning models in the same session never + observe each other's registry state (MLflow is deliberately not reset — design E13). A test that + *wants* the collision, like the second-import case, passes the same names twice. + """ + + def build( + *, + password: str = DEFAULT_PASSWORD, + metadata: dict[str, Any] | None = None, + parameters: dict[str, Any] | None = None, + drop_metadata_field: str | None = None, + drop_top_level_key: str | None = None, + drop_parameter: str | None = None, + files: dict[str, bytes] | None = None, + include_metadata: bool = True, + symlinks: dict[str, str] | None = None, + absolute_entry: str | None = None, + traversal_entry: str | None = None, + bomb_bytes: int = 0, + extra_files: dict[str, bytes] | None = None, + **header_kwargs: Any, + ) -> bytes: + document = build_metadata_document( + metadata={ + 'model_name': import_names['model_name'], + 'experiment_name': import_names['experiment_name'], + **(metadata or {}), + }, + parameters={'model_name': import_names['model_name'], **(parameters or {})}, + drop_metadata_field=drop_metadata_field, + drop_top_level_key=drop_top_level_key, + drop_parameter=drop_parameter, + ) + spec = ZipSpec( + files=default_tree() if files is None else files, + document=document, + include_metadata=include_metadata, + symlinks=symlinks or {}, + absolute_entry=absolute_entry, + traversal_entry=traversal_entry, + bomb_bytes=bomb_bytes, + extra_files=extra_files or {}, + ) + return make_bundle(spec, password, **header_kwargs) + + return build + + +# --- the MLflow tracking server ----------------------------------------------------------------- + + +def _free_port() -> int: + """A loopback port nothing is listening on, taken from an ephemeral bind.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(('127.0.0.1', 0)) + return int(sock.getsockname()[1]) + + +@pytest.fixture(scope='session') +def import_mlflow_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[dict[str, Any]]: + """`mlflow server` from *this* venv, on a free loopback port. + + A subprocess rather than a container because the venv pin **is** the production pin (2.10.1, + the same version the producer pins), so this tests the exact client/server pair that will run + in production — no image tag to drift out of step with `requirements.txt`. + + sqlite for the backend store because the model registry is not available on a file-backed + tracking store, and `create_model_version` is half of what the provisioning activities do. + `--serve-artifacts` sends `log_artifacts` over HTTP through the server, as it goes in + production, while leaving the uploaded tree readable straight off the local filesystem. + + Yields the host, the port and the artifacts directory. A server that never answers `/health` + fails *here*, with its own stderr attached, rather than making the first test fail obscurely. + """ + root = tmp_path_factory.mktemp('import_mlflow') + artifacts = root / 'mlartifacts' + artifacts.mkdir() + log_path = root / 'mlflow-server.log' + port = _free_port() + host = '127.0.0.1' + tracking_uri = f'http://{host}:{port}' + + server: dict[str, Any] = { + 'host': f'http://{host}', + 'port': str(port), + 'tracking_uri': tracking_uri, + 'artifacts_dir': artifacts, + 'root': root, + 'log_path': log_path, + 'process': _launch_mlflow(root, host, port, artifacts, log_path), + } + try: + _wait_for_mlflow(server['process'], tracking_uri, log_path) + yield server + finally: + _terminate_group(server['process']) + + +def _launch_mlflow( + root: Path, host: str, port: int, artifacts: Path, log_path: Path +) -> subprocess.Popen: + """Start the server process. Separate from the fixture so one test can restart it. + + The sqlite file and the artifacts directory are reused across a restart on purpose: a test that + stops the server to fail one activity must not also erase what earlier tests provisioned. + """ + with log_path.open('ab') as log_handle: + return subprocess.Popen( # noqa: S603 - fixed argv, no shell, all values built here + [ + sys.executable, + '-m', + 'mlflow', + 'server', + '--host', + host, + '--port', + str(port), + '--backend-store-uri', + f'sqlite:///{root / "mlflow.db"}', + '--artifacts-destination', + str(artifacts), + '--serve-artifacts', + ], + stdout=log_handle, + stderr=subprocess.STDOUT, + cwd=str(root), + # Its own process group: `mlflow server` is gunicorn, so it has children, and killing + # only the parent would leave workers holding the port for the rest of the session. + start_new_session=True, + ) + + +@pytest.fixture +def import_mlflow_stopped(import_mlflow_server: dict[str, Any]) -> Any: + """A context manager that stops the tracking server and **always** brings it back. + + This is design E7's second preference in its purest form: no patch, no mock — the dependency is + genuinely gone, which is the thing that actually happens in production when a pod is replaced. + + The restart is in a `finally` because the server is session-scoped: a test that left it down + would fail every test after it, and the failure would point at the wrong place. + """ + + @contextlib.contextmanager + def stopped() -> Iterator[None]: + _terminate_group(import_mlflow_server['process']) + try: + yield + finally: + import_mlflow_server['process'] = _launch_mlflow( + import_mlflow_server['root'], + '127.0.0.1', + int(import_mlflow_server['port']), + import_mlflow_server['artifacts_dir'], + import_mlflow_server['log_path'], + ) + _wait_for_mlflow( + import_mlflow_server['process'], + import_mlflow_server['tracking_uri'], + import_mlflow_server['log_path'], + ) + + return stopped + + +def _wait_for_mlflow(process: subprocess.Popen, tracking_uri: str, log_path: Path) -> None: + """Poll `/health` until it answers, or fail with what the server actually said.""" + deadline = time.monotonic() + MLFLOW_HEALTH_TIMEOUT_SECONDS + while time.monotonic() < deadline: + if process.poll() is not None: + _terminate_group(process) + raise RuntimeError( + f'mlflow server exited with {process.returncode} before answering /health:\n' + f'{log_path.read_text(encoding="utf-8", errors="replace")}' + ) + try: + if requests.get(f'{tracking_uri}/health', timeout=2).status_code == 200: + return + except requests.RequestException: + pass + time.sleep(MLFLOW_HEALTH_POLL_SECONDS) + _terminate_group(process) + raise RuntimeError( + f'mlflow server did not answer /health within {MLFLOW_HEALTH_TIMEOUT_SECONDS}s:\n' + f'{log_path.read_text(encoding="utf-8", errors="replace")}' + ) + + +def _terminate_group(process: subprocess.Popen) -> None: + """Stop the server and every gunicorn worker it forked.""" + if process.poll() is not None: + return + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(os.getpgid(process.pid), signal.SIGTERM) + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(os.getpgid(process.pid), signal.SIGKILL) + process.wait(timeout=15) + + +# --- per-test naming ---------------------------------------------------------------------------- + + +@pytest.fixture +def import_names(request: pytest.FixtureRequest) -> dict[str, str]: + """Experiment and model names unique to this test. + + MLflow state is deliberately not reset between tests (design E13): deleting a registered model + leaves a tombstone `get_registered_model` still finds, which would make "second import of the + same model" read as a leak rather than as the thing it is testing. Unique names per test buy + the same isolation without the tombstones. + """ + token = uuid.uuid4().hex[:8] + stem = request.node.name.replace('[', '_').replace(']', '')[:24] + return { + 'token': token, + 'model_name': f'm_{stem}_{token}'[:40], + 'experiment_name': f'e_{stem}_{token}'[:40], + } + + +# --- the import Activities ---------------------------------------------------------------------- + + +@pytest.fixture +def import_limits_override() -> dict[str, str]: + """Extra import environment variables for this test, applied last so they win. + + Empty by default. A test that needs a ceiling it can actually cross — gate 1's object size is + 1 GiB, and uploading a gigabyte to prove a limit is not a test anybody runs twice — overrides + this fixture inside its own class, so the narrowed ceiling is scoped to the cases that need it + and every other test keeps the production defaults. + """ + return {} + + +@pytest_asyncio.fixture +async def import_activities( + monkeypatch: pytest.MonkeyPatch, + import_limits_override: dict[str, str], + tmp_path: Path, + postgres_container, + import_bff_container: PostgresContainer, + import_mongo_container: MongoDbContainer, + import_mlflow_server: dict[str, Any], + import_bucket: Any, + minio_container, + mock_logger, + notification_handler, +) -> Any: + """An `Activities` whose import path talks to nothing fake. + + Note what is *not* in the fixture's parameter list: `patch_mlflow`, `patch_minio_repository` + and `patch_create_engine`. Composing any of them would replace exactly the thing under test, + and because `mlflow.set_tracking_uri` is process-global the substitution would be silent — so + the live tracking URI is asserted below rather than assumed. + + The import configuration is built by the production `build_import_config` / + `build_import_status_config` from environment variables set here, so the variable *names* are + covered too and not just the dictionaries they produce. + """ + from laborious.utils.connectors_config import build_import_config, build_import_status_config + + minio_port = minio_container.get_exposed_port(9000) + + monkeypatch.setenv('IMPORT_PASSWORD_KEY', envelope_key_b64(DEFAULT_ENVELOPE_KEY)) + monkeypatch.setenv('IMPORT_BUNDLE_BUCKET', IMPORT_BUCKET) + monkeypatch.setenv('IMPORT_BUNDLE_PREFIX', IMPORT_PREFIX) + monkeypatch.setenv('IMPORT_MODELS_COLLECTION', MODELS_COLLECTION) + monkeypatch.setenv('IMPORT_WORK_DIR', str(tmp_path / 'import-work')) + monkeypatch.setenv('IMPORT_STATUS_DB_HOST', import_bff_container.get_container_host_ip()) + monkeypatch.setenv('IMPORT_STATUS_DB_PORT', str(import_bff_container.get_exposed_port(5432))) + monkeypatch.setenv('IMPORT_STATUS_DB_NAME', BFF_DATABASE) + monkeypatch.setenv('IMPORT_STATUS_DB_USER', BFF_USER) + monkeypatch.setenv('IMPORT_STATUS_DB_PASSWORD', BFF_PASSWORD) + for name, value in import_limits_override.items(): + monkeypatch.setenv(name, value) + + activities = Activities( + # The predictions database, which the import must never touch. It is pointed at a real + # server on purpose: "no statement reached it" is only worth asserting when there is a log + # that would have shown one. + postgres_config={ + 'host': 'localhost', + 'port': postgres_container.get_exposed_port(5432), + 'user': 'test', + 'password': 'test', + 'dbname': 'test', + 'min_connections': 1, + 'max_connections': 5, + }, + mlflow_config={ + 'host': import_mlflow_server['host'], + 'port': import_mlflow_server['port'], + 'username': 'e2e', + 'password': 'e2e', + }, + minio_config={ + 'endpoint_url': f'localhost:{minio_port}', + 'access_key': 'minioadmin', + 'secret_key': 'minioadmin', + 'default_bucket': IMPORT_BUCKET, + 'retention_hours': 24, + 'secure': False, + }, + opc_config={}, + pi_web_api_config={ + 'base_url': 'http://localhost:8080', + 'auth_type': 'bearer', + 'auth_token': 'e2e', + }, + logger=mock_logger, + notification_handler=notification_handler, + mongo_config={ + 'connection_string': import_mongo_container.get_connection_url(), + 'database_name': MONGO_DATABASE, + }, + import_config=build_import_config(), + import_status_config=build_import_status_config(), + ) + + # If a mocked-MLflow fixture ever leaks into an import test, this is where it stops. + assert mlflow.get_tracking_uri() == import_mlflow_server['tracking_uri'], ( + f'MLflow tracking URI is {mlflow.get_tracking_uri()!r}, not the test server ' + f'{import_mlflow_server["tracking_uri"]!r} — a mocked MLflow fixture has leaked in' + ) + + try: + yield activities + finally: + await activities.shutdown() + + +@pytest_asyncio.fixture +async def import_worker(temporal_test_env, import_activities: Any) -> Any: + """An `ImportModel` worker with the thirteen import activities, on the import queue.""" + async with Worker( + temporal_test_env.client, + task_queue=IMPORT_TASK_QUEUE, + workflows=[ImportModel], + activities=[getattr(import_activities, name) for name in IMPORT_ACTIVITY_NAMES], + ) as worker: + yield worker + + +# --- running an import -------------------------------------------------------------------------- + + +@pytest.fixture +def import_build_input() -> Callable[..., dict[str, Any]]: + """The workflow input, as the frontend assembles it.""" + + def build( + *, + import_run_id: int, + object_key: str, + expected_digest: str | None, + password_envelope: str, + bucket: str | None = IMPORT_BUCKET, + **extra: Any, + ) -> dict[str, Any]: + payload: dict[str, Any] = { + 'import_run_id': import_run_id, + 'object_key': object_key, + 'password_envelope': password_envelope, + } + if expected_digest is not None: + payload['expected_digest'] = expected_digest + if bucket is not None: + payload['bucket'] = bucket + payload.update(extra) + return payload + + return build + + +@pytest.fixture +def import_execute(temporal_test_env, import_worker: Any) -> Callable[..., Any]: + """Run one import to completion and hand back the handle, whatever the outcome. + + The handle rather than the result, because half this suite asserts on *how* a run failed — + the error type Temporal recorded, the event history — and `execute_workflow` would have thrown + that away. + """ + + async def execute(payload: dict[str, Any], workflow_id: str | None = None) -> WorkflowHandle: + handle = await temporal_test_env.client.start_workflow( + ImportModel.run, + payload, + id=workflow_id or f'import-e2e-{uuid.uuid4().hex}', + task_queue=IMPORT_TASK_QUEUE, + ) + with contextlib.suppress(Exception): + # The exception is the assertion's subject in the negative cases; the caller reads it + # back off the handle so a failure here is never mistaken for a harness error. + await handle.result() + return handle + + return execute + + +@pytest.fixture +def import_history_events() -> Callable[[WorkflowHandle], Any]: + """Every event of a run's history, flattened to the text the secrecy assertions scan. + + Returns `(events, blob)`: the event list for structural assertions, and one string holding + every payload in the history for "this value appears nowhere" assertions. + """ + + async def fetch(handle: WorkflowHandle) -> tuple[list[Any], str]: + events = [event async for event in handle.fetch_history_events()] + return events, '\n'.join(str(event) for event in events) + + return fetch + + +# --- reading a failure --------------------------------------------------------------------------- +# +# Temporal wraps what actually went wrong: `WorkflowFailureError` -> `ActivityError` -> +# `ApplicationError`, and only the innermost one carries the sentence and the declared error type. +# `str(exception)` on the outermost is "Workflow execution failed", which is true and useless, so +# every negative assertion in this suite goes through these two functions instead. + + +def failure_chain(error: BaseException) -> list[BaseException]: + """The exception and everything it was caused by, outermost first.""" + chain: list[BaseException] = [] + current: BaseException | None = error + while current is not None and not any(current is seen for seen in chain): + chain.append(current) + current = current.__cause__ + return chain + + +def failure_text(error: BaseException) -> str: + """Every link of the chain as `Type: message`, joined — what a substring assertion reads.""" + return '\n'.join(f'{type(part).__name__}: {part}' for part in failure_chain(error)) + + +def failure_types(error: BaseException) -> set[str]: + """Every Python class name in the chain, plus every `ApplicationError.type` it declares. + + The declared `type` is the one that survives serialisation into the event history, so it is + what "the run failed with the not-claimable error type" actually means. + """ + names: set[str] = set() + for part in failure_chain(error): + names.add(type(part).__name__) + declared = getattr(part, 'type', None) + if isinstance(declared, str): + names.add(declared) + return names + + +# --- the absence assertion ---------------------------------------------------------------------- + + +@pytest.fixture +def import_assert_nothing_provisioned( + import_mlflow_server: dict[str, Any], + import_models_collection: Any, + import_bucket: Any, + tmp_path: Path, +) -> Callable[..., None]: + """Assert a rejected import left nothing behind, in one call over all four places. + + One helper rather than four assertions per test, because "nothing was provisioned" is a single + claim and a test that checks three of its four halves reads as if it checked all four. + """ + + def assert_nothing( + *, + experiment_name: str, + model_name: str, + object_key: str | None = None, + seeded_ids: tuple[str, ...] = SEEDED_MODEL_IDS, + ) -> None: + client = mlflow.tracking.MlflowClient(tracking_uri=import_mlflow_server['tracking_uri']) + + experiment = client.get_experiment_by_name(experiment_name) + assert experiment is None, f'experiment {experiment_name!r} was created' + + versions = client.search_model_versions(f"name='{model_name}'") + assert not list(versions), f'registry holds versions of {model_name!r}' + + documents = list(import_models_collection.find({}, {'_id': False})) + assert {str(document['id']) for document in documents} == set(seeded_ids), ( + f'the models collection changed: {documents}' + ) + assert not [d for d in documents if d.get('name') == model_name], ( + f'a model document for {model_name!r} was written' + ) + + # The uploaded object itself stays — the frontend put it there and retention removes it. + # What must not be there is anything the *import* wrote under the prefix. + stored = { + obj.object_name + for obj in import_bucket.list_objects( + IMPORT_BUCKET, prefix=IMPORT_PREFIX, recursive=True + ) + } + assert stored <= ({object_key} if object_key else set()), ( + f'the import wrote objects under {IMPORT_PREFIX}: {stored}' + ) + + work_dir = tmp_path / 'import-work' + leftovers = list(work_dir.rglob('*')) if work_dir.exists() else [] + assert not leftovers, f'the worker filesystem still holds {leftovers}' + + return assert_nothing + + +# --- reading the golden fixture ----------------------------------------------------------------- + + +@pytest.fixture(scope='session') +def import_golden_bundle() -> dict[str, Any]: + """The producer-emitted bundle: its bytes, its digest and the password it was written with. + + This is the only place in the suite that pays the producer's real Argon2id cost (256 MiB, + 3 passes). Everything else runs the factory's cheap cost, which is the right split — the + reader honours whatever the header declares, up to its ceiling. + """ + payload = GOLDEN_BUNDLE.read_bytes() + return { + 'bytes': payload, + 'digest': hashlib.sha256(payload).hexdigest(), + 'password': GOLDEN_BUNDLE_PASSWORD, + 'path': GOLDEN_BUNDLE, + } diff --git a/e2e/test_import_harness_smoke.py b/e2e/test_import_harness_smoke.py new file mode 100644 index 0000000..579c9bf --- /dev/null +++ b/e2e/test_import_harness_smoke.py @@ -0,0 +1,196 @@ +"""Does the harness itself come up? Diagnosable separately from the suite that runs on it. + +Four containers, an `mlflow server` subprocess and a Temporal worker is a lot of moving parts, and +when one of them fails to start the first *real* test fails somewhere deep inside an activity with +a message about something else. These tests exist so that a broken harness says so in its own +words: each one checks one piece is alive and does nothing with the import path. + +The last test is the exception — it runs a real workflow, but with an input designed to be rejected +at the very first step, so what it proves is only that the wiring reaches the first activity. +""" + +from __future__ import annotations + +import json +from typing import Any + +import mlflow +import pytest +from sqlalchemy import text +from temporalio.client import WorkflowFailureError + +from e2e.import_harness import ( + BFF_DATABASE, + IMPORT_ACTIVITY_NAMES, + IMPORT_BUCKET, + IMPORT_PREFIX, + IMPORT_TASK_QUEUE, + SEEDED_MODEL_IDS, + failure_text, + failure_types, +) + +pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio] + + +class TestContainers: + """Each dependency is real, reachable, and the one the harness says it is.""" + + async def test_bff_container_holds_the_status_table(self, import_status_table: Any) -> None: + with import_status_table.begin() as connection: + database = connection.execute(text('SELECT current_database()')).scalar_one() + columns = { + row[0] + for row in connection.execute( + text( + 'SELECT column_name FROM information_schema.columns ' + "WHERE table_schema = 'public' AND table_name = 'experiment_run'" + ) + ) + } + assert database == BFF_DATABASE + assert {'id', 'status', 'run_type', 'request_data', 'orchestrator_response_data'} <= columns + + async def test_bff_container_logs_every_statement(self, import_bff_container: Any) -> None: + """`log_statement=all` — without it the absence assertions have no vantage point.""" + result = import_bff_container.exec( + ['psql', '-U', 'bff', '-d', BFF_DATABASE, '-tAc', 'SHOW log_statement'] + ) + assert b'all' in result.output + + async def test_the_v12_delta_is_what_separates_the_two_tables( + self, import_status_table_pre_v12: Any + ) -> None: + """The pre-`V12` table has `run_type` and refuses the import terminal statuses.""" + with import_status_table_pre_v12.begin() as connection: + constraints = { + row[0] + for row in connection.execute( + text( + 'SELECT conname FROM pg_constraint ' + "WHERE conrelid = 'public.experiment_run'::regclass" + ) + ) + } + columns = { + row[0] + for row in connection.execute( + text( + 'SELECT column_name FROM information_schema.columns ' + "WHERE table_schema = 'public' AND table_name = 'experiment_run'" + ) + ) + } + assert 'run_type' in columns + assert 'chk_experiment_run_terminal_status_run_type' not in constraints + + async def test_the_seeded_collection_has_gapped_ids_and_a_unique_index( + self, import_models_collection: Any + ) -> None: + ids = { + str(document['id']) for document in import_models_collection.find({}, {'_id': False}) + } + assert ids == set(SEEDED_MODEL_IDS) + unique_keys = { + tuple(index['key'].keys()) + for index in import_models_collection.list_indexes() + if index.get('unique') + } + assert ('id',) in unique_keys + + async def test_the_import_bucket_exists_and_its_prefix_is_empty( + self, import_bucket: Any + ) -> None: + assert import_bucket.bucket_exists(IMPORT_BUCKET) + assert not list( + import_bucket.list_objects(IMPORT_BUCKET, prefix=IMPORT_PREFIX, recursive=True) + ) + + async def test_uploading_a_bundle_returns_a_key_and_its_digest( + self, import_upload_bundle: Any, import_bucket: Any + ) -> None: + object_key, digest = import_upload_bundle(b'not a real bundle, just bytes') + assert object_key.startswith(IMPORT_PREFIX) + assert len(digest) == 64 + assert import_bucket.stat_object(IMPORT_BUCKET, object_key).size > 0 + + async def test_the_mlflow_server_answers_and_serves_a_registry( + self, import_mlflow_server: dict[str, Any] + ) -> None: + client = mlflow.tracking.MlflowClient(tracking_uri=import_mlflow_server['tracking_uri']) + experiment_id = client.create_experiment('harness_smoke_experiment') + run = client.create_run(experiment_id, run_name='harness-smoke-run') + client.create_registered_model('harness_smoke_model') + version = client.create_model_version( + 'harness_smoke_model', f'runs:/{run.info.run_id}/prediction_model', run.info.run_id + ) + assert version.version == '1' + assert version.current_stage in (None, 'None') + + +class TestActivitiesAndWorker: + """The object under test is built from the real dependencies, and the worker registers it.""" + + async def test_activities_point_at_the_real_dependencies( + self, import_activities: Any, import_mlflow_server: dict[str, Any] + ) -> None: + assert mlflow.get_tracking_uri() == import_mlflow_server['tracking_uri'] + assert import_activities.import_config['bucket'] == IMPORT_BUCKET + assert import_activities.import_config['prefix'] == IMPORT_PREFIX + assert import_activities.status_config['dbname'] == BFF_DATABASE + assert import_activities.mongo_repository is not None + # Not a mock: a MagicMock would satisfy any attribute access, so the type is the assertion. + assert type(import_activities.import_minio_repository).__name__ == 'MinioRepository' + + async def test_every_import_activity_is_registered(self, import_activities: Any) -> None: + for name in IMPORT_ACTIVITY_NAMES: + assert callable(getattr(import_activities, name)), f'{name} is missing' + + async def test_the_worker_polls_the_import_queue(self, import_worker: Any) -> None: + assert import_worker.task_queue == IMPORT_TASK_QUEUE + + async def test_a_workflow_input_reaches_the_first_activity( + self, + import_status_table: Any, + import_execute: Any, + import_build_input: Any, + import_read_status_row: Any, + ) -> None: + """The claim runs, and the row it could not claim proves the wiring is live. + + An id no row carries is the cheapest input that gets past workflow validation and into + `claim_import_status`, so a run that ends failed *there* means the client, the worker, the + workflow, the activity and the BFF connection are all in place. Nothing else is exercised. + """ + handle = await import_execute( + import_build_input( + import_run_id=987654321, + object_key=f'{IMPORT_PREFIX}nothing.sientia', + expected_digest='0' * 64, + password_envelope='not-a-real-envelope', + ) + ) + with pytest.raises(WorkflowFailureError) as failure: + await handle.result() + text_of_failure = failure_text(failure.value) + assert 'claim' in text_of_failure.lower() + assert '987654321' in text_of_failure + assert 'ImportLogRowNotClaimableError' in failure_types(failure.value) + assert import_read_status_row(987654321) is None + + async def test_the_insert_helper_writes_a_claimable_row( + self, + import_status_table: Any, + import_insert_status_row: Any, + import_read_status_row: Any, + ) -> None: + object_key = f'{IMPORT_PREFIX}claimable.sientia' + row_id = import_insert_status_row(object_key=object_key, expected_digest='a' * 64) + row = import_read_status_row(row_id) + assert row is not None + assert row['status'] == 'PENDING' + assert row['run_type'] == 'IMPORT' + assert row['file_name'] == object_key + request_data = row['request_data'] + payload = json.loads(request_data) if isinstance(request_data, str) else request_data + assert payload['expected_digest'] == 'a' * 64 diff --git a/e2e/test_import_model_digest.py b/e2e/test_import_model_digest.py new file mode 100644 index 0000000..adcc88b --- /dev/null +++ b/e2e/test_import_model_digest.py @@ -0,0 +1,214 @@ +"""The digest is a promise about the bytes in the bucket, and it is kept against the bucket. + +Three ways the promise can be broken, and where each is caught: + +- the uploader computed the digest over *other* bytes — caught at `download`, after the object is + fetched and before its header is parsed, so a wrong file never costs an Argon2id pass; +- the object was replaced between the upload and the run — the same failure, which is the point: + the check is against what is stored now, not against what was stored when the row was written; +- no digest at all — caught at `received`, before anything is downloaded, because a request with no + digest is not a request this workflow can honour. + +The first two matter because MinIO is real here. A mocked object store would return whatever the +test told it to, and "the digest is enforced against the bytes that were uploaded" would be a +statement about the mock. +""" + +from __future__ import annotations + +import hashlib +import io +import json +from typing import Any + +import pytest +from temporalio.client import WorkflowFailureError + +from e2e.import_harness import IMPORT_BUCKET, failure_text +from laborious.activities.model_import_errors import IMPORT_FAILURE_CATALOG +from laborious.utils.bundle.steps import ImportStep + +pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio] + +VALIDATION_ERROR_STATUS = 'ORCHESTRATOR_VALIDATION_ERROR' +FACTORY_PASSWORD = 'Factory1Model!Test' + + +def _as_dict(value: Any) -> dict[str, Any]: + return json.loads(value) if isinstance(value, str) else (value or {}) + + +def _assert_step(row: dict[str, Any], step: ImportStep, workflow_id: str) -> None: + code, reason = IMPORT_FAILURE_CATALOG[step] + detail = _as_dict(row['orchestrator_response_data'])['import'] + assert row['status'] == VALIDATION_ERROR_STATUS, row['status'] + assert detail['step'] == step.value, detail + assert detail['code'] == code.value, detail + assert detail['workflow_id'] == workflow_id + assert row['error_message'] == reason + + +class TestWrongDigest: + """A digest computed over bytes that are not the ones in the bucket.""" + + async def test_a_digest_from_another_file_fails_at_download( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + """The bundle is perfectly valid; only the digest is another file's. + + The row carries the same wrong digest as the input, so the request and the record agree and + the `received` guard passes — otherwise this would be a `received` failure about a + contradictory request instead of the download-step digest check it is meant to exercise. + """ + payload = import_bundle_factory() + other_digest = hashlib.sha256( + import_bundle_factory(metadata={'run_id': 'other'}) + ).hexdigest() + assert other_digest != hashlib.sha256(payload).hexdigest() + + object_key, _ = import_upload_bundle(payload) + import_run_id = import_insert_status_row( + object_key=object_key, expected_digest=other_digest + ) + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=other_digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + with pytest.raises(WorkflowFailureError) as failure: + await handle.result() + + row = import_read_status_row(import_run_id) + assert row is not None + _assert_step(row, ImportStep.DOWNLOAD, handle.id) + + # No header was parsed: the failure names the digest, not the format. If the digest check + # ran *after* the header the message here would be about magic bytes or a version. + text_of_failure = failure_text(failure.value).lower() + assert 'digest' in text_of_failure, text_of_failure + assert 'marker' not in text_of_failure and 'version' not in text_of_failure + + # The names are still unwritten, which is the other half of "no header was parsed": the + # bundle was never opened, so nothing knew what to call it. + assert row['run_name'] is None + assert row['experiment_name'] == 'import-e2e-placeholder' + + import_assert_nothing_provisioned( + experiment_name=import_names['experiment_name'], + model_name=import_names['model_name'], + object_key=object_key, + ) + + +class TestOverwrittenObject: + """The bytes changed after the row was written. Only a real object store can show this.""" + + async def test_replacing_the_object_after_upload_fails_at_download( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_bucket: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + original = import_bundle_factory() + object_key, digest = import_upload_bundle(original) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + + # Same key, different bytes — also a valid bundle, so nothing but the digest can tell. + replacement = import_bundle_factory(metadata={'run_id': 'ff00ff00ff00ff00ff00ff00ff00ff00'}) + assert hashlib.sha256(replacement).hexdigest() != digest + import_bucket.put_object( + IMPORT_BUCKET, + object_key, + io.BytesIO(replacement), + length=len(replacement), + content_type='application/octet-stream', + ) + + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + with pytest.raises(WorkflowFailureError): + await handle.result() + + row = import_read_status_row(import_run_id) + assert row is not None + _assert_step(row, ImportStep.DOWNLOAD, handle.id) + import_assert_nothing_provisioned( + experiment_name=import_names['experiment_name'], + model_name=import_names['model_name'], + object_key=object_key, + ) + + +class TestMissingDigest: + """No digest is refused at the door, before a single byte is fetched.""" + + async def test_an_absent_digest_fails_at_received_with_nothing_downloaded( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + tmp_path: Any, + ) -> None: + object_key, digest = import_upload_bundle(import_bundle_factory()) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=None, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + with pytest.raises(WorkflowFailureError) as failure: + await handle.result() + + row = import_read_status_row(import_run_id) + assert row is not None + _assert_step(row, ImportStep.RECEIVED, handle.id) + assert 'expected_digest' in failure_text(failure.value) + + # Nothing downloaded: the work directory was never even created. + work_dir = tmp_path / 'import-work' + assert not work_dir.exists() or not list(work_dir.rglob('*')) + import_assert_nothing_provisioned( + experiment_name=import_names['experiment_name'], + model_name=import_names['model_name'], + object_key=object_key, + ) diff --git a/e2e/test_import_model_gates.py b/e2e/test_import_model_gates.py new file mode 100644 index 0000000..593c7fc --- /dev/null +++ b/e2e/test_import_model_gates.py @@ -0,0 +1,675 @@ +"""One rejection per gate, against a real object, a real archive and a real filesystem. + +The gates are unit-tested already. What only a real run can show is that a rejection is *recorded* +correctly and that it provisions nothing: the row ends `ORCHESTRATOR_VALIDATION_ERROR` naming that +gate's step, its catalog code and its catalog sentence, and no experiment, run, artifact, registry +version, model document or extracted file survives. + +Every case here is hostile **input** — design E7's first preference. Nothing is patched, no +dependency is broken; the bundles are built by the shared factory and the gates do the rest. The +one exception to "input only" is the object-size ceiling, where the ceiling itself is narrowed +rather than a gigabyte being uploaded; that is a configuration change, not a patched code path, and +it is scoped to the one class that needs it. + +`_assert_rejected` is where the shared claim of task 5.8 lives — terminal status, recorded step, +recorded code, catalog sentence, and `assert_nothing_provisioned` — so no case can quietly assert +four fifths of it. +""" + +from __future__ import annotations + +import json +import tempfile +from typing import Any + +import pytest +from temporalio.client import WorkflowFailureError + +from e2e.import_harness import ( + IMPORT_BUCKET, + IMPORT_PREFIX, + failure_text, +) +from laborious.activities.model_import_errors import IMPORT_FAILURE_CATALOG +from laborious.utils.bundle.steps import ImportStep + +pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio] + +VALIDATION_ERROR_STATUS = 'ORCHESTRATOR_VALIDATION_ERROR' +FACTORY_PASSWORD = 'Factory1Model!Test' + + +def _as_dict(value: Any) -> dict[str, Any]: + return json.loads(value) if isinstance(value, str) else (value or {}) + + +async def _run_rejected_import( + *, + payload: bytes, + password: str, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + envelope_password: str | None = None, +) -> tuple[Any, int, str]: + """Upload a hostile bundle, insert the row, run the import and expect it to fail.""" + object_key, digest = import_upload_bundle(payload) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope( + envelope_password if envelope_password is not None else password + ), + ) + ) + return handle, import_run_id, object_key + + +async def _assert_rejected( + *, + handle: Any, + import_run_id: int, + object_key: str, + step: ImportStep, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + import_names: dict[str, str], + experiment_name: str | None = None, + model_name: str | None = None, +) -> dict[str, Any]: + """The whole of task 5.8, in one call. Returns the row so a case can add to it.""" + with pytest.raises(WorkflowFailureError): + await handle.result() + + row = import_read_status_row(import_run_id) + assert row is not None, 'the rejection lost the row it was recorded on' + assert row['status'] == VALIDATION_ERROR_STATUS, ( + f'terminal status is {row["status"]!r}, expected {VALIDATION_ERROR_STATUS!r}' + ) + + expected_code, expected_reason = IMPORT_FAILURE_CATALOG[step] + detail = _as_dict(row['orchestrator_response_data'])['import'] + assert detail['step'] == step.value, ( + f'recorded step is {detail["step"]!r}, expected {step.value!r}' + ) + assert detail['code'] == expected_code.value, ( + f'recorded code is {detail["code"]!r}, expected {expected_code.value!r}' + ) + assert detail['workflow_id'] == handle.id + assert row['error_message'] == expected_reason, ( + 'error_message is not the catalog sentence for this step' + ) + + import_assert_nothing_provisioned( + experiment_name=experiment_name or import_names['experiment_name'], + model_name=model_name or import_names['model_name'], + object_key=object_key, + ) + return row + + +class TestGate1ObjectLimits: + """Gate 1 rejects before the object's body is fetched at all.""" + + @pytest.fixture + def import_limits_override(self) -> dict[str, str]: + """A 1 KiB ceiling, so a factory bundle is over it. + + The production ceiling is 1 GiB. Narrowing it is the only way to cross gate 1 without + uploading a gigabyte, and it changes a configured limit rather than a code path — the same + comparison runs, against a different number. + """ + return {'IMPORT_MAX_OBJECT_BYTES': '1024'} + + async def test_an_oversized_object_is_rejected_at_download( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + payload = import_bundle_factory() + assert len(payload) > 1024, 'the fixture bundle must exceed the narrowed ceiling' + handle, import_run_id, object_key = await _run_rejected_import( + payload=payload, + password=FACTORY_PASSWORD, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + await _assert_rejected( + handle=handle, + import_run_id=import_run_id, + object_key=object_key, + step=ImportStep.DOWNLOAD, + import_read_status_row=import_read_status_row, + import_assert_nothing_provisioned=import_assert_nothing_provisioned, + import_names=import_names, + ) + + +class TestGate2Header: + """Four header rejections, all landing on `download` — the step that says "not these bytes".""" + + @pytest.mark.parametrize( + ('case', 'header_kwargs'), + [ + ('wrong magic', {'magic': b'NOTSIENT'}), + ('unsupported version', {'format_version': 9}), + ('kdf memory above the ceiling', {'declared_memlimit': 512 * 1024 * 1024}), + ('kdf passes above the ceiling', {'declared_opslimit': 9}), + ('digest not of the chunk stream', {'digest_override': bytes(32)}), + ], + ) + async def test_a_malformed_header_is_rejected_at_download( + self, + case: str, + header_kwargs: dict[str, Any], + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + handle, import_run_id, object_key = await _run_rejected_import( + payload=import_bundle_factory(**header_kwargs), + password=FACTORY_PASSWORD, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + row = await _assert_rejected( + handle=handle, + import_run_id=import_run_id, + object_key=object_key, + step=ImportStep.DOWNLOAD, + import_read_status_row=import_read_status_row, + import_assert_nothing_provisioned=import_assert_nothing_provisioned, + import_names=import_names, + ) + # None of these may borrow the decryption sentence: the password is not implicated. + assert 'password' not in (row['error_message'] or '').lower(), case + + +class TestGate3Decryption: + """The password, the tag and the framing — the three ways a stream refuses to open.""" + + async def test_a_wrong_password_is_rejected_at_decryption( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + """The envelope opens — it is the *bundle* password inside it that is wrong. + + Worth separating: an envelope the worker cannot open is also a decryption-step failure, but + for a different reason, and only this one proves the KDF actually ran and produced a key + the stream rejected. + """ + handle, import_run_id, object_key = await _run_rejected_import( + payload=import_bundle_factory(password=FACTORY_PASSWORD), + password=FACTORY_PASSWORD, + envelope_password='Wrong1Password!Entirely', + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + await _assert_rejected( + handle=handle, + import_run_id=import_run_id, + object_key=object_key, + step=ImportStep.DECRYPTION, + import_read_status_row=import_read_status_row, + import_assert_nothing_provisioned=import_assert_nothing_provisioned, + import_names=import_names, + ) + + async def test_a_tampered_chunk_is_rejected_at_decryption( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + """One flipped byte inside the stream, with the header digest repaired to match. + + Repairing the digest is what makes this gate 3 rather than gate 2: the file is + self-consistent, so only the AEAD can tell it was altered. + """ + from tests.helpers.bundle_factory import tamper_ciphertext + + handle, import_run_id, object_key = await _run_rejected_import( + payload=tamper_ciphertext(import_bundle_factory()), + password=FACTORY_PASSWORD, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + await _assert_rejected( + handle=handle, + import_run_id=import_run_id, + object_key=object_key, + step=ImportStep.DECRYPTION, + import_read_status_row=import_read_status_row, + import_assert_nothing_provisioned=import_assert_nothing_provisioned, + import_names=import_names, + ) + + async def test_a_stream_without_its_final_tag_is_rejected_at_decryption( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + """Every chunk framed as MESSAGE, so the stream claims more is coming and never ends. + + The digest cannot see this — the bytes on disk are exactly the ones it covers. Only the + `secretstream` framing knows the file is incomplete. + """ + handle, import_run_id, object_key = await _run_rejected_import( + payload=import_bundle_factory(drop_final_tag=True), + password=FACTORY_PASSWORD, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + await _assert_rejected( + handle=handle, + import_run_id=import_run_id, + object_key=object_key, + step=ImportStep.DECRYPTION, + import_read_status_row=import_read_status_row, + import_assert_nothing_provisioned=import_assert_nothing_provisioned, + import_names=import_names, + ) + + +class TestGate4ArchiveInspection: + """The central directory alone decides — nothing has been written to disk yet.""" + + @pytest.fixture + def import_limits_override(self) -> dict[str, str]: + """A 1 MiB declared-expansion ceiling, for the oversized-expansion case in this class.""" + return {'IMPORT_MAX_UNCOMPRESSED_BYTES': str(1024 * 1024)} + + @pytest.mark.parametrize( + ('case', 'kwargs'), + [ + ('entry escaping the tree', {'traversal_entry': 'artifacts/../../escaped.csv'}), + ('absolute entry path', {'absolute_entry': '/opt/absolute.csv'}), + ('symbolic link', {'symlinks': {'artifacts/link.csv': 'artifacts/train_data.csv'}}), + ('declared expansion over the ceiling', {'bomb_bytes': 4 * 1024 * 1024}), + ], + ) + async def test_a_hostile_archive_is_rejected_before_anything_is_extracted( + self, + case: str, + kwargs: dict[str, Any], + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + tmp_path: Any, + ) -> None: + handle, import_run_id, object_key = await _run_rejected_import( + payload=import_bundle_factory(**kwargs), + password=FACTORY_PASSWORD, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + await _assert_rejected( + handle=handle, + import_run_id=import_run_id, + object_key=object_key, + step=ImportStep.ARCHIVE_INSPECTION, + import_read_status_row=import_read_status_row, + import_assert_nothing_provisioned=import_assert_nothing_provisioned, + import_names=import_names, + ) + # "Nothing extracted" is stronger than "nothing provisioned": not one member reached disk. + work_dir = tmp_path / 'import-work' + assert not list(work_dir.rglob('*')) if work_dir.exists() else True, case + + +class TestGate5Extraction: + """A member that gate 4 passed and extraction still cannot place. + + Gate 5 is about *placement*, not size: containment on the resolved path, and never overwriting + a file already written. A zip carrying the same name twice passes gate 4 — which checks names + and modes, not duplicates — and is refused here on the second member. + """ + + async def test_a_duplicate_member_is_rejected_at_extraction( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + from tests.helpers.bundle_factory import default_tree + + # `bomb_bytes` writes `artifacts/bomb.csv`, and `files` already carries that name, so the + # archive declares it twice. Small enough that no size ceiling is involved. + files = dict(default_tree()) + files['artifacts/bomb.csv'] = b'first\n' + handle, import_run_id, object_key = await _run_rejected_import( + payload=import_bundle_factory(files=files, bomb_bytes=64), + password=FACTORY_PASSWORD, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + await _assert_rejected( + handle=handle, + import_run_id=import_run_id, + object_key=object_key, + step=ImportStep.EXTRACTION, + import_read_status_row=import_read_status_row, + import_assert_nothing_provisioned=import_assert_nothing_provisioned, + import_names=import_names, + ) + + +class TestGate6Structure: + """The layout and the metadata schema, checked on the extracted tree.""" + + @pytest.mark.parametrize( + ('case', 'kwargs'), + [ + ('no metadata.json', {'include_metadata': False}), + ('metadata missing run_id', {'drop_metadata_field': 'run_id'}), + ('parameters missing target_variable', {'drop_parameter': 'target_variable'}), + ], + ) + async def test_an_incomplete_bundle_is_rejected_at_structure_validation( + self, + case: str, + kwargs: dict[str, Any], + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + handle, import_run_id, object_key = await _run_rejected_import( + payload=import_bundle_factory(**kwargs), + password=FACTORY_PASSWORD, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + await _assert_rejected( + handle=handle, + import_run_id=import_run_id, + object_key=object_key, + step=ImportStep.STRUCTURE_VALIDATION, + import_read_status_row=import_read_status_row, + import_assert_nothing_provisioned=import_assert_nothing_provisioned, + import_names=import_names, + ) + + async def test_a_bundle_without_prediction_model_is_rejected( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + """Without `prediction_model` there is nothing to register a version against.""" + from tests.helpers.bundle_factory import default_tree + + files = { + name: content + for name, content in default_tree().items() + if not name.startswith('artifacts/prediction_model/') + } + handle, import_run_id, object_key = await _run_rejected_import( + payload=import_bundle_factory(files=files), + password=FACTORY_PASSWORD, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + await _assert_rejected( + handle=handle, + import_run_id=import_run_id, + object_key=object_key, + step=ImportStep.STRUCTURE_VALIDATION, + import_read_status_row=import_read_status_row, + import_assert_nothing_provisioned=import_assert_nothing_provisioned, + import_names=import_names, + ) + + async def test_an_over_long_experiment_name_never_reaches_the_varchar_column( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + import_bff_engine: Any, + ) -> None: + """A 60-character experiment name is refused at gate 6, not by Postgres three steps later. + + `experiment_name` is `VARCHAR(50)` and the import writes the bundle's own name onto the row + *before* provisioning. If the gate did not enforce the column's limit, the names write would + be a `DataError` — non-retryable, recorded as a status failure, on a row whose import was + otherwise fine. Asserting the row's `experiment_name` is untouched is what proves the write + never happened. + """ + from sqlalchemy import text + + long_name = 'x' * 60 + handle, import_run_id, object_key = await _run_rejected_import( + payload=import_bundle_factory(metadata={'experiment_name': long_name}), + password=FACTORY_PASSWORD, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + row = await _assert_rejected( + handle=handle, + import_run_id=import_run_id, + object_key=object_key, + step=ImportStep.STRUCTURE_VALIDATION, + import_read_status_row=import_read_status_row, + import_assert_nothing_provisioned=import_assert_nothing_provisioned, + import_names=import_names, + experiment_name=long_name, + ) + assert row['experiment_name'] == 'import-e2e-placeholder', ( + 'the over-long name reached the row, so the gate did not hold the column' + ) + with import_bff_engine.begin() as connection: + width = connection.execute( + text( + 'SELECT character_maximum_length FROM information_schema.columns ' + "WHERE table_name = 'experiment_run' AND column_name = 'experiment_name'" + ) + ).scalar_one() + assert width == 50, 'the column this gate exists to protect is no longer VARCHAR(50)' + + +class TestGate7ContentPolicy: + """A file kind an MLflow artifact tree has no reason to carry.""" + + async def test_a_forbidden_file_type_is_rejected_at_content_policy( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + handle, import_run_id, object_key = await _run_rejected_import( + payload=import_bundle_factory(extra_files={'artifacts/payload.exe': b'MZ\x00\x00'}), + password=FACTORY_PASSWORD, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + await _assert_rejected( + handle=handle, + import_run_id=import_run_id, + object_key=object_key, + step=ImportStep.CONTENT_POLICY, + import_read_status_row=import_read_status_row, + import_assert_nothing_provisioned=import_assert_nothing_provisioned, + import_names=import_names, + ) + + +class TestTheRejectionSaysNothingTechnical: + """`error_message` is a sentence for a person: no path, no object key, no traceback. + + One case is enough to make the point structurally — the catalog is keyed by step and + `classify_import_failure` discards the exception's text entirely — and the per-gate cases above + each assert the sentence is *exactly* the catalog's, which is the same guarantee stated + positively. + """ + + async def test_no_technical_detail_reaches_the_recorded_reason( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + handle, import_run_id, object_key = await _run_rejected_import( + payload=import_bundle_factory(include_metadata=False), + password=FACTORY_PASSWORD, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + with pytest.raises(WorkflowFailureError) as failure: + await handle.result() + + row = import_read_status_row(import_run_id) + assert row is not None + message = row['error_message'] or '' + for forbidden in ( + object_key, + IMPORT_BUCKET, + IMPORT_PREFIX, + 'experiment_run', + 'Traceback', + 'BundleGateError', + tempfile.gettempdir(), + 'gate', + ): + assert forbidden not in message, f'{forbidden!r} leaked into error_message: {message!r}' + + # The technical truth is not lost — it is on the Temporal error, where support can read it. + assert 'metadata' in failure_text(failure.value).lower() diff --git a/e2e/test_import_model_happy_path.py b/e2e/test_import_model_happy_path.py new file mode 100644 index 0000000..30bf253 --- /dev/null +++ b/e2e/test_import_model_happy_path.py @@ -0,0 +1,314 @@ +"""An import that works, twice over: once from the producer's own bytes and once from the factory. + +Two origins, because they prove different things. The **golden** bundle is bytes the producer's code +emitted — it decrypts at the producer's real Argon2id cost (256 MiB, 3 passes) and carries the tree +the producer really writes, so it is the only case that would notice the producer's layout moving. +The **factory** bundle is synthesised here, cheaply, with names unique to the test; it is what the +rest of the suite runs on, so a happy path over it is what makes the negative cases' "and nothing +was provisioned" meaningful. + +What "it worked" means, in full: the experiment exists, a run exists inside it carrying the derived +run name, the artifact tree is under the run with `prediction_model` and `data_model` at its root, +every parameter the bundle declared is logged plus the origin's identity under `source.*`, a +registry version exists, the MongoDB document exists, and the status row ends `IMPORT_SUCCESS`. + +Two things a successful import must **not** have done are asserted here too, because they are the +difference between "imported" and "live": no version is in `Production`, and the document is +`active: false`. Promotion and activation are manual operator steps (QTZPOC-21). +""" + +from __future__ import annotations + +import json +from typing import Any + +import mlflow +import pytest + +from e2e.import_harness import ( + IMPORT_BUCKET, + IMPORT_PREFIX, + SEEDED_MODEL_IDS, +) + +pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio] + +GOLDEN_MODEL_NAME = 'fixture_model' +GOLDEN_EXPERIMENT_NAME = 'fixture_experiment' +GOLDEN_MODEL_VERSION = '3' +GOLDEN_TARGET = 'SE' + +REQUIRED_ARTIFACT_ROOTS = {'prediction_model', 'data_model'} + + +def _mlflow_client(server: dict[str, Any]) -> Any: + return mlflow.tracking.MlflowClient(tracking_uri=server['tracking_uri']) + + +def _as_dict(value: Any) -> dict[str, Any]: + """`request_data` / `orchestrator_response_data` come back as a dict or as JSON text.""" + return json.loads(value) if isinstance(value, str) else (value or {}) + + +async def _run_import( + *, + payload: bytes, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + password: str, + import_envelope: Any, +) -> tuple[Any, int, str]: + """Upload, insert the row the BFF would have inserted, run the workflow. Return the outcome.""" + object_key, digest = import_upload_bundle(payload) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(password), + ) + ) + return handle, import_run_id, object_key + + +def _assert_provisioned( + *, + client: Any, + experiment_name: str, + model_name: str, + run_name: str, + expected_parameters: dict[str, str], + expected_source_fields: dict[str, str], +) -> tuple[str, Any]: + """The whole positive claim about MLflow, in one place. Returns `(run_id, version)`.""" + experiment = client.get_experiment_by_name(experiment_name) + assert experiment is not None, f'no experiment named {experiment_name!r}' + + runs = [ + run + for run in client.search_runs([experiment.experiment_id]) + if run.info.run_name == run_name + ] + assert len(runs) == 1, f'expected exactly one run named {run_name!r}, found {len(runs)}' + run = runs[0] + assert run.info.status == 'FINISHED' + + artifact_roots = {artifact.path for artifact in client.list_artifacts(run.info.run_id)} + assert REQUIRED_ARTIFACT_ROOTS <= artifact_roots, ( + f'the run holds {sorted(artifact_roots)}, missing {sorted(REQUIRED_ARTIFACT_ROOTS - artifact_roots)}' + ) + # `prediction_model` is a directory the legacy loader walks, not an opaque blob. + prediction_files = { + artifact.path.rsplit('/', 1)[-1] + for artifact in client.list_artifacts(run.info.run_id, 'prediction_model') + } + assert {'MLmodel', 'model.pkl'} <= prediction_files, prediction_files + + for name, value in expected_parameters.items(): + assert run.data.params.get(name) == value, ( + f'parameter {name!r} is {run.data.params.get(name)!r}, expected {value!r}' + ) + for name, value in expected_source_fields.items(): + assert run.data.params.get(f'source.{name}') == value, ( + f'source.{name} is {run.data.params.get(f"source.{name}")!r}, expected {value!r}' + ) + + versions = list(client.search_model_versions(f"name='{model_name}'")) + assert len(versions) == 1, f'expected one registered version, found {len(versions)}' + version = versions[0] + assert version.run_id == run.info.run_id + return run.info.run_id, version + + +class TestGoldenBundle: + """The producer's own bytes, at the producer's own KDF cost.""" + + async def test_the_producer_artefact_imports_end_to_end( + self, + import_status_table: Any, + import_models_collection: Any, + import_golden_bundle: dict[str, Any], + import_mlflow_server: dict[str, Any], + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + handle, import_run_id, object_key = await _run_import( + payload=import_golden_bundle['bytes'], + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + password=import_golden_bundle['password'], + import_envelope=import_envelope, + ) + result = await handle.result() + + assert result['model_name'] == GOLDEN_MODEL_NAME + assert result['import_run_id'] == import_run_id + assert result['version'] == '1', 'the destination numbering starts at 1, not at the bundle' + + run_id, version = _assert_provisioned( + client=_mlflow_client(import_mlflow_server), + experiment_name=GOLDEN_EXPERIMENT_NAME, + model_name=GOLDEN_MODEL_NAME, + run_name=f'import-{GOLDEN_MODEL_NAME}-v{GOLDEN_MODEL_VERSION}', + expected_parameters={'target_variable': GOLDEN_TARGET, 'nan_treatment': 'drop'}, + expected_source_fields={ + 'model_name': GOLDEN_MODEL_NAME, + 'experiment_name': GOLDEN_EXPERIMENT_NAME, + 'model_version': GOLDEN_MODEL_VERSION, + 'model_project': 'fixture_project', + }, + ) + assert result['run_id'] == run_id + + document = import_models_collection.find_one({'name': GOLDEN_MODEL_NAME}, {'_id': False}) + assert document is not None, 'no model document was written' + assert document['id'] == str(max(int(i) for i in SEEDED_MODEL_IDS) + 1) + assert document['model_config']['target'] == GOLDEN_TARGET + + row = import_read_status_row(import_run_id) + assert row is not None + assert row['status'] == 'IMPORT_SUCCESS' + assert row['error_message'] is None + assert row['experiment_name'] == GOLDEN_EXPERIMENT_NAME + assert row['run_name'] == f'import-{GOLDEN_MODEL_NAME}-v{GOLDEN_MODEL_VERSION}' + assert row['file_name'] == object_key + assert row['bucket_name'] == IMPORT_BUCKET + assert object_key.startswith(IMPORT_PREFIX) + + # Imported, not live. + assert version.current_stage in (None, 'None'), version.current_stage + assert document['active'] is False + + +class TestFactoryBundle: + """A synthesised tree with names unique to this test — the shape the rest of the suite runs on.""" + + async def test_a_factory_bundle_imports_end_to_end( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_mlflow_server: dict[str, Any], + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + payload = import_bundle_factory(parameters={'target_variable': 'SE'}) + handle, import_run_id, _ = await _run_import( + payload=payload, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + password='Factory1Model!Test', + import_envelope=import_envelope, + ) + result = await handle.result() + + assert result['model_name'] == import_names['model_name'] + run_id, version = _assert_provisioned( + client=_mlflow_client(import_mlflow_server), + experiment_name=import_names['experiment_name'], + model_name=import_names['model_name'], + # DEFAULT_METADATA declares model_version '2'. + run_name=f'import-{import_names["model_name"]}-v2', + expected_parameters={'target_variable': 'SE', 'retrain': 'False'}, + expected_source_fields={ + 'model_name': import_names['model_name'], + 'experiment_name': import_names['experiment_name'], + 'model_version': '2', + }, + ) + assert result['run_id'] == run_id + assert result['version'] == '1' + + document = import_models_collection.find_one( + {'name': import_names['model_name']}, {'_id': False} + ) + assert document is not None + assert document['model_config'] == { + 'transform_flavor': 'sklearn', + 'predict_flavor': 'joblib', + 'retention_minutes': 60, + 'target': 'SE', + } + + row = import_read_status_row(import_run_id) + assert row is not None + assert row['status'] == 'IMPORT_SUCCESS' + detail = _as_dict(row['orchestrator_response_data']) + assert detail['import']['workflow_id'] == handle.id + + assert version.current_stage in (None, 'None') + assert document['active'] is False + + +class TestLegacyLoaderResolution: + """The registry source an import writes has to be one the *existing* loader can parse. + + This is the assertion that could only ever be made against a real registry: the importer writes + the run's resolved artifact URI with `prediction_model` appended, because `get_model_run_id` + extracts the run id as `source.split('/')[2]` — an index that is the run id only in the resolved + form. A literal `runs://prediction_model` would put `prediction_model` at index 2, and a + `models:/m-*` source would have none of the parts at all. + + It is called with the stage the version is actually in rather than `Production`, because an + import deliberately promotes nothing — passing `Production` here would test the promotion step + this feature does not have. + """ + + async def test_the_imported_source_resolves_to_the_import_run( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_activities: Any, + import_mlflow_server: dict[str, Any], + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + ) -> None: + payload = import_bundle_factory() + handle, _, _ = await _run_import( + payload=payload, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + password='Factory1Model!Test', + import_envelope=import_envelope, + ) + result = await handle.result() + model_name = import_names['model_name'] + + client = _mlflow_client(import_mlflow_server) + version = next(iter(client.search_model_versions(f"name='{model_name}'"))) + assert version.source.endswith('/prediction_model'), version.source + + # The production method, on the production repository object, against the real registry. + resolved = import_activities.import_mlflow_repository.get_model_run_id( + model_name, stage=version.current_stage + ) + assert resolved == result['run_id'], ( + f'get_model_run_id resolved {resolved!r} from source {version.source!r}, ' + f'but the import created run {result["run_id"]!r}' + ) + + artifact_roots = {artifact.path for artifact in client.list_artifacts(resolved)} + assert REQUIRED_ARTIFACT_ROOTS <= artifact_roots, sorted(artifact_roots) diff --git a/e2e/test_import_model_provisioning_failures.py b/e2e/test_import_model_provisioning_failures.py new file mode 100644 index 0000000..a0f15e0 --- /dev/null +++ b/e2e/test_import_model_provisioning_failures.py @@ -0,0 +1,364 @@ +"""Each provisioning step fails as itself: its own code, its own sentence, nothing half-built. + +The four steps after the gates — experiment creation, artifact upload, registration, the model +document — each get one failure, and every one lands `IMPORT_ERROR` rather than +`ORCHESTRATOR_VALIDATION_ERROR`. That split is not cosmetic: it is the table's existing distinction +between "the person who submitted this can fix it" and "only support can", and a provisioning +failure is always the second kind. + +**How each failure is caused**, in design E7's order of preference: + +| Step | Cause | Preference | +|---|---|---| +| `experiment_creation` | the MLflow server is genuinely stopped | broken dependency | +| `artifact_upload` | targeted patch on `log_artifacts` | patch, justified in place | +| `registration` | targeted patch on `create_model_version` | patch, justified in place | +| `model_document` | a document already holding the id the activity will compute, against the real unique index | broken dependency | + +Two patches rather than the one design E7 anticipated. The reason is the same for both and is +recorded in each test: with one server serving both tracking and registry, and MLflow validating +nothing about a registered model name except that it is non-empty (which gate 6 already requires), +there is no way to make the world hostile to *just* one of these two steps. Stopping the server +fails experiment creation first; a malformed name is not malformed to MLflow. The patch is on the +client call the step makes, not on the activity or the workflow, so what is being tested is still +"the workflow attributes this step's failure to this step". +""" + +from __future__ import annotations + +import json +import tempfile +from typing import Any + +import pytest +from temporalio.client import WorkflowFailureError + +from e2e.import_harness import IMPORT_BUCKET, IMPORT_PREFIX, SEEDED_MODEL_IDS +from laborious.activities.model_import_errors import IMPORT_FAILURE_CATALOG +from laborious.utils.bundle.steps import ImportStep + +pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio] + +PLATFORM_ERROR_STATUS = 'IMPORT_ERROR' +FACTORY_PASSWORD = 'Factory1Model!Test' + + +def _as_dict(value: Any) -> dict[str, Any]: + return json.loads(value) if isinstance(value, str) else (value or {}) + + +def _assert_platform_failure( + *, + row: dict[str, Any] | None, + step: ImportStep, + workflow_id: str, + object_key: str, +) -> None: + """The shared claim of task 7.5, asserted by every case in this module. + + `error_message` is the catalog sentence *exactly* — not "contains" — which is the strongest + possible form of "carries no path, table name, object key or traceback": there is nothing in it + that did not come from the catalog. + """ + assert row is not None, 'the failure lost the row it was recorded on' + assert row['status'] == PLATFORM_ERROR_STATUS, ( + f"status is {row['status']!r}; a provisioning failure is not the user's to fix" + ) + + code, reason = IMPORT_FAILURE_CATALOG[step] + detail = _as_dict(row['orchestrator_response_data'])['import'] + assert detail['step'] == step.value, detail + assert detail['code'] == code.value, detail + assert detail['workflow_id'] == workflow_id, detail + assert row['error_message'] == reason + + message = row['error_message'] or '' + for forbidden in ( + object_key, + IMPORT_BUCKET, + IMPORT_PREFIX, + 'experiment_run', + 'Traceback', + 'Error', + 'mlflow', + 'Mongo', + tempfile.gettempdir(), + ): + assert forbidden not in message, f'{forbidden!r} leaked into error_message: {message!r}' + + +async def _upload_and_start( + *, + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + **factory_kwargs: Any, +) -> tuple[Any, int, str]: + object_key, digest = import_upload_bundle(import_bundle_factory(**factory_kwargs)) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + return handle, import_run_id, object_key + + +class TestExperimentCreation: + """The tracking server is not there. No patch, no mock — the process is stopped.""" + + async def test_a_stopped_tracking_server_fails_at_experiment_creation( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_mlflow_stopped: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + with import_mlflow_stopped(): + handle, import_run_id, object_key = await _upload_and_start( + import_bundle_factory=import_bundle_factory, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + with pytest.raises(WorkflowFailureError): + await handle.result() + row = import_read_status_row(import_run_id) + + _assert_platform_failure( + row=row, + step=ImportStep.EXPERIMENT_CREATION, + workflow_id=handle.id, + object_key=object_key, + ) + + # The bundle *was* opened before MLflow was reached, so the names are on the row — which is + # what task 9.3 asks for, observed from a failure that stopped exactly there. + assert row is not None + assert row['experiment_name'] == import_names['experiment_name'] + assert row['run_name'] == f'import-{import_names["model_name"]}-v2' + + # Nothing in Mongo: the document is the last step and never ran. + documents = list(import_models_collection.find({}, {'_id': False})) + assert {str(d['id']) for d in documents} == set(SEEDED_MODEL_IDS) + import_assert_nothing_provisioned( + experiment_name=import_names['experiment_name'], + model_name=import_names['model_name'], + object_key=object_key, + ) + + +class TestArtifactUpload: + """The step design E7 names as the one a patch is needed for.""" + + async def test_a_failing_artifact_upload_is_recorded_as_its_own_step( + self, + monkeypatch: pytest.MonkeyPatch, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_activities: Any, + import_mlflow_server: dict[str, Any], + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + """Why a patch: with the server up and the bundle valid, nothing else fails only this step. + + Hostile input cannot reach here — a bundle that gets past gate 7 has a readable tree, and + one that does not never reaches artifact upload. A broken dependency cannot isolate it + either: the only dependency this step has is the tracking server, and stopping that fails + experiment creation one step earlier. So the patch is on the single client call the step + makes, `log_artifacts`; everything around it — the run creation, the parameters, the + workflow's attribution of the failure — is the real code. + """ + client = import_activities.import_mlflow_repository.client + + def refuse(*_args: Any, **_kwargs: Any) -> None: + raise OSError('artifact store is unreachable') + + monkeypatch.setattr(client, 'log_artifacts', refuse) + + handle, import_run_id, object_key = await _upload_and_start( + import_bundle_factory=import_bundle_factory, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + with pytest.raises(WorkflowFailureError): + await handle.result() + + _assert_platform_failure( + row=import_read_status_row(import_run_id), + step=ImportStep.ARTIFACT_UPLOAD, + workflow_id=handle.id, + object_key=object_key, + ) + + # The experiment exists — it was the previous step and it succeeded. Nothing after does. + mlflow_client = _fresh_client(import_mlflow_server) + assert mlflow_client.get_experiment_by_name(import_names['experiment_name']) is not None + assert not list(mlflow_client.search_model_versions(f"name='{import_names['model_name']}'")) + assert import_models_collection.find_one({'name': import_names['model_name']}) is None + + +class TestRegistration: + """The artifacts are stored and the model still cannot be registered.""" + + async def test_a_failing_registration_is_recorded_as_its_own_step( + self, + monkeypatch: pytest.MonkeyPatch, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_activities: Any, + import_mlflow_server: dict[str, Any], + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + """Why a patch here too, contrary to design E7's expectation that this step needs none. + + Hostile input was the first thing tried: a registered model name MLflow refuses. It does not + exist — `_validate_model_name` rejects only the empty string, which gate 6 already refuses, + and the sqlite backend store does not enforce the `VARCHAR(256)` the schema declares, so + even a 300-character name is accepted. A broken dependency was the second: one process + serves both tracking and the registry, so stopping it fails experiment creation three steps + earlier. The patch is on `create_model_version`, the one call this step is. + """ + client = import_activities.import_mlflow_repository.client + + def refuse(*_args: Any, **_kwargs: Any) -> None: + raise RuntimeError('registry write rejected') + + monkeypatch.setattr(client, 'create_model_version', refuse) + + handle, import_run_id, object_key = await _upload_and_start( + import_bundle_factory=import_bundle_factory, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + with pytest.raises(WorkflowFailureError): + await handle.result() + + _assert_platform_failure( + row=import_read_status_row(import_run_id), + step=ImportStep.REGISTRATION, + workflow_id=handle.id, + object_key=object_key, + ) + + # The run and its artifacts survive — "the model's files were stored, but the model itself + # could not be registered" is exactly what the catalog sentence for this step says. + mlflow_client = _fresh_client(import_mlflow_server) + experiment = mlflow_client.get_experiment_by_name(import_names['experiment_name']) + assert experiment is not None + runs = mlflow_client.search_runs([experiment.experiment_id]) + assert len(runs) == 1 + assert {a.path for a in mlflow_client.list_artifacts(runs[0].info.run_id)} >= { + 'prediction_model', + 'data_model', + } + assert import_models_collection.find_one({'name': import_names['model_name']}) is None + + +class TestModelDocument: + """The last step, failed by a real unique index on a real collection.""" + + async def test_a_duplicate_id_fails_at_model_document( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_mlflow_server: dict[str, Any], + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + """A document already holding the id `max(int(id)) + 1` will compute, under another name. + + Under another name on purpose: a document with the *same* name would make the activity skip + the insert entirely, which is a different behaviour (task 8.6) and would not fail. This one + collides only on `id`, so the insert is attempted and the index refuses it. + + No patch and no mock: the failure is `pymongo`'s duplicate-key error from a real index, + which is the thing that would happen in production if two imports raced. + """ + collision_id = str(max(int(seeded) for seeded in SEEDED_MODEL_IDS) + 1) + import_models_collection.insert_one( + {'id': collision_id, 'name': 'a_different_model', 'active': True, 'model_config': {}} + ) + + handle, import_run_id, object_key = await _upload_and_start( + import_bundle_factory=import_bundle_factory, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + with pytest.raises(WorkflowFailureError): + await handle.result() + + _assert_platform_failure( + row=import_read_status_row(import_run_id), + step=ImportStep.MODEL_DOCUMENT, + workflow_id=handle.id, + object_key=object_key, + ) + + # No document for this model, and the colliding one is untouched. + assert import_models_collection.find_one({'name': import_names['model_name']}) is None + blocker = import_models_collection.find_one({'id': collision_id}, {'_id': False}) + assert blocker == { + 'id': collision_id, + 'name': 'a_different_model', + 'active': True, + 'model_config': {}, + } + + # Everything before it stands: "the model was registered, but the settings ... could not be + # saved" is the catalog sentence, and this is what makes it true rather than merely stored. + mlflow_client = _fresh_client(import_mlflow_server) + versions = list(mlflow_client.search_model_versions(f"name='{import_names['model_name']}'")) + assert len(versions) == 1 + + +def _fresh_client(server: dict[str, Any]) -> Any: + """An MlflowClient bound to the test server, independent of the one the activities hold.""" + import mlflow + + return mlflow.tracking.MlflowClient(tracking_uri=server['tracking_uri']) diff --git a/encode.sh b/encode.sh deleted file mode 100755 index 2beb0d1..0000000 --- a/encode.sh +++ /dev/null @@ -1,13 +0,0 @@ -source ./venv/bin/activate - -pip install pathspec -pip install pyyaml - -echo " -.git" >> .gitignore - -python encrypt.py ./ code --ignore .gitignore --chunk-size 100000 - -sed -i '/.git/d' .gitignore - -xdg-open . \ No newline at end of file diff --git a/laborious/activities/activities.py b/laborious/activities/activities.py index f4a9a2c..e62e0d1 100644 --- a/laborious/activities/activities.py +++ b/laborious/activities/activities.py @@ -181,8 +181,11 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, ModelImport): import_config=import_config or build_import_config(), minio_repository=minio_repository, # The MLflow repository is shared rather than rebuilt: setting the tracking URI is a - # process-wide side effect, so two instances would be two chances to disagree. - mlflow_repository=self.model_monitoring_repository, + # process-wide side effect, so two instances would be two chances to disagree. Read + # through `getattr` because `MLFlow.__init__` is what creates it — if that ever stops + # running before this line, the import activities refuse with "MLflow repository not + # initialized" at the point of use rather than silently provisioning nothing. + mlflow_repository=getattr(self, 'model_monitoring_repository', None), mongo_repository=mongo_repository, logger=logger, notification_handler=notification_handler, diff --git a/laborious/activities/model_import.py b/laborious/activities/model_import.py index 125ee97..0af7a88 100644 --- a/laborious/activities/model_import.py +++ b/laborious/activities/model_import.py @@ -853,8 +853,15 @@ class ModelImport(SientiaMonitoring): async def register_import_model_version(self, input_data: dict[str, Any]) -> dict[str, Any]: """Register a version whose source the legacy loader can parse. No stage change. - The source is `runs://prediction_model`, because `get_model_run_id` extracts the run - id as `source.split('/')[2]` — a `models:/m-*` source, what the new stack emits, breaks it. + The source is the run's **resolved artifact URI** with `prediction_model` appended — + `mlflow-artifacts:///artifacts/prediction_model` on this deployment — + because `get_model_run_id` extracts the run id as `source.split('/')[2]`. That index is the + run id only in the resolved form: a literal `runs://prediction_model` has just two + slashes, so index 2 is the string `prediction_model`, and a `models:/m-*` source (what the + new stack emits) has none of the parts at all. This is the same value the training flow + stores, because `mlflow.register_model` resolves the `runs:/` URI before writing it + (`model_repository.py`, `register_model`) — `create_model_version` does not. + The stage is left untouched: a `COMPLETED` import is a registered model, not a servable one, and promoting it to `Production` is a manual operator step outside this workflow. @@ -880,7 +887,7 @@ class ModelImport(SientiaMonitoring): # lands as a new version of the same registered model, never a differently named one. client.create_registered_model(model_name) - source = f'runs:/{run_id}/prediction_model' + source = f'{str(client.get_run(run_id).info.artifact_uri).rstrip("/")}/prediction_model' version = client.create_model_version(name=model_name, source=source, run_id=run_id) self.info( f'import registered {model_name} version {version.version}', diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index 9f76141..f10a6cb 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -77,6 +77,19 @@ with workflow.unsafe.imports_passed_through(): POD_ID = os.getenv('HOSTNAME') SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091')) +# The values that turn the import worker on. Anything else — unset, empty, `0`, `false`, a typo — +# leaves the four default workers alone and polls no import queue at all. +IMPORT_ENABLED_VALUES = ('1', 'true', 'yes', 'on') + + +def import_worker_enabled() -> bool: + """Whether this runtime registers the fifth (import) worker. + + Read at worker start, not per workflow: containment (QTZPOC-19) wants the importer in its own + runtime with its own credentials, and this flag is what makes that a deployment decision. + """ + return os.getenv('IMPORT_MODEL_ENABLED', '').strip().lower() in IMPORT_ENABLED_VALUES + async def main(): """ @@ -250,7 +263,7 @@ async def main(): # its own credentials, and a flag lets that be a deployment decision instead of forcing every # Laborious runtime to poll the import queue today. Unset, the four workers above are unaffected # and no import queue is polled at all. - if os.getenv('IMPORT_MODEL_ENABLED', '').strip().lower() in ('1', 'true', 'yes', 'on'): + if import_worker_enabled(): logger.custom_info('Import model worker enabled', metadata_runtime) workers.append( prepare_worker( diff --git a/laborious/workflows/import_model.py b/laborious/workflows/import_model.py index 39cbbf6..aed2dcf 100644 --- a/laborious/workflows/import_model.py +++ b/laborious/workflows/import_model.py @@ -356,7 +356,9 @@ class ImportModel: start_to_close_timeout=STATUS_WRITE_TIMEOUT, schedule_to_close_timeout=STATUS_WRITE_ENVELOPE, ) - except BaseException as error: # NOSONAR - the outcome stands even when it cannot be written + except ( + BaseException + ) as error: # NOSONAR - the outcome stands even when it cannot be written await self._report_status_write_failure( base, error, @@ -377,7 +379,9 @@ class ImportModel: retry_policy=retry_policy, start_to_close_timeout=STATUS_WRITE_TIMEOUT, ) - except BaseException: # NOSONAR - reporting the report is where this stops + except BaseException: # noqa: S110 # NOSONAR - the report of a report is where this stops + # Deliberately silent: the channel this was trying to reach is the thing that logs, so + # there is nowhere left to say it, and an import that worked must not fail over it. pass # ----------------------------------------------------------- side-effect steps @@ -389,8 +393,8 @@ class ImportModel: The rule is applied by the importer rather than assumed from a Helm value, so the acceptance criterion is verifiable from the code that depends on it. But a bucket policy the importer cannot set is an operational problem, not a bad import: failing a valid model import because - a lifecycle API refused would be the wrong trade, so this is reported and dropped like a - lost status hint. + a lifecycle API refused would be the wrong trade, so the failure is dropped here and left + where Temporal already records it — the activity's own failure in the event history. """ try: await workflow.execute_activity_method( @@ -399,7 +403,11 @@ class ImportModel: retry_policy=retry_policy, start_to_close_timeout=RETENTION_TIMEOUT, ) - except BaseException: # NOSONAR - retention is not a precondition of a correct import + except BaseException: # noqa: S110 # NOSONAR - retention is not a precondition of an import + # Dropped, not reported: the activity's own failure is already in the event history, + # and a bucket policy the importer cannot set is an operational problem rather than a + # bad import. Failing a valid model import over a lifecycle API would be the wrong + # trade. pass @staticmethod diff --git a/pyproject.toml b/pyproject.toml index 4a90296..06f349f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,11 @@ ignore = [ # must be in sys.modules before Activities is imported, so they cannot move up. "E402", ] +"scripts/**/*.py" = [ + # Notebook-style scripts split into `# %%` cells: each cell imports what it needs + # where it needs it, so imports legitimately appear below other statements. + "E402", +] [tool.ruff.lint.mccabe] max-complexity = 15 @@ -133,6 +138,10 @@ markers = [ "integration: marks tests as integration tests", "unit: marks tests as unit tests", "opc: marks tests that use the in-process OPC UA server (OpcRepository E2E)", + # The .sientia import e2e suite. Outside `testpaths`, so `validate --only-tests` and the + # coverage gate never collect it; run it on request with `pytest e2e -m import_e2e`. + "import_e2e: marks the model-import end-to-end tests (needs Docker and an MLflow server)", + "slow: marks tests that run on a real clock and take tens of seconds", ] [tool.coverage.run] diff --git a/requirements-dev.txt b/requirements-dev.txt index 4d0f89d..a2b8893 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,9 +13,10 @@ types-requests>=2.31.0 # Type stubs for requests pytest>=7.4.0 # Testing framework pytest-cov>=4.1.0 # Coverage plugin for pytest pytest-asyncio>=0.21.0 # Async test support (already in main requirements) -testcontainers[postgres,minio] # PostgreSQL and MinIO containers for E2E tests +testcontainers[postgres,minio,mongodb] # PostgreSQL, MinIO and MongoDB containers for E2E tests # Development Tools ipython>=8.12.0 # Enhanced Python shell ipdb>=0.13.13 # IPython debugger ipykernel==6.30.1 # IPython kernel for Jupyter notebooks +python-dotenv>=1.0.0 # .env loading for scripts/import_model_manual_run.py diff --git a/requirements-local.txt b/requirements-local.txt index 8d060e3..4e6a625 100644 --- a/requirements-local.txt +++ b/requirements-local.txt @@ -4,7 +4,7 @@ sqlalchemy asyncua==1.0.6 redis git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.1 -/home/grezewave/Documents/projects/sientia/sientia-mlops-library +git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.41.1 prometheus-client # fsspec/s3fs/gcsfs pinned as a set: nannyml (via the mlops library) pulls both s3fs and # gcsfs, and each pins fsspec exactly, so a free `s3fs` makes the resolver fail against diff --git a/requirements.txt b/requirements.txt index 5b70c81..31fe203 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ sqlalchemy asyncua==1.0.6 redis sientia_do>=1.12.1 -sientia>=0.41.0 +sientia>=0.41.1 prometheus-client # fsspec/s3fs/gcsfs pinned as a set: nannyml (via sientia) pulls both s3fs and gcsfs, and # each pins fsspec exactly, so a free `s3fs` makes the resolver fail against evidently's diff --git a/scripts/encode.sh b/scripts/encode.sh new file mode 100755 index 0000000..73d0534 --- /dev/null +++ b/scripts/encode.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Resolve paths from the script's own location so this works whether it is run +# from the repository root or from inside scripts/. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +source "${REPO_ROOT}/venv/bin/activate" + +pip install pathspec +pip install pyyaml + +echo " +.git" >> "${REPO_ROOT}/.gitignore" + +python "${SCRIPT_DIR}/encrypt.py" "${REPO_ROOT}" "${REPO_ROOT}/code" --ignore "${REPO_ROOT}/.gitignore" --chunk-size 100000 + +sed -i '/.git/d' "${REPO_ROOT}/.gitignore" + +xdg-open "${REPO_ROOT}" diff --git a/encrypt.py b/scripts/encrypt.py similarity index 61% rename from encrypt.py rename to scripts/encrypt.py index ad990a6..14eb3da 100644 --- a/encrypt.py +++ b/scripts/encrypt.py @@ -1,22 +1,23 @@ -import os import argparse -from pathspec import PathSpec -import yaml # type: ignore +import os from typing import Any -''' +import yaml # type: ignore +from pathspec import PathSpec + +r""" Usage: python .\encrypt.py path_to_dir output_file --ignore ignore_file --chunk-size 100000 -''' +""" def load_ignore_patterns(ignore_file, include_library): # Ensure the .gitignore file exists if not os.path.exists(ignore_file): - raise FileNotFoundError(f"Ignore file not found at {ignore_file}") + raise FileNotFoundError(f'Ignore file not found at {ignore_file}') # Load and parse the .gitignore patterns - with open(ignore_file, 'r') as file: + with open(ignore_file) as file: patterns = file.readlines() if not include_library: patterns.append('**/deploy/library/') @@ -32,14 +33,12 @@ def is_ignored(file_path, spec): def encode_file_tree_to_yaml(directory, ignore_file, include_library): """Encode the file tree into a single YAML file.""" - ignore_patterns = load_ignore_patterns( - ignore_file, include_library) if ignore_file else None + ignore_patterns = load_ignore_patterns(ignore_file, include_library) if ignore_file else None file_tree: dict[str, Any] = {} for root, dirs, files in os.walk(directory): # Skip ignored directories - dirs[:] = [d for d in dirs if not is_ignored( - os.path.join(root, d), ignore_patterns)] + dirs[:] = [d for d in dirs if not is_ignored(os.path.join(root, d), ignore_patterns)] for file in files: file_path = os.path.join(root, file) @@ -50,10 +49,10 @@ def encode_file_tree_to_yaml(directory, ignore_file, include_library): # Read file content try: - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, encoding='utf-8') as f: content = f.read() except Exception as e: - print(f"Error reading file {file_path}: {e}") + print(f'Error reading file {file_path}: {e}') raise # Create nested dictionary structure @@ -72,27 +71,30 @@ def encode_file_tree_to_yaml(directory, ignore_file, include_library): def chunk_and_write_file_tree_to_yaml(yaml_content, output_file, chunk_size=None): """Chunk the YAML content and write it to the output file.""" - chunks = [yaml_content] if chunk_size is None else [ - yaml_content[i:i + chunk_size] for i in range(0, len(yaml_content), chunk_size)] + chunks = ( + [yaml_content] + if chunk_size is None + else [yaml_content[i : i + chunk_size] for i in range(0, len(yaml_content), chunk_size)] + ) for i, chunk in enumerate(chunks): - chunk_file = f"{output_file}_{i}.yaml" + chunk_file = f'{output_file}_{i}.yaml' # Write the file tree to the output YAML file with open(chunk_file, 'w', encoding='utf-8') as yaml_file: yaml_file.write(chunk) def main(): - parser = argparse.ArgumentParser( - description="Encrypts file tree to yaml file") - parser.add_argument("input_directory", help="Directory to encode") - parser.add_argument("output_yaml_file", help="Output YAML file") - parser.add_argument("--ignore", default=None, - help="Path to the ignore file") - parser.add_argument("--chunk-size", type=int, default=None, - help="Chunk size for the output YAML file") - parser.add_argument("--library", type=bool, default=False, - help="Incude the library in the output YAML file") + parser = argparse.ArgumentParser(description='Encrypts file tree to yaml file') + parser.add_argument('input_directory', help='Directory to encode') + parser.add_argument('output_yaml_file', help='Output YAML file') + parser.add_argument('--ignore', default=None, help='Path to the ignore file') + parser.add_argument( + '--chunk-size', type=int, default=None, help='Chunk size for the output YAML file' + ) + parser.add_argument( + '--library', type=bool, default=False, help='Incude the library in the output YAML file' + ) # Parse arguments args = parser.parse_args() @@ -103,11 +105,9 @@ def main(): output_yaml_file = args.output_yaml_file include_library = args.library - content = encode_file_tree_to_yaml( - directory_to_encode, ignore_file_path, include_library) - chunk_and_write_file_tree_to_yaml( - content, output_yaml_file, args.chunk_size) + content = encode_file_tree_to_yaml(directory_to_encode, ignore_file_path, include_library) + chunk_and_write_file_tree_to_yaml(content, output_yaml_file, args.chunk_size) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/scripts/import_model_manual_run.py b/scripts/import_model_manual_run.py new file mode 100644 index 0000000..2c393f7 --- /dev/null +++ b/scripts/import_model_manual_run.py @@ -0,0 +1,497 @@ +"""Run one `.sientia` model import by hand, standing in for the frontend that does not exist yet. + +**What this is.** The Angular screen plus the Spring Boot BFF will, between them, pick a `.sientia` +file, hash it, wrap the bundle password for the worker, upload the object and insert one `PENDING` +row into `public.experiment_run` before starting `import_model`. None of that exists yet. This +script does exactly those four things, against whatever environment `.env` points at, so the import +path can be exercised for real and so whoever writes the frontend has a working reference for the +wire shapes (the digest, the envelope, the object key, the row). + +**What it writes.** Two things, and it can undo both: one object under +`{IMPORT_BUNDLE_BUCKET}/{IMPORT_BUNDLE_PREFIX}` in MinIO, and one row in `public.experiment_run` in +the BFF Postgres database. Everything else it does is a read. If the workflow runs, *it* creates the +MLflow experiment, run and registered version and the MongoDB model document — cell 10 can remove +those too, behind its own flag. + +**The safety rule.** Every target comes from the environment, and this file contains no host, user, +password or key literal. A `.env` inherited from a cluster deployment can perfectly well name a +production database and a production bucket, so cell 5 and cell 6 — the only cells that write — are +gated on cell 2, where the operator has to restate the Postgres host that cell 1 printed. This is a +development tool. It is not for production, and the guard is a mitigation, not a guarantee. + +**Cells.** Run them one at a time in the VS Code interactive window, or run the module top to +bottom. Cell order is the contract: + + 1 resolve every target and print it 6 insert the PENDING row -> import_run_id + 2 confirmation guard (first write is after) 7 print the workflow input + 3 pick the bundle, compute expected_digest 8 optionally start the workflow and wait + 4 build the password envelope 9 read the outcome back + 5 upload the object, verify the stored bytes 10 undo + +**Boundaries.** This script imports nothing from `laborious.activities` or `laborious.workflows` — +it stands in for the frontend, so it must not borrow the consumer's code. The one exception is +`laborious.utils.bundle.format`, which is pure `struct`/`hashlib` over bytes and lets cell 3 print +the header of the file it is about to upload. The envelope is built here in four visible lines +rather than imported, because those four lines are what the frontend has to transliterate to +WebCrypto; `laborious/utils/import_password.py` is the decrypt counterpart. +""" + +# %% +# Cell 1 — resolve every target from the environment and print what we are pointed at. +# Reads only. No secret is printed here or anywhere else in this script. +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import Any + +from dotenv import load_dotenv + +REPO_ROOT = Path(__file__).resolve().parent.parent +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +load_dotenv(REPO_ROOT / '.env') + + +def env(name: str, default: str = '') -> str: + """The environment variable, stripped, or `default`.""" + return os.getenv(name, default).strip() + + +def env_required(name: str) -> str: + """The environment variable, or a `RuntimeError` naming it.""" + value = env(name) + if not value: + raise RuntimeError(f'{name} is not set — see .env.example') + return value + + +def _line(label: str, value: object) -> str: + """One aligned `label value` line, for output that stays readable in a terminal.""" + return f' {label:<24} {value}' + + +# Every connection target below is *required* rather than defaulted. The worker's own config falls +# back to localhost, which is right for a service that must boot; it is wrong for a script that +# writes, because a silent fallback is a write to somewhere nobody read out loud. +# +# The import log database: a second Postgres connection, independent of the POSTGRES_* one above it +# in .env. The schema and table are constants of the worker's code, never configuration. +STATUS_DB_HOST = env_required('IMPORT_STATUS_DB_HOST') +STATUS_DB_PORT = int(env_required('IMPORT_STATUS_DB_PORT')) +STATUS_DB_NAME = env_required('IMPORT_STATUS_DB_NAME') +STATUS_DB_USER = env_required('IMPORT_STATUS_DB_USER') +STATUS_DB_PASSWORD = os.getenv('IMPORT_STATUS_DB_PASSWORD', '') +STATUS_TABLE = 'public.experiment_run' + +# Where the frontend uploads the bundle. The prefix keeps the worker's own default, which is a +# format decision rather than a target. +MINIO_ENDPOINT_URL = env_required('MINIO_ENDPOINT_URL') +MINIO_ACCESS_KEY = env_required('MINIO_ACCESS_KEY') +MINIO_SECRET_KEY = env_required('MINIO_SECRET_KEY') +BUNDLE_BUCKET = env_required('IMPORT_BUNDLE_BUCKET') +_raw_prefix = env('IMPORT_BUNDLE_PREFIX', 'imported_models/') +BUNDLE_PREFIX = _raw_prefix if _raw_prefix.endswith('/') else f'{_raw_prefix}/' + +# Read-back targets. This script never writes to these; the workflow does. +MONGODB_URL = env_required('MONGODB_URL') +MONGODB_USERNAME = env('MONGODB_USERNAME') +MONGODB_PASSWORD = os.getenv('MONGODB_PASSWORD', '') +MONGODB_DATABASE = env_required('MONGODB_DATABASE') +MODELS_COLLECTION = env('IMPORT_MODELS_COLLECTION', 'models') + +MLFLOW_HOST = env_required('MLFLOW_HOST') +MLFLOW_PORT = env_required('MLFLOW_PORT') +MLFLOW_TRACKING_URI = f'{MLFLOW_HOST}:{MLFLOW_PORT}' + +TEMPORAL_HOST = env_required('TEMPORAL_HOST') +TEMPORAL_NAMESPACE = env_required('TEMPORAL_NAMESPACE') +RUNTIME = env_required('RUNTIME') +IMPORT_TASK_QUEUE = f'import_model-{RUNTIME}-queue' + +# This script's own inputs. +BUNDLE_PATH = Path( + env('IMPORT_SCRIPT_BUNDLE_PATH', 'tests/fixtures/bundle/golden_model_v3.sientia') +) +if not BUNDLE_PATH.is_absolute(): + BUNDLE_PATH = REPO_ROOT / BUNDLE_PATH +IMPORT_USERNAME = env_required('IMPORT_SCRIPT_USERNAME') + +print('resolved targets — nothing has been written yet') +print(_line('status postgres', f'{STATUS_DB_HOST}:{STATUS_DB_PORT}/{STATUS_DB_NAME}')) +print(_line('status user', STATUS_DB_USER)) +print(_line('status table', STATUS_TABLE)) +print(_line('minio endpoint', MINIO_ENDPOINT_URL)) +print(_line('minio bucket/prefix', f'{BUNDLE_BUCKET}/{BUNDLE_PREFIX}')) +print(_line('mongo', f'{MONGODB_URL}/{MONGODB_DATABASE}.{MODELS_COLLECTION}')) +print(_line('mlflow', MLFLOW_TRACKING_URI)) +print(_line('temporal', f'{TEMPORAL_HOST} ns={TEMPORAL_NAMESPACE}')) +print(_line('task queue', IMPORT_TASK_QUEUE)) +print(_line('bundle', BUNDLE_PATH)) +print(_line('username', IMPORT_USERNAME)) +print() +print(f'to continue, set the cell 2 confirmation to: {STATUS_DB_HOST}') + +# %% +# Cell 2 — the confirmation guard. Cells 5 and 6 write; this is the last cell before them. +# +# Restate the Postgres host cell 1 printed. Either set IMPORT_SCRIPT_CONFIRM_TARGET in .env, or +# replace the right-hand side below with the value cell 1 told you. It is a read-then-restate on +# purpose: a boolean flag can be flipped without ever looking at what it is pointed at. +CONFIRM_TARGET = env('IMPORT_SCRIPT_CONFIRM_TARGET') + +if CONFIRM_TARGET != STATUS_DB_HOST: + raise RuntimeError( + 'refusing to write: the confirmation does not match the resolved status database host. ' + f'confirmation={CONFIRM_TARGET!r}, resolved IMPORT_STATUS_DB_HOST={STATUS_DB_HOST!r}. ' + 'Read the targets cell 1 printed, then restate the host here.' + ) + +print(f'target confirmed: {STATUS_DB_HOST} — the write cells may run') + +# %% +# Cell 3 — pick the bundle, compute the digest the frontend must send, print its header. +# +# `expected_digest` is the SHA-256 of the exact bytes that get uploaded, computed the way the +# frontend computes it: over the file, not over anything derived from it. The header fields printed +# here are plaintext at the front of the file — reading them decrypts nothing. +import hashlib + +from laborious.utils.bundle.format import DIGEST_OFFSET, HEADER_SIZE, parse_header + +bundle_bytes = BUNDLE_PATH.read_bytes() +expected_digest = hashlib.sha256(bundle_bytes).hexdigest() +object_name = BUNDLE_PATH.name +object_key = f'{BUNDLE_PREFIX}{object_name}' + +header = parse_header(bundle_bytes[:HEADER_SIZE]) + +print(_line('file', BUNDLE_PATH)) +print(_line('size', f'{len(bundle_bytes)} bytes')) +print(_line('expected_digest', expected_digest)) +print(_line('object key', f'{BUNDLE_BUCKET}/{object_key}')) +print(_line('format version', header.format_version)) +print(_line('aead / kdf id', f'{header.aead_id} / {header.kdf_id}')) +print(_line('kdf memlimit', f'{header.kdf_memlimit_bytes} bytes')) +print(_line('kdf opslimit', header.kdf_opslimit)) +print(_line('kdf parallelism', header.kdf_parallelism)) +print(_line('salt', header.salt.hex())) +print(_line('ciphertext digest', header.ciphertext_digest.hex())) +print(_line('chunk stream from', f'offset {DIGEST_OFFSET}')) + +# %% +# Cell 4 — build the password envelope, exactly as the frontend must build it. +# +# Wire shape: base64( nonce(12) || ciphertext || GCM tag(16) ) under AES-256-GCM with the 32-byte +# key that `IMPORT_PASSWORD_KEY` carries base64-encoded. The four lines below are the whole of it, +# written out rather than imported so the Angular side can transliterate them to WebCrypto: +# +# const k = await crypto.subtle.importKey('raw', keyBytes, 'AES-GCM', false, ['encrypt']); +# const nonce = crypto.getRandomValues(new Uint8Array(12)); +# const sealed = await crypto.subtle.encrypt({name: 'AES-GCM', iv: nonce}, k, passwordBytes); +# const envelope = base64(concat(nonce, new Uint8Array(sealed))); +# +# WebCrypto appends the GCM tag to the ciphertext, and so does `cryptography`'s AESGCM, so the two +# produce byte-identical payloads. The decrypt counterpart the worker runs is +# `laborious/utils/import_password.py` (`decrypt_password_envelope`). +# +# Neither the key nor the password nor the plaintext is printed — only the envelope's length. +import base64 +import secrets + +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +envelope_key = base64.b64decode(env_required('IMPORT_PASSWORD_KEY'), validate=True) +nonce = secrets.token_bytes(12) +sealed = AESGCM(envelope_key).encrypt( + nonce, env_required('IMPORT_SCRIPT_BUNDLE_PASSWORD').encode('utf-8'), None +) +password_envelope = base64.b64encode(nonce + sealed).decode('ascii') + +print(_line('envelope key', f'{len(envelope_key)} bytes (not printed)')) +print(_line('envelope', f'{len(password_envelope)} base64 characters (not printed)')) + +# %% +# Cell 5 — FIRST WRITE. Upload the object, then prove the stored bytes are the ones we hashed. +# +# The re-read is the point: `expected_digest` is a promise about what is in the bucket, and the +# worker's gate 1 holds the upload to it. If the digest of the stored object differs, the frontend +# contract is already broken and there is no reason to insert a row. +import io +from urllib.parse import urlparse + +from minio import Minio + +_endpoint = urlparse(MINIO_ENDPOINT_URL) +minio_client = Minio( + _endpoint.netloc or MINIO_ENDPOINT_URL, + access_key=MINIO_ACCESS_KEY, + secret_key=MINIO_SECRET_KEY, + secure=_endpoint.scheme == 'https', +) + +minio_client.put_object( + BUNDLE_BUCKET, + object_key, + io.BytesIO(bundle_bytes), + length=len(bundle_bytes), + content_type='application/octet-stream', +) + +stat = minio_client.stat_object(BUNDLE_BUCKET, object_key) +_response = minio_client.get_object(BUNDLE_BUCKET, object_key) +try: + stored_bytes = _response.read() +finally: + _response.close() + _response.release_conn() + +stored_digest = hashlib.sha256(stored_bytes).hexdigest() +if stored_digest != expected_digest: + raise RuntimeError( + f'the stored object does not hash to the digest we computed: stored {stored_digest}, ' + f'expected {expected_digest} — do not insert a row for this upload' + ) + +print(_line('uploaded', f'{BUNDLE_BUCKET}/{object_key}')) +print(_line('reported size', f'{stat.size} bytes')) +print(_line('etag', stat.etag)) +print(_line('re-read digest', f'{stored_digest} (matches expected_digest)')) + +# %% +# Cell 6 — SECOND WRITE. Insert the import log row: the BFF's half of the contract. +# +# `URL.create` rather than a URL string, so a password containing `@ : /` survives. `experiment_name` +# is a placeholder: the column is NOT NULL and the frontend cannot see inside an encrypted bundle, +# so the worker overwrites it with the bundle's own experiment name as soon as the bundle is open +# (`record_import_names`). `file_name` is the object key, because the worker compares the two and +# refuses the import at the `received` step when they disagree. +import json +from datetime import UTC, datetime + +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL + +status_engine = create_engine( + URL.create( + 'postgresql', + username=STATUS_DB_USER, + password=STATUS_DB_PASSWORD, + host=STATUS_DB_HOST, + port=STATUS_DB_PORT, + database=STATUS_DB_NAME, + ), + pool_pre_ping=True, +) + +request_data = { + 'expected_digest': expected_digest, + 'bucket': BUNDLE_BUCKET, + 'object_key': object_key, + 'file_name': object_name, + 'source': 'scripts/import_model_manual_run.py', +} + +_now = datetime.now(UTC).replace(tzinfo=None) +_placeholder_experiment = f'import-{Path(object_name).stem}'[:50] + +INSERT_ROW_SQL = text( + 'INSERT INTO public.experiment_run ' + '(experiment_name, username, status, created_at, updated_at, ' + ' bucket_name, file_name, request_data, run_type) ' + 'VALUES (:experiment_name, :username, :status, :created_at, :updated_at, ' + ' :bucket_name, :file_name, CAST(:request_data AS JSONB), :run_type) ' + 'RETURNING id' +) + +with status_engine.begin() as connection: + import_run_id = connection.execute( + INSERT_ROW_SQL, + { + 'experiment_name': _placeholder_experiment, + 'username': IMPORT_USERNAME, + 'status': 'PENDING', + 'created_at': _now, + 'updated_at': _now, + 'bucket_name': BUNDLE_BUCKET, + 'file_name': object_key, + 'request_data': json.dumps(request_data), + 'run_type': 'IMPORT', + }, + ).scalar_one() + +print(_line('import_run_id', import_run_id)) +print(_line('status / run_type', 'PENDING / IMPORT')) +print(_line('experiment_name', f'{_placeholder_experiment} (the worker overwrites this)')) +print(_line('file_name', object_key)) + +# %% +# Cell 7 — the workflow input, ready to paste into the Temporal UI. +# +# These five keys are the whole contract. `bucket` is optional — the worker falls back to +# IMPORT_BUNDLE_BUCKET — and is sent explicitly so the printed input is self-describing. No model +# name is sent: the worker takes it from the bundle's own metadata, verbatim. +workflow_input = { + 'import_run_id': import_run_id, + 'bucket': BUNDLE_BUCKET, + 'object_key': object_key, + 'expected_digest': expected_digest, + 'password_envelope': password_envelope, +} + +print(f'workflow: import_model task queue: {IMPORT_TASK_QUEUE}') +print(json.dumps(workflow_input, indent=2)) + +# %% +# Cell 8 — optionally start the workflow and wait for it. +# +# Skipping this cell is normal: the operator may prefer to start the run from the Temporal UI with +# the JSON cell 7 printed. Cell 9 works either way, because it looks the outcome up by row id. +# +# The coroutine runs on a loop of its own in a worker thread rather than through `asyncio.run`, so +# this cell behaves the same in the interactive window (which already has a running loop) as it +# does when the module is executed top to bottom (which does not). +START_THE_WORKFLOW = False + +if START_THE_WORKFLOW: + import asyncio + from concurrent.futures import ThreadPoolExecutor + + from temporalio.client import Client + + async def start_and_wait() -> Any: + """Start `import_model` on the import queue and return its result.""" + client = await Client.connect(TEMPORAL_HOST, namespace=TEMPORAL_NAMESPACE) + handle = await client.start_workflow( + 'import_model', + workflow_input, + id=f'manual-import-{import_run_id}', + task_queue=IMPORT_TASK_QUEUE, + ) + print(_line('workflow id', handle.id)) + print(_line('run id', handle.result_run_id)) + return await handle.result() + + with ThreadPoolExecutor(max_workers=1) as _pool: + workflow_result = _pool.submit(asyncio.run, start_and_wait()).result() + print(json.dumps(workflow_result, indent=2, default=str)) +else: + print('not started — paste the input from cell 7 into the Temporal UI, or set') + print(f'START_THE_WORKFLOW = True and re-run this cell. Task queue: {IMPORT_TASK_QUEUE}') + +# %% +# Cell 9 — read the outcome back from all three places it lands. +# +# The row is authoritative for the verdict; MLflow and MongoDB are what the verdict is about. Each +# block is labelled, so the output can be pasted into a ticket as it is. +print('--- import log row ---') +READ_ROW_SQL = text( + 'SELECT id, experiment_name, run_name, status, error_message, run_type, ' + ' bucket_name, file_name, created_at, updated_at, ' + ' request_data, orchestrator_response_data ' + 'FROM public.experiment_run WHERE id = :id' +) + +with status_engine.begin() as connection: + _result = connection.execute(READ_ROW_SQL, {'id': import_run_id}) + row = _result.mappings().one_or_none() + +if row is None: + print(f' no row with id {import_run_id}') +else: + for _column, _value in row.items(): + print(_line(_column, _value)) + +print() +print('--- mlflow ---') +import mlflow + +os.environ['MLFLOW_TRACKING_USERNAME'] = env('MLFLOW_USERNAME') +os.environ['MLFLOW_TRACKING_PASSWORD'] = os.getenv('MLFLOW_PASSWORD', '') +mlflow.set_tracking_uri(MLFLOW_TRACKING_URI) +mlflow_client = mlflow.tracking.MlflowClient() + +_experiment_name = row['experiment_name'] if row is not None else '' +_experiment = mlflow_client.get_experiment_by_name(_experiment_name) if _experiment_name else None +if _experiment is None: + print(f' no experiment named {_experiment_name!r}') +else: + print(_line('experiment', f'{_experiment.name} ({_experiment.experiment_id})')) + for _run in mlflow_client.search_runs([_experiment.experiment_id]): + print(_line('run', f'{_run.info.run_name} ({_run.info.run_id}) {_run.info.status}')) + for _artifact in mlflow_client.list_artifacts(_run.info.run_id): + print(_line(' artifact', _artifact.path)) + +# The run name is `import-{model_name}-v{version}` (`derive_run_name`), so the registered model +# name is what sits between the prefix and the version suffix. +_run_name = (row['run_name'] if row is not None else '') or '' +_registered = _run_name.removeprefix('import-').rsplit('-v', 1)[0] if _run_name else '' +if _registered: + try: + for _version in mlflow_client.search_model_versions(f"name='{_registered}'"): + print( + _line( + 'version', + f'{_version.name} v{_version.version} stage={_version.current_stage} ' + f'run={_version.run_id}', + ) + ) + print(_line(' source', _version.source)) + except Exception as error: + # A read-back cell reports what it could not read; it does not fail the inspection. + print(f' registry read failed: {type(error).__name__}: {error}') +else: + print(' no run_name on the row yet, so no registered model to look up') + +print() +print('--- mongodb model document ---') +from pymongo import MongoClient + +_credentials = f'{MONGODB_USERNAME}:{MONGODB_PASSWORD}@' if MONGODB_USERNAME else '' +mongo_client: MongoClient = MongoClient(f'mongodb://{_credentials}{MONGODB_URL}') +_collection = mongo_client[MONGODB_DATABASE][MODELS_COLLECTION] +for _document in _collection.find({}, {'_id': False}).sort('id', 1): + print(_line('document', _document)) + +# %% +# Cell 10 — undo. +# +# The object and the row always: they are what this script created, and leaving them behind leaves a +# PENDING import nobody will ever run. What MLflow and MongoDB hold is left alone unless +# DELETE_PROVISIONING is set, because after a successful run that is the thing you wanted to look at. +DELETE_PROVISIONING = False + +try: + minio_client.remove_object(BUNDLE_BUCKET, object_key) + print(_line('deleted object', f'{BUNDLE_BUCKET}/{object_key}')) +except Exception as error: + # Undo reports what it could not remove and carries on to the row. + print(f' could not delete the object: {type(error).__name__}: {error}') + +with status_engine.begin() as connection: + _deleted = connection.execute( + text('DELETE FROM public.experiment_run WHERE id = :id'), + {'id': import_run_id}, + ).rowcount +print(_line('deleted rows', _deleted)) + +if DELETE_PROVISIONING: + if _experiment is not None: + mlflow_client.delete_experiment(_experiment.experiment_id) + print(_line('deleted experiment', _experiment.experiment_id)) + if _registered: + try: + mlflow_client.delete_registered_model(_registered) + print(_line('deleted model', _registered)) + except Exception as error: + # Same rule as above: report and carry on to the document. + print(f' could not delete the registered model: {type(error).__name__}: {error}') + _removed = _collection.delete_many({'name': _registered}).deleted_count + print(_line('deleted documents', _removed)) +else: + print(' provisioning kept — set DELETE_PROVISIONING = True to remove it too') + +status_engine.dispose() +mongo_client.close() diff --git a/inter_arrival.py b/scripts/inter_arrival.py similarity index 92% rename from inter_arrival.py rename to scripts/inter_arrival.py index bce9cee..f3e2cef 100644 --- a/inter_arrival.py +++ b/scripts/inter_arrival.py @@ -1,11 +1,12 @@ # %% # Load logs.txt -with open('logs.txt', 'r') as file: +with open('logs.txt') as file: lines = file.readlines() # %% import re + # Grep "inter-arrival_s=number" with regex intervals = [] for line in lines: @@ -17,6 +18,7 @@ for line in lines: print(intervals) # %% import matplotlib.pyplot as plt + plt.plot(intervals) plt.ylabel('Inter-arrival time (s)') plt.xlabel('Sample') diff --git a/model_convert.ipynb b/scripts/model_convert.ipynb similarity index 54% rename from model_convert.ipynb rename to scripts/model_convert.ipynb index 42f6c3d..d04f63e 100644 --- a/model_convert.ipynb +++ b/scripts/model_convert.ipynb @@ -9,6 +9,7 @@ "source": [ "import csv\n", "\n", + "\n", "def csv_to_tag_lists(csv_path: str) -> dict:\n", " read_tags = []\n", " write_tags = []\n", @@ -19,48 +20,48 @@ " except Exception:\n", " return None\n", "\n", - " with open(csv_path, newline=\"\", encoding=\"utf-8\") as f:\n", + " with open(csv_path, newline='', encoding='utf-8') as f:\n", " reader = csv.DictReader(f)\n", " for row in reader:\n", " # Basic normalization\n", - " op = (row.get(\"operation\") or \"\").strip()\n", + " op = (row.get('operation') or '').strip()\n", "\n", - " if op == \"READ\":\n", + " if op == 'READ':\n", " # Build common tag payload with required mappings\n", " tag = {\n", - " \"server_id\": \"1\",\n", - " \"tag_address\": row.get(\"opc_tag\"),\n", - " \"tag_name\": row.get(\"name\"),\n", - " \"data_range\": [to_float(row.get(\"min_value\")), to_float(row.get(\"max_value\"))],\n", - " \"aggr_func\": row.get(\"aggregation_func\").lower(),\n", + " 'server_id': '1',\n", + " 'tag_address': row.get('opc_tag'),\n", + " 'tag_name': row.get('name'),\n", + " 'data_range': [to_float(row.get('min_value')), to_float(row.get('max_value'))],\n", + " 'aggr_func': row.get('aggregation_func').lower(),\n", " # keep other fields with their original names\n", - " \"frequency\": row.get(\"frequency\"),\n", - " \"local\": row.get(\"local\"),\n", - " \"area\": row.get(\"area\"),\n", - " \"description\": row.get(\"description\"),\n", + " 'frequency': row.get('frequency'),\n", + " 'local': row.get('local'),\n", + " 'area': row.get('area'),\n", + " 'description': row.get('description'),\n", " }\n", "\n", " read_tags.append(tag)\n", "\n", " else:\n", " tag = {\n", - " \"server_id\": \"1\",\n", - " \"addr\": row.get(\"opc_tag\"),\n", - " \"tag_name\": row.get(\"name\"),\n", - " \"local\": row.get(\"local\"),\n", - " \"area\": row.get(\"area\"),\n", - " \"description\": row.get(\"description\"),\n", + " 'server_id': '1',\n", + " 'addr': row.get('opc_tag'),\n", + " 'tag_name': row.get('name'),\n", + " 'local': row.get('local'),\n", + " 'area': row.get('area'),\n", + " 'description': row.get('description'),\n", " }\n", - " \n", - " if op == \"WRITE_PREDICTION\":\n", - " tag[\"type\"] = \"prediction\"\n", + "\n", + " if op == 'WRITE_PREDICTION':\n", + " tag['type'] = 'prediction'\n", " write_tags.append(tag)\n", - " elif op == \"WRITE_CONFIDENCE\":\n", - " tag[\"type\"] = \"confidence\"\n", + " elif op == 'WRITE_CONFIDENCE':\n", + " tag['type'] = 'confidence'\n", " write_tags.append(tag)\n", " # ignore any other operation values silently\n", "\n", - " return {\"read_tags\": read_tags, \"write_tags\": write_tags}" + " return {'read_tags': read_tags, 'write_tags': write_tags}" ] }, { @@ -72,12 +73,12 @@ "source": [ "import json\n", "\n", - "file_names = [\"Courier - Página1.csv\"]\n", + "file_names = ['Courier - Página1.csv']\n", "\n", "for file_name in file_names:\n", - " write_file = file_name.replace(\".csv\", \".json\")\n", + " write_file = file_name.replace('.csv', '.json')\n", "\n", - " with open(write_file, \"w\", encoding=\"utf-8\") as f:\n", + " with open(write_file, 'w', encoding='utf-8') as f:\n", " json.dump(csv_to_tag_lists(file_name), f, indent=2, ensure_ascii=False)" ] } diff --git a/run_coverage.sh b/scripts/run_coverage.sh similarity index 100% rename from run_coverage.sh rename to scripts/run_coverage.sh diff --git a/run_local.sh b/scripts/run_local.sh similarity index 100% rename from run_local.sh rename to scripts/run_local.sh diff --git a/tests.ipynb b/tests.ipynb deleted file mode 100644 index 8926385..0000000 --- a/tests.ipynb +++ /dev/null @@ -1,938 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "b10e5c25", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "\n", - "# Set random seed for reproducibility\n", - "rng = np.random.default_rng(42)\n", - "\n", - "# Generate random walks starting at 0\n", - "counter = np.zeros(300)\n", - "rollout = np.zeros(300)\n", - "\n", - "# Generate random steps between -1 and 1\n", - "counter_steps = rng.uniform(-1, 1, 299)\n", - "rollout_steps = rng.uniform(-1, 1, 299)\n", - "\n", - "print(counter_steps)\n", - "print(rollout_steps)\n", - "\n", - "# Calculate cumulative sum and scale to -100 to 100 range\n", - "for i in range(1, 300):\n", - " counter[i] = counter[i-1] + counter_steps[i-1]\n", - " rollout[i] = rollout[i-1] + rollout_steps[i-1]\n", - "\n", - "# normalize values between -100 and 100, lowest value is -100, highest value is 100\n", - "counter = (counter - min(counter)) / (max(counter) - min(counter)) * 200 - 100\n", - "rollout = (rollout - min(rollout)) / (max(rollout) - min(rollout)) * 200 - 100\n", - "\n", - "# Create DataFrame\n", - "df = pd.DataFrame({\n", - " 'Counter': counter,\n", - " 'Rollout': rollout,\n", - " 'CounterPlusRollout': counter + rollout\n", - "})\n", - "\n", - "# add a timestamp column\n", - "df['Timestamp'] = pd.date_range(start='2025-01-01', periods=300, freq='1s')\n", - "\n", - "# Save to CSV\n", - "df.to_csv('random_walks.csv', index=False)\n", - "\n", - "# Display first few rows\n", - "print(df.head())\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c61be7ab", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "\n", - "# Set random seed for reproducibility\n", - "rng = np.random.default_rng(42)\n", - "\n", - "size = 50\n", - "\n", - "# Generate random walks starting at 0\n", - "counter = np.zeros(size)\n", - "rollout = np.zeros(size)\n", - "square = np.zeros(size)\n", - "\n", - "# Generate random steps between -1 and 1\n", - "counter_steps = rng.uniform(-1, 1, size-1)\n", - "rollout_steps = rng.uniform(-1, 1, size-1)\n", - "square_steps = rng.uniform(-1, 1, size-1)\n", - "\n", - "print(counter_steps)\n", - "print(rollout_steps)\n", - "\n", - "# Calculate cumulative sum and scale to -100 to 100 range\n", - "for i in range(1, size):\n", - " counter[i] = counter[i-1] + counter_steps[i-1]\n", - " rollout[i] = rollout[i-1] + rollout_steps[i-1]\n", - " square[i] = square[i-1] + square_steps[i-1]\n", - "\n", - "# normalize values between -100 and 100, lowest value is -100, highest value is 100\n", - "counter = (counter - min(counter)) / (max(counter) - min(counter)) * 200 - 100\n", - "rollout = (rollout - min(rollout)) / (max(rollout) - min(rollout)) * 200 - 100\n", - "square = (square - min(square)) / (max(square) - min(square)) * 200 - 100\n", - "\n", - "# Create DataFrame\n", - "df = pd.DataFrame({\n", - " 'Counter': counter,\n", - " 'Rollout': rollout,\n", - " 'Square': square\n", - "})\n", - "\n", - "# add a timestamp column\n", - "df['Timestamp'] = pd.date_range(start='2025-01-01', periods=size, freq='5s')\n", - "\n", - "# Save to CSV\n", - "df.to_csv('random_walks_demo.csv', index=False)\n", - "\n", - "# Display first few rows\n", - "print(df.head())\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e7c8eeb1", - "metadata": {}, - "outputs": [], - "source": [ - "from pandas import DataFrame\n", - "\n", - "a = DataFrame({\n", - " \"a\": {\"2025-01-01\": 1, \"2025-01-02\": 2, \"2025-01-03\": 3},\n", - " \"b\": {\"2025-01-01\": 4, \"2025-01-02\": 5, \"2025-01-03\": 6},\n", - "})\n", - "\n", - "a.index.name = \"timestamp\"\n", - "\n", - "display(a)\n", - "\n", - "print(\"a\" in a.columns)\n", - "\n", - "b = a.tail(1)\n", - "\n", - "display(b)\n", - "\n", - "print(len(a))\n", - "print(len(b))\n", - "print(b.size)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f3374174", - "metadata": {}, - "outputs": [], - "source": [ - "from pandas import DataFrame\n", - "\n", - "data = DataFrame({\n", - " \"a\": {\"2025-01-01\": 1, \"2025-01-02\": 2, \"2025-01-03\": 3},\n", - " \"b\": {\"2025-01-01\": 4, \"2025-01-02\": 5, \"2025-01-03\": 6},\n", - "})\n", - "\n", - "index = data.index\n", - "\n", - "# Get type of first element of index\n", - "index_type = type(index[0])\n", - "\n", - "print(index_type)\n", - "\n", - "# Check if all in index are of the same type\n", - "if all(isinstance(i, index_type) for i in index):\n", - " print(\"All elements in index are of the same type\")\n", - "else:\n", - " print(\"Elements in index are of different types\")\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "40e72c60", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1fbb3788", - "metadata": {}, - "outputs": [], - "source": [ - "from unittest.mock import MagicMock\n", - "from asyncua.ua.uaerrors import BadAlreadyExists\n", - "\n", - "mock1 = MagicMock(\n", - " side_effect = Exception(\"test\")\n", - ")\n", - "\n", - "mock2 = MagicMock(\n", - " side_effect = BadAlreadyExists(\"test\")\n", - ")\n", - "\n", - "try:\n", - " mock1()\n", - "except ValueError as e:\n", - " try:\n", - " mock2()\n", - " except BadAlreadyExists as e:\n", - " print(e)\n", - "except Exception as e:\n", - " print(e)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "771ab4ee", - "metadata": {}, - "outputs": [], - "source": [ - "from pandas import DataFrame, merge\n", - "\n", - "retrain_dataset = DataFrame({\n", - " 'a': {'2025-01-01': 1, '2025-01-02': 2, '2025-01-03': 3},\n", - " 'b': {'2025-01-01': 4, '2025-01-02': 5, '2025-01-03': 6},\n", - " 'c': {'2025-01-01': 7, '2025-01-02': 8, '2025-01-03': 9},\n", - "})\n", - "\n", - "prediction_data = DataFrame({\n", - " 'prediction': {'1': 1, '2': 2, '3': 3},\n", - "})\n", - "\n", - "prediction_data.index = retrain_dataset.index\n", - "\n", - "prediction_data = merge(\n", - " retrain_dataset, prediction_data, left_index=True, right_index=True, how='left')\n", - "\n", - "prediction_data.rename(columns={'c': 'target'}, inplace=True)\n", - "\n", - "prediction_data['timestamp'] = prediction_data.index\n", - "\n", - "prediction_data.reset_index(drop=True, inplace=True)\n", - "\n", - "display(prediction_data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "486b95b3", - "metadata": {}, - "outputs": [], - "source": [ - "from pandas import DataFrame\n", - "\n", - "data = DataFrame()\n", - "\n", - "display(data.to_dict(orient='records'))\n", - "\n", - "data = DataFrame({\n", - " \"a\": {\"2025-01-01\": 1, \"2025-01-02\": 2, \"2025-01-03\": 3},\n", - " \"b\": {\"2025-01-01\": 4, \"2025-01-02\": 5, \"2025-01-03\": 6},\n", - "})\n", - "\n", - "display(data)\n", - "\n", - "data_list = data.to_dict('split')\n", - "\n", - "display(data_list)\n", - "\n", - "data_rec = DataFrame.from_dict(data_list, orient='index')\n", - "\n", - "display(data_rec)\n", - "\n", - "data.shape[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d67d551f", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import shutil\n", - "import mlflow\n", - "from mlflow.tracking import MlflowClient\n", - "from rich.console import Console\n", - "import sys\n", - "\n", - "if \"src\" not in sys.path:\n", - " sys.path.insert(0, \"src\")\n", - "\n", - "console = Console()\n", - "\n", - "os.environ[\"MLFLOW_TRACKING_URI\"] = \"http://localhost:35785/\"\n", - "os.environ[\"MLFLOW_TRACKING_USERNAME\"] = \"aignosi\"\n", - "os.environ[\"MLFLOW_TRACKING_PASSWORD\"] = \"1L0FP50j3ncp123\"\n", - "\n", - "client = MlflowClient()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "98bdd6af", - "metadata": {}, - "outputs": [], - "source": [ - "from pandas import DataFrame, read_json\n", - "\n", - "# Load from data file (json)\n", - "data = read_json('data.json')\n", - "\n", - "column_order = [\n", - " 'CI-J3J01S1', 'CI-J3P01T1A', 'CI-J3P03S1', 'CI-W3A05F1',\n", - " 'CI-W3A50A1', 'CI-W3A50A2', 'CI-W3A50A3', 'CI-W3A50P1', 'CI-W3A50T1',\n", - " 'CI-W3A55P1', 'CI-W3A55T1', 'CI-W3A65_Cl', 'CI-W3A65_SO3', 'CI-W3A71P1',\n", - " 'CI-W3A71P2', 'CI-W3A71P3', 'CI-W3E01F1', 'CI-W3K01S1', 'CI-W3K01T1',\n", - " 'CI-W3K01T2', 'CI-W3K01T4', 'CI-W3K14P1', 'CI-W3P17S1', 'CI-W3V04P1',\n", - " 'CI-W3V04P3', 'CI-W3V21F1', 'CI-W3V21P1', 'CI-W3V30F1', 'CI-W3V33P1',\n", - " 'CI-W3W01G1', 'CI-W3W01P1', 'CI-W3W01P2', 'CI-W3W03I1', 'CI-W3W03S1',\n", - " 'CI-W3_C3S', 'CI-W3_CAO', 'CI-W3_MA', 'CI-W3_MS', 'CI-W3_PL']\n", - "\n", - "ordered_data = data[column_order]\n", - "\n", - "display(ordered_data.head(3))\n", - "display(data.head(3))\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d7e73c30", - "metadata": {}, - "outputs": [], - "source": [ - "import mlflow\n", - "\n", - "model_name = \"vcm-o2-vanilla-ice\"\n", - "\n", - "model = mlflow.pyfunc.load_model(f'models:/{model_name}/production')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4e5bae06", - "metadata": {}, - "outputs": [], - "source": [ - "model.predict(ordered_data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cd1bfaa0", - "metadata": {}, - "outputs": [], - "source": [ - "# Download pkl file from mlflow\n", - "model_uri = f'models:/{model_name}/production'\n", - "model_path = mlflow.artifacts.download_artifacts(model_uri, dst_path='./backup-model')\n", - "\n", - "print(model_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0ef9c913", - "metadata": {}, - "outputs": [], - "source": [ - "file_path = f'{model_path}/artifacts/xgboost_model.pkl'\n", - "\n", - "# Verificar os primeiros bytes do arquivo\n", - "with open(file_path, 'rb') as f:\n", - " first_bytes = f.read(10)\n", - " print(first_bytes)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "6bbb8cf7", - "metadata": {}, - "outputs": [], - "source": [ - "from pandas import DataFrame, to_datetime\n", - "from pandas import DataFrame, to_datetime\n", - "from sientia_do.temporal.activities.postgres import Postgres\n", - "from unittest.mock import MagicMock, AsyncMock\n", - "import matplotlib.pyplot as plt\n", - "\n", - "postgres = Postgres(\n", - " host=\"localhost\",\n", - " port=5432,\n", - " dbname=\"sientia\",\n", - " user=\"sientia\",\n", - " password=\"sientia\",\n", - " min_connections=1,\n", - " max_connections=10,\n", - " logger=MagicMock(),\n", - " notification_handler=MagicMock(),\n", - " metrics_controller=AsyncMock()\n", - ")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "74378ac4", - "metadata": {}, - "outputs": [], - "source": [ - "from pandas import DataFrame, merge\n", - "\n", - "async def load_data(period_hours, model_id, variable):\n", - " query = f\"\"\"\n", - " select p.prediction, ld.variable, ld.value, p.\"timestamp\" from sientia_data.predictions p\n", - " join sientia_data.laborious_data ld on p.\"timestamp\" = ld.\"timestamp\"\n", - " where p.model_id = '{model_id}' and variable in ('{variable}', 'CI-W3X21IN')\n", - " and ld.model_id = '4' and p.\"timestamp\" >= NOW() - INTERVAL '{period_hours} HOUR'\n", - " order by p.created_at;\n", - " \"\"\"\n", - "\n", - " data = DataFrame(await postgres.load_custom_query(\n", - " {\n", - " \"query\": query,\n", - " \"metadata\": {},\n", - " \"datetime_columns\": [\"timestamp\"],\n", - " }\n", - " ))\n", - "\n", - " data.sort_values(by='timestamp', inplace=True)\n", - "\n", - " # print(data.head())\n", - "\n", - " data.drop_duplicates(subset=['timestamp', 'prediction', 'variable'], inplace=True, keep='first')\n", - "\n", - " data = data.pivot(index=['timestamp', 'prediction'], columns='variable', values='value').reset_index()\n", - "\n", - " # print(data.head())\n", - "\n", - " data.rename(columns={variable: 'value', 'CI-W3X21IN': 'sensor_in'}, inplace=True)\n", - "\n", - " data.sort_values(by='timestamp', inplace=True)\n", - "\n", - " data['timestamp'] = to_datetime(data['timestamp'])\n", - "\n", - " return data\n", - "\n", - "async def load_transformed_data(period_hours, model_id, variable):\n", - " query = f\"\"\"\n", - " WITH original_data AS (\n", - " SELECT * \n", - " FROM sientia_data.laborious_data \n", - " WHERE model_id = '4' \n", - " AND \\\"timestamp\\\" >= NOW() - INTERVAL '{period_hours} hours'\n", - " AND variable in ('{variable}', 'CI-W3X21IN')\n", - " ),\n", - " ci1_with_prev AS (\n", - " SELECT \n", - " *,\n", - " (\n", - " SELECT value\n", - " FROM sientia_data.laborious_data\n", - " WHERE model_id = '4' \n", - " AND variable = '{variable}'\n", - " AND \\\"timestamp\\\" < od.\\\"timestamp\\\"\n", - " ORDER BY \\\"timestamp\\\" DESC\n", - " LIMIT 1\n", - " ) as prev_value\n", - " FROM original_data od\n", - " WHERE variable = '{variable}'\n", - " ),\n", - " ci1_with_predictions AS (\n", - " SELECT \n", - " ci.*,\n", - " p.prediction as current_prediction,\n", - " (\n", - " SELECT prediction\n", - " FROM sientia_data.predictions\n", - " WHERE model_id = '{model_id}'\n", - " AND \\\"timestamp\\\" < ci.\\\"timestamp\\\"\n", - " ORDER BY \\\"timestamp\\\" DESC\n", - " LIMIT 1\n", - " ) as prev_prediction\n", - " FROM ci1_with_prev ci\n", - " LEFT JOIN sientia_data.predictions p \n", - " ON p.model_id = '{model_id}' \n", - " AND p.\\\"timestamp\\\" = ci.\\\"timestamp\\\"\n", - " )\n", - " SELECT * \n", - " FROM original_data\n", - " UNION ALL\n", - " SELECT\n", - " model_id,\n", - " variable || '_AR' as variable,\n", - " COALESCE(prev_value, 0) as value,\n", - " \\\"timestamp\\\",\n", - " created_at\n", - " FROM ci1_with_prev\n", - " UNION ALL\n", - " SELECT\n", - " model_id,\n", - " variable || '_PRa' as variable,\n", - " COALESCE(current_prediction, 0) as value,\n", - " \\\"timestamp\\\",\n", - " created_at\n", - " FROM ci1_with_predictions\n", - " UNION ALL\n", - " SELECT\n", - " model_id,\n", - " variable || '_PR' as variable,\n", - " COALESCE(prev_prediction, 0) as value,\n", - " \\\"timestamp\\\",\n", - " created_at\n", - " FROM ci1_with_predictions\n", - " ORDER BY \\\"timestamp\\\" DESC, variable;\n", - "\n", - " \"\"\"\n", - "\n", - " data = DataFrame(await postgres.load_custom_query(\n", - " {\n", - " \"query\": query,\n", - " \"metadata\": {},\n", - " \"datetime_columns\": [\"timestamp\"],\n", - " }\n", - " ))\n", - "\n", - " data.sort_values(by='timestamp', inplace=True)\n", - "\n", - " data.drop_duplicates(subset=['timestamp', 'variable'], inplace=True, keep='first')\n", - "\n", - " data = data.pivot(index=['timestamp'], columns='variable', values='value').reset_index()\n", - "\n", - " data.rename(columns={\n", - " 'CI-W3X21IN': 'sensor_in',\n", - " variable: 'current_value',\n", - " variable + '_AR': 'previous_value',\n", - " variable + '_PRa': 'current_prediction',\n", - " variable + '_PR': 'previous_prediction'\n", - " }, inplace=True)\n", - "\n", - "\n", - " query_transformed = f\"\"\"\n", - " select td.\"timestamp\", td.value, td.variable from sientia_data.transformed_data td\n", - " where\n", - " td.model_id = '{model_id}' and\n", - " td.variable = '{variable}_AR' and\n", - " td.\"timestamp\" >= NOW() - INTERVAL '{period_hours} hours';\n", - " \"\"\"\n", - "\n", - " transformed_data = DataFrame(await postgres.load_custom_query(\n", - " {\n", - " \"query\": query_transformed,\n", - " \"metadata\": {},\n", - " \"datetime_columns\": [\"timestamp\"],\n", - " }\n", - " ))\n", - "\n", - " transformed_data.sort_values(by='timestamp', inplace=True)\n", - "\n", - " transformed_data.drop_duplicates(subset=['timestamp', 'variable'], inplace=True, keep='first')\n", - "\n", - " transformed_data = transformed_data.pivot(index=['timestamp'], columns='variable', values='value').reset_index()\n", - "\n", - " transformed_data.rename(columns={variable + '_AR': 'selected_value'}, inplace=True)\n", - "\n", - " data = merge(\n", - " data,\n", - " transformed_data,\n", - " left_on='timestamp',\n", - " right_on='timestamp',\n", - " how='left'\n", - " )\n", - "\n", - "\n", - " data['timestamp'] = to_datetime(data['timestamp'])\n", - "\n", - " return data\n", - "\n", - "# display(await load_transformed_data(1, '4', 'CI-W3W01A3'))\n", - "\n", - "def plot_data(data: dict[str, DataFrame]):\n", - " size = len(data)\n", - "\n", - " plt.figure(figsize=(10, size * 5))\n", - "\n", - " i = 1\n", - "\n", - " print(f\"Plotting {size} plots\")\n", - "\n", - " for title, df in data.items():\n", - " print(f\"Plotting {i}: {title}\")\n", - "\n", - " # Eixo principal (esquerdo)\n", - " ax1 = plt.subplot(size, 1, i)\n", - "\n", - " l1, = ax1.plot(df['timestamp'], df['value'], label=\"real\")\n", - " l2, = ax1.plot(df['timestamp'], df['prediction'], label=\"prediction\")\n", - " ax1.set_ylabel(\"value / prediction\")\n", - "\n", - " ax1.set_title(title)\n", - " ax1.set_xlim(\n", - " df['timestamp'].min(),\n", - " df['timestamp'].max()\n", - " )\n", - " ax1.set_ylim(\n", - " min(df['value'].min(), df['prediction'].min()),\n", - " max(df['value'].max(), df['prediction'].max())\n", - " )\n", - "\n", - " # Eixo secundário (direito) SOMENTE para sensor_in\n", - " ax2 = ax1.twinx()\n", - " l3, = ax2.plot(df['timestamp'], df['sensor_in'], label=\"sensor_in\", color='black')\n", - " ax2.set_ylabel(\"sensor_in\")\n", - "\n", - " # Legenda combinada\n", - " ax1.legend(\n", - " handles=[l1, l2, l3],\n", - " loc=\"upper left\"\n", - " )\n", - "\n", - " i += 1\n", - "\n", - " plt.tight_layout()\n", - " plt.show()\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "46340afd", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/grezewave/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", - " self.metrics_controller.start()\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", - "/home/grezewave/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", - " self.metrics_controller.start()\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", - "/home/grezewave/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", - " self.metrics_controller.start()\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Plotting 3 plots\n", - "Plotting 1: NOx\n", - "Plotting 2: CO\n", - "Plotting 3: O2\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA90AAAXRCAYAAACD3P7HAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsnXeYXGXZ/z9n6vaaZDebTgJphBAIiaFIC4SqiIUmIPVVCNJUQGkiioUmRRE0gP5A8QVeREUkhB5KCgQICSG9bLKbzfY+7fz+OGXKzuzO7s6ZOc/M87muubI5c+aZZ+5Tv+duiqqqKhKJRCKRSCQSiUQikUhSjiPTE5BIJBKJRCKRSCQSiSRbkaJbIpFIJBKJRCKRSCQSi5CiWyKRSCQSiUQikUgkEouQolsikUgkEolEIpFIJBKLkKJbIpFIJBKJRCKRSCQSi5CiWyKRSCQSiUQikUgkEouQolsikUgkEolEIpFIJBKLkKJbIpFIJBKJRCKRSCQSi5CiWyKRSCQSiUQikUgkEouQolsikUgkEolEIpFIJBKLkKJbIpFIJBIb8sQTT6AoCnl5edTW1vZ5/5hjjuHAAw+MWub3+3nggQc47LDDKC4upqioiMMOO4wHHngAv9+frqlLJBKJRCKJQIpuiUQikUhsTG9vL7/85S8HXK+zs5MTTjiBq6++murqan75y1/ym9/8hpqaGq6++mpOOOEEOjs70zBjiUQikUgkkUjRLZFIJBKJjTn44IN57LHH2L17d7/rXXfddbz55ps8+OCD/POf/+TKK6/ke9/7Hv/4xz946KGHePPNN/nBD36QpllLJBKJRCIxkKJbIpFIJBIb8+Mf/5hgMNivt3vXrl386U9/4rjjjmPx4sV93r/yyis59thj+eMf/8iuXbsAePzxx1EUhSVLlkSt+4tf/AJFUXjppZdS+0MkEolEIslRpOiWSCQSicTGTJo0iQsuuKBfb/d//vMfgsEgF1xwQcJxLrjgAgKBAC+//DIAF110EaeddhrXXXcdO3fuBODTTz/lpz/9KZdccgmnnHJK6n+MRCKRSCQ5iBTdEolEIpHYnJ/85CcEAgF+9atfxX1/3bp1AMyePTvhGMZ769evN5c99thjOJ1OLrnkEnw+HxdeeCHV1dXce++9KZy9RCKRSCS5jRTdEolEIpHYnP3224/zzz+fRx99lD179vR5v729HYDi4uKEYxjvtbW1mcuqq6t5+OGHWbp0KUcddRRr1qxhyZIllJSUpPgXSCQSiUSSu0jRLZFIJBKJANx8880EAoG4ud2GoDbEdzwSCfOzzz6bU089lRUrVnDZZZdx/PHHp3DWEolEIpFIpOiWSCQSiUQA9ttvP7797W/H9XZPnz4dgE8++STh5433ZsyYEbW8sbGRVatWAVqYeigUSuW0JRKJRCLJeaTolkgkEolEEAxvd2xu98knn4zT6eQvf/lLws/++c9/xuVycdJJJ0Utv/LKK2lvb+euu+7inXfe4f7777di6hKJRCKR5CxSdEskEolEIgiTJ0/m29/+Nn/4wx+oq6szl48bN46LLrqIV199ld///vd9PvfII4/w2muvcckllzB27Fhz+bPPPsszzzzDL3/5S2688UbOPvtsbr75Zr744ou0/B6JRCKRSHIBRVVVNdOTkEgkEolEEs0TTzzBRRddxMqVK5k7d665fNOmTUybNo1gMMjMmTNZu3YtAB0dHZx88sm88847fOUrXzE92v/973/5xz/+wdFHH82///1vCgsLAdi7dy8zZ85k1qxZLFu2DEVRaGxsZObMmey333688847OBzy2bxEIpFIJMNFXk0lEolEIhGIKVOm8O1vf7vP8qKiIpYtW8Z9991HbW0tP/zhD/nBD37Azp07uf/++1m6dKkpuAG+973v0dvby+OPP46iKABUVlby6KOP8t5773H33Xen7TdJJBKJRJLNSE+3RCKRSCQSiUQikUgkFiE93RKJRCKRSCQSiUQikViEFN0SiUQikUgkEolEIpFYhBTdEolEIpFIJBKJRCKRWIQU3RKJRCKRSCQSiUQikViEFN0SiUQikUgkEolEIpFYhCvTExCVQCDARx99RFVVlexjKpFIJBKJRCKRSCRJEAqFqK+vZ86cObhcuSFHc+NXWsBHH33EvHnzMj0NiUQikUgkEolEIhGOFStWcNhhh2V6GmlBiu4hUlVVBWg7y+jRozM8G4lEIrGWL/1imfn32z86FrdLRvhIJBKJRCIZPHv27GHevHmmnsoFpOgeIkZI+ejRoxk7dmyGZyORSCTWMmH8OGpbugFY2+ri9Nk1GZ6RRCKRSCQSkcmlFN3c+aUSiUQiGTK9gaD591/e357BmUgkEolEIpGIhRTdEolEIhmQ9p6A+feKrU3saOzK4GwkEolEIpFIxEGKbolEIpH0iy8QojcQAmBqVTEA727el8kpSSQSiUQikQiDzOm2mGAwiN/vz/Q0JCnC7XbjdDozPQ2JJK109oa93AtnjGJDfTvvbm7k7HnjMzgriUQikUgkEjGQotsiVFWlrq6OlpaWTE9FkmLKysqorq5GUZRMT0UiSQsduujOdzs5av+RPPz6Zt7b0oiqqvI4kEgkEolEIhkAKbotwhDco0aNoqCgQN6YZgGqqtLV1cXevXsBZKs4Sc5giO6iPBdzxpfhdTloaO9lc0MHU0YVZ3h2EolEIpFIJPZGim4LCAaDpuCurKzM9HQkKSQ/Px+AvXv3MmrUKBlqLskJDNFd7HXhdTmZO7Gc5ZsaeXdzoxTdEolEIpFIJAMgC6lZgJHDXVBQkOGZSKzA2K4yV1+SK3T0hD3dAIdPHgHAu5saMzYniUQikUgkElGQottCZEh5diK3qyTXaDfCy72a6J4/qQKAj3Y2Z2xOEolEIpFIJKIgRbdEIpFI+sX0dOuie3yFFu3R0N5LMKRmbF4SiUQikUgkIiBFt0QikUj6paNXS6UwRHdlkReHAiEVGjt7Mzk1iUQikUgkEtsjRbfEUrZt24aiKKxZsybTU5FIJEMkNqfb6VCoKPQCmrdbIpFIJBKJRJIYKbolEolE0i+xOd0AI4ul6JZIJBKJRCJJBim6JQnx+XyZnoJEIrEBsZ5ukKJbIpFIJBKJJFmk6E4DqqrS5Qtk5KWqyRc5OuaYY1i8eDHXXHMNI0aMYNGiRaxdu5aTTz6ZoqIiqqqqOP/889m3b5/5mZdffpkjjzySsrIyKisrOe2009i8ebMVZpRIJBkisk+3wShddO+VolsikUgkEomFvPXWW5x++unU1NSgKAovvPDCgJ954403OOSQQ/B6vUyZMoUnnnjC8nn2h2vgVSTDpdsfZMat/83Id6+7YxEFnuQ385NPPsn3vvc9li9fTktLC8cddxyXXnop9913H93d3dxwww1861vf4rXXXgOgs7OT6667joMOOoiOjg5uvfVWvva1r7FmzRocDvlMRyLJBgzRLT3dEolEIpFI0k1nZyezZ8/m4osv5swzzxxw/a1bt3Lqqafy3e9+l6eeeoply5Zx6aWXMnr0aBYtWpSGGcdBlQyJnTt3qoC6c+fOqOU9PT3q7NmzVafTGfVCcQz55SwZqY654gl13DV/V10VYwf32Zh59PcCVMD8v8PhUBVFibveQOPcdNNNqqqq6i9/+csB18/ka8KECWptba0aCATUI444IuPzGejl8XjUX/ziF6qqqur111+vulyujM8p0Wv//fdXm5ub1fr6enXSpEkZn89Ar+LiYvW5555TVVVVL7vsspSM6XK51MWLF6uqqqpPP/20WlhYmNI5n3/++aqqquo///lPtaSkxDLbKA7tHOaIWOZwassUR+a3nRWvdNk2115WHxO5/HK73epPfvITVVVV9Q9/+IOan5+f8TkN9DrwwAPVrq4udevWrerYsWMzPp9EL4/Ho/7qV79SVVVVf/Ob36herzfjcxroNW/ePNXv96tr165Vq6ur465zwgknqKFQSH3vvffUyspK1el0qpMnT1YbGxutvIW2lFAopH7lK1/JuP0z9Tr55JPVUCikvv3222pFRUXG55Po5XA4VEBdt26d2traar56enoG3MaA+n//93/9rvOjH/1InTlzZtSys846S120aNFwdq9hIUX3EEkkuj/99FNTvKby9fQzf1dfe/MtS8a24jV79mxVVVV14cKFGZ/LQK/nn3/e3J4ivA477DBVVVV14sSJGZ/LQK/XXntN/fe//53xeST7uuSSS1RVVdWysrKUjVldXa2qqqqed955KZ9vXl6eqqqqeuWVV2bcdtn2kra17mXlMZHrr2nTpqmqqqqnnnpqxueS7Gv16tXq008/nfF5DPT60pe+pKqqqh5xxBEZn0uyr82bN6u/+93v+l2nublZvfPOO6OWvfLKKxbeQVtLT09Pxu2e6Vd7e7t6++23Z3weQ3nddtttA25jGFh0H3XUUerVV18dtWzJkiVqSUnJ0HeuYSLDy1NMKBQCoLy8nFWrVpGXlzes8b72ta+xYsUKPE4HHqcWrj1hwgTefffdYc81lq9//evMnDmTO+64A4DzzjuP/Px8fvKTn/RZt6qqioKCAr785S8zZswYrrjiCqqrq3n11Ve58847TTsY/1577bX84Ac/SPmch8MZZ5zBypUrUVXVnKfH42Hr1q39fq6np4edO3cybty4YW/fwfDqq69y4YUXmnn6xpz//e9/c/DBB6dtHslw7LHH8sUXX0TZdvbs2bz00ksZnll8HnzwQX75y1/2se3bb7/NfvvtN6Qx161bxwknnNBnzNtvv53LLrtsWPOtra1l3rx5fca26jg77YG32dvey58vnse00SUAfLi9ie/+vw8ZX1HAs987POXfmSl27drF/Pnz+9j2uuuu4/rrr8/k1IRn/fr1LFy40JJjItd5//33+frXv97Htvfeey9nnXVWJqeWkLlz57Jnz56o68SRRx7JM888k+GZRfPKK69w0UUX9bHtn/70J0466aRMTi0hBxxwAJ2dnVG2PfXUU3n00UcB8Pv9TJw4ESBqHQN1EPWA7Ebkb9mwYQNFRUUZnE366OnpYfLkyUD0Nr3gggu46667Mjm1uOzZs4e5c+eybt06xowZYy73er0pGb+uro6qqqqoZVVVVbS1tdHd3U1+fn5KvmcwSNGdYowTldPppKamZtiizNj5VC0qAdCEYU1NzfAmmuC7ioqKzLEPP/xwnnvuOebPn4/L1XdXaWxsZPPmzTz++OMcddRRALz++uvmfCMpKSmxZM7DIZ5tHQ7HgPPs6emht7c3Jdt3MIwYMQII29b4d9SoUbazrcfjAaJt6/V6bTdPg9LSUqCvbaurq4c8Z6PgYOyYZWVlw7ZDMBiMO7ZVx5nPW4aLAPtNGEfNiEIAetwluIq30+Zw2Xa7DoVAQMtfT5dtc4nGxkbAmmMi1xk5ciTQ17YVFRW2ta1xXxF5ncjPz7fdfCsrK4G+tq2srLTdXA2MmjqRti0oKDDna5znYteJXCYqkXOvqanJKdFtELlNi4uLbbmfGg8FiouLKSkpyfBs0oOsdJVijJ1cUZSUjGeME3kApWrsgbjyyitpamrinHPOYeXKlWzevJn//ve/XHTRRQSDQcrLy6msrOTRRx9l06ZNvPbaa/zlL3+JGiPdcx4MmbTtUIicb+S/dpyztK212yud+0IopPZbSK2jV+uUkC2IdJyJhrStdYhoW1GuE9lo28i/s1l023kbpZpE2zSXbBBJdXU19fX1Ucvq6+spKSnJiJcbpOhOOdkkumtqali+fDnBYJATTzyRWbNmcc0111BWVobD4cDhcPC3v/2N1atXc+CBB3Lttddy/vnnm/ON/NeOB70oF3wDkS780rbZI7o7IwR1UUTLsCKvizy3dgnJpgrmIh1noiFtax0i2laU60Q22laK7uxDiu5oFixYwLJly6KWLV26lAULFmRoRjK8POWkeic3QoRCoZDlB9Abb7zRZ9n+++/P888/n/AzCxcuZN26deb///d//xcIh2MZIdxG7pCdiLwoGWEudm5zJtKFX5SbKQMpuhNjeLndTgWvK3x8KIrCyGIvO5u6aWjvZUJlYcq/OxOIdJyJhrStdYhoW1GuE9lo24FEd2yOt0hEzt3O93SpJvK3pkMzpJuOjg42bdpk/n/r1q2sWbOGiooKxo8fz0033URtbS1//vOfAfjud7/LQw89xI9+9CMuvvhiXnvtNf7+97/z73//O1M/QXq6U002ebqHgkgXJ2lb65C2zSLR3aOHlntdfcYfVazVNJCebkkySNtah4i2FeU6kY22lZ7u7CPbPd2rVq1izpw5zJkzB9CKm86ZM4dbb70V0Aqz7dixw1x/0qRJ/Pvf/2bp0qXMnj2be+65hz/+8Y+Z69GN9HSnnFSfqES5KBmIdHGStrUOY06iPG2NnC9YI7qNsVMpulM530S0x8nnNhhZpEWy7M1i0Z3K7ZbrSNtah4i2jRdtZsf5imzbZK7BkesYSNEtHtkuuo855ph+98snnngi7mc++ugjC2c1OKSnO8WkOkxZNGEYGQ4P9haG6QzdTwWJLvx2DJ+KrJxq53kaRM4XUmPbRMdCKuwQO4aVNg57ut193jOKqWWTpzt2X0jldst1rDwmch0RbRvvGmzH+Yps2/6uwfHWMZCiWzyyXXRnA/Y9YwiKDC8XzxsrbZt6pG3TE15ujJuOnO5ibxxPdxaKbnnjYh0incNEQ0TbinKdyFbbxlvHQIpu8ZDXLvsjRXeKSfVOLr2x1hEvtM2O8zQQ6cIvys2UgRTdiTE93XHCy6tKNNG9p62nz3uiIm9crEOkc5hoiGhbUa4T2Wrb/kS3LKQmJqKl9uUaubU3pgHp6Rbn4iRtax3Sttkjus2c7jie7poyrdflnpbulH9vppCi2zpEOoeJhoi2FeU6ka22lZ7u7EOUYypXkaI7xaT6RCXaASTSxUna1joyYdu2Hv+Qjz8puhNjeLoL+xHdu1u6hb5Ji0SKbusQ6RwmGiLaVpRrcLbaVoru7EOUYypXkaI7xUhPtzgXJ2lb60i3bZdv2sdBt7/Cfa9uHNLnpehOTJdfE90FHmef98boorvTF6StO5Dy784EUnRbh0jnMNEQ0baiXIOz1ba5ILpzDVGOqVxFiu4Uk+rcYNFyumOrfNo5Vzpyrnaep0Fkrk7kv3acc7pt+5f3tgPwwLKN+IODz0WzojqtlcdC5BhW27jXr42d5+47dp7bSWWhB4BdLV0p/+5MkE7b5hoiXR9EQ0TbinINzlbbxlvHIBtyuu28faxCNM2Qa+TeHmkxVnq6RTiRJPtEeOLEidx///1Rn3vhhReG/d2DGUO0J4KJWhnZcc7ptu3EEYXm32990TDozyd64i893dDjDwKQ5+rr6YbIEPPsKKYmPd3WIaLHUBREtK0o1+BstW22e7rtvH2sQjTNkGv0TdKTDAsZXj60i9OePXsoLy9P6jtuv/12XnjhBdasWTPkMWLnms22zQTptm2kd/u5D3exs6mLXc3d/PiU6TgcA3+vFN2JMUW3O77oHlOWz6e1rezOkmJqUnRbh0jnMNEQ0baiXIOz1bZSdGcfohxTuYoU3Skm1ScqOx9APp8Pj8cTtWyoF6fq6uphz2ewY9jZtvEQ6cKfbtt268IQ4KVP63jp0zoAvnrwGGaNLR3w81J0J6ann/ByiC6mlg1I0W0dIp3DRENE24pyDc5W20rRnX2IckzlKjLmIMXE3clVFXydQ3opqiYmQr5uVJ92U6swiPEGceI85phjWLx4MYsXL6a0tJQRI0Zwyy23mL9p4sSJ/OxnP+OCCy6gpKSEyy+/HIB33nmHo446ivz8fL797W8DEAzq89bDW37+85+Tn5/PpEmTeOqpp/p8d2xo+K5duzjnnHOoqKigsLCQuXPn8sEHH/DEE0/w05/+lI8//hhFUVAUhSeeeCLuGJ9++inHHXcc+fn5VFZWcvnll9PR0WG+b3jK//vf/3LCCScA0Nrait/vT9pm6USkC3+6T/w9vmDc5fVJ9o+WojsxPQHNtt4Enu6asjwAalu6eeTNzVzyxErTOy4iUnRbh0jnMNEQ0baiCIRstW1/ojsbcrrtvH2sIrL2jwj7aa4hPd0pJu5O7u+CX9QMaTzHRq04kfqfG1FLtDGV3R8lP96Pd4OncOD1dJ588kkuueQSVqxYwapVq7j88ssZP348l112GQB33303t956K7fddhsAmzdv5qSTTuLOO+9kyZIl/Pvf/+baa69l165d2rx1e+zbt4/XX38dt9vN97//ffbu3ZtwDh0dHRx99NGMGTOGF198kerqaj788ENCoRBnnXUWa9eu5eWXX+bVV18FoLS0ryezs7OTRYsWsWDBAlauXMnevXu59NJLWbx4sSnSDfbu3csjjzzCN7/5Tbq6unjiiSfM32snRLrwZ8rTvXD6KEIq7Gru4ov6Dho6epP6vBTdiUkmvBxg094OXllXjy8Q4p2N+1g4oyrlc0kHUnRbh0jnMNEQ0bZSdFuH9HTnZi5zZO0fEfbTXEOK7hST8pxuY1zCTmsrj59x48Zx3333oSgKU6dO5dNPP+W+++4zRehxxx3H9ddfb65/6aWXct5553HNNdcAcOCBBwLQ3NxMT08P3d2ad/6qq67iS1/6EgB/+tOfmD59esI5PP300zQ0NLBy5UoqKioAmDJlivl+UVERLper33Dyp59+mp6eHv785z9TWKg9dHjooYc4/fTT+dWvfkVVVVgQnH322UyYMAGA/Px8li1bJkX3MEn309Yu3dN90oGj+cahY7nxuU800d0+ONEdOd/I5UMhdnul8ul75BhW29gML3fFv4EZU66J7s/r2s1ln+1uyxrRnctek1Rj5TGR64ho28g523m+Itu2v+tDouseZIfotvP2sQpRHmTlKlJ0p5i41QLdBZrHeQgon34b1r+AeuLPUWtq4JlzUMbOhR8vS24Ad8Ggvu9LX/pS1AG6YMEC7rnnHjNcfO7cuVHrf/zxx3zyySdmyLixHsDWrVtN0b3//vuby6dNm0ZZWVnCOaxZs4Y5c+aYgnsorF+/ntmzZ5uCG+CII44gFAqxYcMGqqqq+txcAzidzn698Jkk0YXfjk9zI5+2pmOehqc7X/fGjiz2AiQtuuPNN3L5ULCiDVm8eVlt44E83UZOdyTr9rSmfB7pIrZlWC57TVKNlcdEriOibeO1N7LjfEW2bX/Xh0TXPWOZqOSy2JSi295I0Z1i4p6oFGVQId5RH3W6tXGdHlSXJiQUh3PI4w2XSBELWij4//zP//D9738fgPfff5/zzz+fKVOmMHny5CEd9Pn5fW/irSDRycmuuUwierrTltNtiG6PdhMxWNFtZXi5gajh5b0Bo5BafNFdWejB63KY64Hm6RYVGV5uHVYeE7mOSNcHA1EEQrbaNtvDy+28faxClGMqV7HvYzpBSfVOnu5G9x988EHU/99//332339/nM74N9yHHHII69atY8qUKUyZMoWxY8cC4Ha78Xg8eL2a+NmyZYv5mQ0bNtDS0pJwDgcddBBr1qyhqakp7vsejyfKox6P6dOn8/HHH9PZ2WkuW758OQ6Hg6lTp0ata/fQNgORLvxpz+n2RXtjRxbpotsGOd2x44omusOe7viXC0VRzLxujx6Cvqu5m9ZuexYkHAgpuq1D2tY6RLo+GIgiELLVttleSM3OkQhWkW7NIBkcubdHWkzKc7rTfFHasWMH1113HRs2bOCvf/0rDz74IFdffXXC9W+44QbeffddFi9ezJo1a6itrQWgrk5r2WS0FHvggQf44IMPWL16NZdeemm/3uxzzjmH6upqzjjjDJYvX86WLVt47rnneO+99wCtivrWrVtZs2YN+/bto7e3r7A677zzyMvL48ILL2Tt2rW8/vrrXHXVVZx//vlR+dwQ/4JjR0S68Ce13+75GN59sO/rwz9DIDmxbDDc8HIpuhMzUHg5hEPMj5wywhTg6wT1dkthaB3SttYh0vXBQIpu65CebntvH6sQ5ZjKVWR4eYoRXXRfcMEFdHd3M2/ePJxOJ1dffbXZGiweBx10EG+++SY/+clPOOqoowgEAgCmZ9yYc2VlJUcffTRVVVXceeed3HLLLQnH9Hg8vPLKK1x//fWccsopBAIBZsyYwcMPPwzA17/+dZ5//nmOPfZYWlpaePzxx/nOd74TNUZBQQH//e9/ufrqqznssMMoKCjg61//Ovfee6+5Tn8XHDsi0oU/qf326bOgfU/8AQK9MC/5YnaGp7vAo53SIkW3qqoD2kiK7sSEC6klFt2HTijnnU37+NqcMfzz493UtnSzbk8bCyZXpnw+ViOFoXVI21qHSNcHA1EEQrbaVoru7EOUYypXkaI7xYguut1uN/fffz+///3v+7y3bdu2uJ857LDDeOWVVwB4++23+fKXv2wWQTPmfOedd5q9sAHOP//8qDFiT/ATJkzg2Wefjft9Xq837nuxY8yaNYvXXnst7hgA8+fPZ9u2bVG2HTVqFG+88UbCz2QSkS78Se23nQ3av9O/Ei7417gRalfD9ncHJ7pjPN0j9PDybn+QTl+QIm//pzopuuOjqqrZpztReDnA4uOm8LU5Y5g4opDNDVrrMOnplsQibWsdIl0fDEQRCNlqWym6sw9RjqlcRYruFJPq3GDR8jOMuRl2sHNuTeRc7TxPg1jb2nl/iJxr3HmqKoS0qAhOvReKRmp/b34N/vI12P1h0t+lqqopuvP0QmqFXheFHiedviAN7b1Ji24rWoYZcxSxZZgvGMIwh7ef8HK308HEEVqRxZk1pQB8trs1pXNJF1Zut1xH2tY6YoWhCLaNnLOd5yuybYfaMkzmdIuJaJoh18i9PdJirPR0i3Cij2xBEfmvHeccOVc7z9Mg0dN2O15YBmxXEooohOeMEMQ1c7R/m7dBV/xCerH0BsLCMD9CGA4mr9vKlmGQ+pY4icRLqvcFI7Qc+vd0RzJ9dDEAm/Z2EAyJ5y1J9EDDjseZaMh2bNYhclsru+8LItu2v+uDbBmWfYimGXIN6elOMekIL7fqRJ+KsGqRwrBEC8OJ95AA7DnnAW1reLkBHBGnofxyqJgMTZuh9kPYf+GA32Xkc0N0sa+RxV62NXYlJbpFCy83xjHGtWr/7dUjCBQFPM7kzjvVJXkoCgRCKk2dPvPhhyjIEGjrkLa1DpGuvQaiXIOz1bYyvDz7SKdmkAweuSVSjOg53cNFpIuTyLYVX3RHtJNyuKM/POZQ7d8kQ8yN0HK3U8EdIQwNsbcvibZhoopuq8Y2iCyiluzYLqeDykKta0Eytrcjop0bREGKbusQ6dprIMpxlq22laI7+xDlmMpVpKc7xYh4Y51KRLo4iWxb4UV3MFJ0x5yGxhwCn/4dtr4FDRsGFN8jgiFe93Tjc+TB+hBMPw2I6NWdxZ5uq8Y2SKaIWjxGFHnZ1+Gjob2X6aNTOqW0kI4oglxEim7rEOnaayDKNThbbStFd/YhyjGVq0jRnWJSvZOLVhQhUcERO4a3xMt9seM8DbJKdEfmdDtiCnTVHKL9u+3tpL7LA0wyNtuzF8Mlr0DNwYPK6ZaiOz7J9OiOx8hiL5/XtSfdJ91uyBsXa5Ci2zqyVRjagWy1bX+iWxZSExPRNEOuIUV3ihHxxjqViHRxEtm2oojuhCd+I7zc4dIShiMZfRAoTlCD4CmCM34PRaMSftfne9r4yQtr+UHBv1gQXA3PfBumnsyi+naKXI2M3Z4PL1VpK7vy4PCr+owXb7595jxIYgVGqouaDGjjFGCGlw9WdBtRBlkQXi6L0aQOq4+JXCbRA28721aU40xk2w61ern0dIuJaPe1uYYU3SnGqhtrUQ6gRFU+7Thn0Z4IxrtBAXs+zY1X9C26erleSC02nxvAnQ9TFsK2d+Dsp2C/Y/r9rr09DaxW/dxTOI1nnTdplc9XPMr+wP4uoB1YEfGB1p3wzScGnC8Mb3+I/L0J7TAMBrRxCjA83V7X4MY18+kH8HSvrW3loidW8oMTD+Csw8YPbZIWkA7b5iJWHxO5TKLOIXa2rSjHWbbaNtF1z1gmKiLcz1mFaJoh15CiO8VIT7f0dFuFiJ7uhB6MYISnOx7n/BX8XeAtHvC7jEJqwbxS+PZ/YM3TEOilvr2Xv63cQZHHxSVHToJgLyx/AD77PzjiGqg5OPn5DoHIz1vxYCcdXqKhhpePSNLT/fSKHTS093LDc59y+uwaCjz2uCSlI4ogF7H6mMhlIvdZsPe110CU40xk2/Z3fUj0IN9YJioibB+rEO2+Ntew1WO6t956i9NPP52amhoUReGFF14w3/P7/dxwww3MmjWLwsJCampquOCCC9i9e3fUGE1NTZx33nmUlJRQVlbGJZdcQkdHR9Q6n3zyCUcddRR5eXmMGzeOX//61yn7DTKn21453bH7Uex7IHO6rSDpnG5nApHlcCYluCEsDAs8TiipgS//AI77CRz7Y+4LfJNf9JxJ4Oib4IQ7YNY3tQ8tu2Nw8x0CWZHTHTDCy4fm6R4op9sIQwf431W7Bjk765A3LtYgc7qtI9aGIthWlONMJGeCgczptvf9nFWIphlyDVvtkZ2dncyePZuHH364z3tdXV18+OGH3HLLLXz44Yc8//zzbNiwga985StR65133nl89tlnLF26lH/961+89dZbXH755eb7bW1tnHjiiUyYMIHVq1fzm9/8httvv51HH300Jb9BxBvrVGK3i9OePXs4+eST474nsm3FF90DeLoHgdGnOz/GGzuyyIvH6SAYUqk3xN+xN2nfuXkZ1H2a/HyHQFaI7mEUUoOBW4YFIm7s/vjOFgJBe9zoiXZuEAUpuq1DRNuKcpzZ7b4mGYYruqWnW0xEOaZyFXvE8umcfPLJCQVSaWkpS5cujVr20EMPMW/ePHbs2MH48eNZv349L7/8MitXrmTu3LkAPPjgg5xyyincfffd1NTU8NRTT+Hz+ViyZAkej4eZM2eyZs0a7r333ihxPlREvLFOJem8OPl8PjweT7/rVFdXJ3xPZNuKL7r7yekeJN0JhKHDoTC6LI/tjV3UNnczpiwfKvaD6oO0NmSttVA9K7n5DoFsEN29hm1dQxPdA3m6e/1hkb2zqZtln+9l0czEx2y6EO3cIAoiCkNRENG2ohxnUnSLhQjbxypEOaZyFVt5ugdLa2sriqJQVlYGwHvvvUdZWZkpuAEWLlyIw+Hggw8+MNf58pe/HCXWFi1axIYNG2hubk74Xb29vbS1tZmv9vb2uOvFO1GpqkpnZ+eQXoFAwPz+np4eQAsbSfbzgz1xPvvss8yaNYv8/HwqKytZuHAhnZ2dAPzxj39k+vTp5OXlMW3aNH73u9+Zn9u2bRuKorBs2TIA6urqmD17tjlnRVHYvn07p59+OuXl5RQWFjJz5kxeeuklc4w333yTefPm4fV6GT16NDfeeKP5+wGOOeYYFi9ezDXXXMOIESNYtGjRgL8nMrzcmOPzzz/Psccey5IlSwDYsWOHECenrBLdQUN0D/+5X1cCTzegCW2gtqUrvNClhzQHfcnPdwhkg+gOVy8fZHi5Hjbe3OXHF0jsve6NeW/NzpbBTdAi5I2LNYgoDEVBRNuKcpxJ0S0WImwfqxDlmMpVbOXpHgw9PT3ccMMNnHPOOZSUlACa0Bs1KroVkMvloqKigrq6OnOdSZMmRa1TVVVlvldeXh73++666y5++tOfDjiveDt5V1cXRUVFSf6y+Pzyl780//7Pf/6T9HgdHR0UFhYmte6ePXs455xz+PWvf83XvvY12tvbefvtt1FVlaeeeopbb72Vhx56iDlz5vDRRx9x2WWXUVhYyIUXXmiO8eCDDwJQWVnJAQccwGeffQZo9rjyyivx+Xy89dZbFBYWsm7dOvN31NbWcsopp/Cd73yHP//5z3z++edcdtll5OXlcfvtt5vjP/nkk3zve99j+fLlSf2mePzkJz/h7rvvpqqqimeeeYa///3vZoSFnU9OxtxS2dbKKuLNNa6nO1FO9yAwQqDzPf2I7ubu8EKn7l2PI7pTmQcVexMsZsuwoYWXl+a7cTkUAiGVxs5eqorzcDj6zq03oI1fVeKlvq2XjfUdfdbJBJE3LiK0BxIFq4+JXEZE24pynMWKbjvP1SCZ60OiewpjmaiIsH2sIh33BZKhI6Sn2+/3861vfQtVVfn973+flu+86aabaG1tNV/r1q2Lu57IBRz27NlDIBDgzDPPZOLEicyaNYsrrriCoqIibrvtNu655x7OPPNMJk2axJlnnsm1117LH/7wh6gxLrroIkA7yH/6058SDGo31Q6Hgx07dnDEEUcwa9Ys9ttvP0477TS+/OUvA/C73/2OcePG8dBDDzFt2jTOOOMMfvrTn3LPPfdEnfz3339/fv3rXzN16lSmTp06pN/5gx/8gFNPPZWKigpAi5gwCvLZebvFu0EBe845shVJ3GPCipzueKK73PB0R4puPcrFqKCezHyHQOTnIy+AVrQMs+q80xMYmuh2OBSzgvkT725j/5v/w38+3dNnPcPTfWBNKQAb98aPIEo38YrR2PE4E43Imz9p29Ri9fnGCkQ5zhK1QrXjXA2SuT7EW8cgGzzddt4+VhGvDZwU3fZBuD3SENzbt29n6dKlppcbtPzdvXv3Rq0fCARoamoyc3urq6upr6+PWsf4f3/5v16vl5KSEvNVXBy/snK8nbygoICOjo4hvb73ve8BmlD87W9/C8Bpp52W9OcLCgqSsivA7NmzOf7445k1axbf/OY3eeyxx2hubqazs5PNmzdzySWXUFRUZL7uvPNONm/eHDWGIYRVVWX06NHmckVR+P73v8+dd97JEUccwW233cYnn3xivr9+/XoWLFgQZbcjjjiCjo4Odu0KVzU+9NBDk/49iTjooIPMORm0tLT0WWY3rOglbRWDyene2dRFW48/doiE3PXSembd9l+27tPSHoyc7v7Cy3c1xxPdMrx8IIzwcu8gw8shnNe95J2tBEMqr67f22cdI6f7wDGa6N7R1GV61zOJDNGzBhFDoEVBRNuKcpzJ8HKxEGH7WEU850wuPnywK0JtCUNwb9y4kVdffZXKysqo9xcsWEBLSwurV682l7322muEQiHmz59vrvPWW2/h94dv8pcuXcrUqVMThpYPhngHu6IoFBYWDull5J673e6ov5P9/GBOOk6nk6VLl/Kf//yHGTNm8OCDDzJ16lTWrl0LwGOPPcaaNWvM19q1a3n//fejxnC73aYdYm1w6aWXsmXLFs4//3w+/fRT5s6da4ajJ0uyofL9Ycwx1usSu8xuZGNOt091cPy9b3LqA2/T2pWc8H5hTS3tvQH+s1bznPYrussHF15u1c2fuKJ7aIXUICy6/UFtbjubu/qsY4SXjynLp6zAjarC5obMh5iLIgZERNrWGqTotg4pusVChO1jFaIcU7mKrUR3R0eHKegAtm7dypo1a9ixYwd+v59vfOMbrFq1iqeeeopgMEhdXR11dXX4fNrN8/Tp0znppJO47LLLWLFiBcuXL2fx4sWcffbZ1NTUAHDuuefi8Xi45JJL+Oyzz3jmmWf47W9/y3XXXZeS35DqE1W6DyBFUTjiiCP46U9/ykcffYTH42H58uXU1NSwZcsWpkyZEvWKzY+PvTjFLh83bhzf/e53ef7557n++ut57LHHAG3bvffee1GfW758OcXFxYwdO9ay32ogwskpq0S37unu8IMvEGJnUzc3PPfJgMfPvo5e6tu0itgfbtcKH/YXXj62TIv0qG3pDo8dJ7w86eOsd3Dhz1Yev+n0dMezbUK6muAfV3Jx++/Jp8dcvLMpnugOe9L3H6XVd9i0V4rubEba1hqk6LYOKbrFQoTtYxWiHFO5iq0Kqa1atYpjjz3W/L8hhC+88EJuv/12XnzxRQAOPvjgqM+9/vrrHHPMMQA89dRTLF68mOOPPx6Hw8HXv/51HnjgAXPd0tJSXnnlFa688koOPfRQRowYwa233pqSdmGQ+oM9nY3uP/jgA5YtW8aJJ57IqFGj+OCDD2hoaGD69On89Kc/5fvf/z6lpaWcdNJJ9Pb2smrVKpqbm6MeWCQS3Q6Hg2uuuYaTTz6ZAw44gObmZl5//XWmT58OwBVXXMH999/PVVddxeLFi9mwYQO33XYb1113nWWhMfFEt53DcLJLdGuCtzPCuf3yZ3Ucc/cblBd4uOdbs5k8sm+xwHW728y/V29vRlXVhC3DAKpL81AUTdzt6/Bp3tehhpeveAxe+iGc8zeYelIyZkBRFHNM4US3v5vJLcvZX3GS55oB+zbBjndB7afATigAyx+Alu0cCTzvWcGfgyeioqB0gH/FdtxO4xhTKOktATwUhjr4pvsdJjvr6d3QBLO/DRncr+WNi3VI21qDFN3WkYuiOxsKqdn5fs4q0qkZJIPHVqL7mGOO6ffpWjJP3ioqKnj66af7Xeeggw7i7bffHvT8kkG4G+sISkpKeOutt7j//vtpa2tjwoQJ3HPPPWZl74KCAn7zm9/wwx/+kMLCQmbNmsU111yTcL6xy4PBIFdeeSW7du2ipKSEk046ifvuuw+AMWPG8NJLL/HDH/6Q2bNnU1FRwSWXXMLNN99syW+NnGvkfO18chJRdA9UvbxN172HTSxn5bZmtjd2sb2xi2dW7uTHp0zvM+66PWHR3dzlZ3NDp+npLojjjfW4HFQV51HX1kNtS7cuuo3w8r6e7n4vVLUfAirs+XhQohsEql6+5Q346CntociWN7iiu5krvND27u9g6bbkxykbT3dXJ9N9O7nL8afw8peiV7vWdQAvczvT193HmJ1/5VtuYD3w3OtwxNXQ3QxjDgVv/BoaVmHldst1pG2tIfZ6JoJtRdkXYu9r7DxXg2SuD4nWiSfCRUKE+zmrEOVBVq5iK9GdDVh1Y52OA2j69Om8/PLLCd8/99xzOffcc+O+N3HiRFRVZePGjYBmh7KyMkaMGMG+fftQFGXA/O2jjz6aFStWJHz/jTfeGPhHxBB54TDmaGDY8cYbb2TGjBlRy+yIiKJ7oJzuNp/23i++NoveQIhX19dz/6sb+WBrU9xxP4vwdAOs3t4UbhmWoML2mPJ8TXQ3d3PwuLJoT7eqwgePoHz2sTbfzn2or/1Cm28oED1QsDf8uSQRLrz8lZuh7lPzvy2OCoqCLZR0bgMUmHgkeEsSfhyAsnFw9A28/9lO9vzjdsZ5O3AoCp29AQ6ZUM6IQi/0tsG2t6kINgKQ36MVWfs8NI79HbU41z4Ha5/Txpt9LnwtPV0qDOSNi3VI21qD9HRbRy55uh0OB8FgUIpuQRHlmMpVpOhOMakOUxbtAIqssB35rx3nLFprhXhegcjldmLglmGaoO0NOSnwONlvZBFOh0J5oYf7X93I2tpWOnsDFHqjT1HrdrcCMGN0Cev2tLF6e3O/hdRAK9K1ensztS16TnGk6K5dDS/fiGOLNh+1vY7QJ3/X5hvshpYdUDY+vD6Exfcg7CBMyzAjZ33+92DysVz+agE7dmzlD1/uYfaCE6F8YtJDHXlIGXfv+wVTplXxhzc3s+zzvdx54IF8+0sTYO96+N2XcKtatIEzpNn2D4HT2EEV/zvmGRxNWzRbN28b3m8aAqK0MhIRaVtrkC3DrCOXWoZJ0S02ommGXMO+ZwxBSfWJSrT8jERhWFZcnJ566qmoFmaRr5kzZw5qriLkAIl0Mk02pzuAg5k1JTgd2ntjyvIZU5ZPMKTy0Y6WqDG7fAG26G3CLjx8AgCrtjfTpYeX5yUo9tWngnlkeHm3VoxNcWvrqJ5i1EMv0pYB9ER41gO+6H+TQDhPtxFyP/ssOGARXUGoo5KmyWcMSnADuJ0Objp5OvMmVTCuQitoZ1Yw1x98uNAedrgM8e3xsjp0AJu+sRS++YQ+p+QfcqQKkY410ZC2tQbp6baOXPJ0O53adVTmdIuJaJoh15Ce7hQjxI21haTz4vSVr3zFbAUXi9EWrD9Etq3d5zuw6NbEVgAXs8aURX123qQK/u+jWlZsbeTI/UeYyz+va0dVtVZUi2ZWc8Nzn7KloZM8vYd0f55ugJ2m6I7wdAc0QaeUjwcaUUvGoH7pCuAhbb6Rgi8Xwst1e+xoDRL0dJrVy4fSMiwSU3Q3RYtuN5rYdugPYQryC6BHq1J/gEvfToN4yJEqRDrWREPa1hqk6LaO2DnZea4GwxXd0tMtJqIcU7mKFN0pRogbawtJp+guLi6muHjoBZZEtS3Y/6KSbE53AAezxkbnCBuiOzav28jnnjG6hLICD1NGFbFpb4cpDOMVUgOYVq3tI2t2thAKqTgi+3TrQlrRRWDUfCFa8Bl/Z7Po1n/bd/+2llpHk/lAw/h3qIzTow12NukPPlxaD28vfkA1w8u93jwArWd7kVefk/R0ZxPSttYgRbd1ZKttE+V0G8tERYTtYxWiHFO5Su7FXlhM5E6eivAc0Q4gkcKwhmrbTIVdRc7N7tVTB7Jtd68mpAI443q6QRPJvYGguXyz3r952mhNRM+dUB71uUSe7oPGllHgcdLU6WNDfXt0n25dSCsud5z5EuPpzgHRrXu6W3oVWrv9Zk/0eO3YBsP4yvjh5QBugjgM0Z2nrdfc5TeF+WDsnSpEO++KhLStNWSrMLQD8R54xy63G1J023v7WIUox1SuIj3dKSZyJ9+9ezcjR47E4/EMeacPBjXR4fP58Pm0m89QKERPT09qJpxienUxpaoqPT09pj18Pp/t5mzY1u/3m7Y15h0PVVXx+Xw0NDTgcDjweDxx17MKEUV3oryiT3bsYz5QmJfH5JGFUZ/db0QhlYUeGjt9fL6nndnjygCobdG8pGP1cPFDJpTzt5U7zc8lyun2uBwcNrGCN79o4L3NjUx3RYSXG55uh7vvfCGqrZgpwAPJe17j1Q1ItehOWe6Wqpq/0Ud0esZwRfe4ck1Mt3T5aevxU2IIasCDH0UX1nn52rZt6faFc+8zHF5u92NNNKRtrSFWGIpgW1H2hXjX3tjldiOZ60O8daToFpuU3xdIUooU3SnGOCEXFxfjdrvZvXv3sMZra2sz/21s1NrrdHZ2snXr1uFN1CLq6uoATdBu3bqVQEALI66trbVdUQvDti0tLdTX1wPQ09MzoG0LCgoYP3582n9P5InTeGBgN5saxKsMHy7wobJmuya696sqi3sjMKY8n8ZOHw3tYYG7WxfdNbroTtbTDbBgciVvftHAu5sbuXhGRHi5Lugc7r7h5Q6FaIFthpdHCPEB6M8OwyXlY0f8Lh/RthxueHmh12U+SNnZ1MXM6iLzPTcBFN3TXZCnhZe3dPnBmbnwciu3W64jbWsNkTYUxbai7AuRczKuvbHL7UYyto23jiykJjaideXJNaToTjGRJ67x48cTCASiTtKDpbxcExbFxcWUlZUBUFJSwqRJk4Y9VyuI9P5OmjTJPNjHjx/PxIkTMzSr+ETadsQIrWBXYWFhv7Z1Op24XK6MnMRiW8KAfU+m/Xkw3vyigc6uHnDD+JHxez5XFGr7UVNn2Mu5p1WLQBhdqonuSSMKqSj00NTpw+VQcDsTX2APn1wJwAdbGgnOdGuSMuiPyOkOh5eH50uC8PLBe7qteOqcci9RRBh3H0/3MAupAYytKNBFdzcza0pRFSeKGsRDAEUX/AUFhUCIli4fuHRhnkFPt/QWpB5pW2uI9caKYFtR9gWRPd39XR/irSMLqYlNvG2aiw8f7IoU3Skm8mBXFAW3251UJe1EGJ91OBy4XNrmcrlc5OkeIbuRr4eHqqpKXl6eaY+8vDzbzdmwrSGkjb/tNk8DEcPL4z1tfXb1Lg5QtAdRbnf8EH1DdDfqorvHHzQFuFGNXFEUDhlfzqvr68lPEFpuMLOmlOI8F+09AWrbgoyH6Orl8XK6AQI+OnoDPLhsIz/w9WhSdBCebqFyuvsT3cMMLweoKtY81w3t2sMT1elBCXRT6Ayg6A8yCgrygU4tp9vMvZeF1LIJaVvrEcW2ouwLootumdOdO4hyTOUq8vFHirH9jbXF5EIhtUyRLaK7oaMXN3r0hyP+c78ReuXqxg5NcBmh5QUeJyX54c/MnahFK/QXWg7gdCjMn6R5u79o1HP2gz5TaMatXq57uv/z6R7+8NYWurv1yttDzOm2/bkhYBS3cxCKuTR4XcO/VIwqMUS3XvdBDx8vcoVMb3ZRgZbf3xolun1avnkaEe3cIBLSttYhmm1Fma8U3WJh533JakQ5pnIVKbpTTKp3ctEa3ceKbjuHt4gWhpMtotsXCOEcQHTHhpeHQ8vzon6zETY+stjLQEzV84gbuvSbiaA/7Onup2VYfZv23U516OHlQohuo4iaGu3l9rgcOBzDn/PIIi2CZK8pujWbFziD5ncXF2qiu7nLB66IKIhBRBekAnnjYh3SttYhmm1FmW+8eiqxy+3GcEW3zOkWE9E0Q64hw8tTjO1vrC1GerqtQ0TRHe/E7w+GBvR0x4aXxxZRMzhobBmPXTCXSSOiK6DHw/CeNxuaOcrTHad6uaKts69DW8elBvTPDS283PbVy3Vvsw8X+W4nFYUealu6yUuBlxv6erpDesX4Yqcf9F7rxUV6lfPuiEJqoIlyV/q6BVi53XIdaVvrEM22osxXZE/3YKuXy5xusRHtvjbXkKI7xVh1Yy3KARQ5N7vPWWTbGk/b7Trf/mzrD0Z4up3x6x2MKIr2dO9u0bzNNaX5fdY9YUZVUnOqNELWjY5wkZ5uVwJPd7BXn4OKW/VrC7M1vFx/AOHHxYTKAkYWezXRnYJ8boBRejTC3hjRXeYIt+grLdKiEVq6fFrOt/FGwAcDBzOkDNHODSIhbWsdotlWlPmKLLpleHluIcoxlavkXuyFxcSeuIaLaAdQorYldpyzaGE48S78dg2f6q9diT+oJuHpjs7p3tOqebpHlw29yJ0p5LuN8PJwn25HHNGttQzz0djZi4sgDiUiLD1JxGoZptmiFzcTKguYPFITwKkS3SOLoz3dQYdm81Jn+CFGabEWseAPqnQFAMUZNbd0IUorIxGRtrUO0WwrynxzqWWYFN1iI5pmyDXse8YQFKu8WSIKQ7sf9KKdnEQML48XNugLhHANILorI8LLVVVlt57THRtePhhG6p7ufZGiWw+pjvR0x7YMa+zw4SYQHihbW4YZ4eWqiwmVhew3UhPAw+3RbTCqWHtgsq+jl1BIJahoNi8xPd0K+V4vHj2cvbnLF11MLY2Idt4VCWlb6xDNtqLMV3RP91BahmVDTredt49ViHJM5SpSdKcYqwqpiSgM7V6gTBZSs45+C6kFQziVAUS37pXuDYTo8gXDOd1xwsuTJTanu6O7m811TQB0BrUbDX8w4kIFENByuj1Rojt5AShWeLleSE33dM8ZVw7A2PKC4U1Up7LIg6JAIKTS3OUjqIeXlyh6VXiXF8XhoCxfW97S5Q/ncae5V7doD+REQtrWOkSzrSjzFV1052p4uZ3v56xCNM2Qa8ic7hRj+xtri4kV3fGW2wWRbSt6Trd7gJzuAo+LPLeDHn+Ixg4fe8xCakMPLy/Nd+N0KPhV7bTX3d3Nls4mJjthxc52ADp6/FGF1NSgj6bOXiqJCCkfhAAUSXSrgV4UtEJqEysLmTW2lBcXH8GEioGL1CWD2+mgosBDY6ePve29jFC0bV+k6J5u3atdXuBhb3uvJrqNYmppDi8X7dwgEtK21iGabUWZby6JbllITWxEOaZyldx7DGQxdr+xthopuq0jazzdSbQMA6jU87q3NnbS6dPWHz0MT7fDoVBZ6MGvP2t0E8Cri+mN+zRRF+vp7u3tJqSCV8l+T3d9cxsAAVwcPK4M0KrDlxbEfzAyFCLzugOG6Eb3dOui2/g+rW2YLrqlpztrkLa1DtFsK8p84z3wjl1uN6Sn297bxypEOaZyFSm6U4zdb6ytRqSLk8i2FUV0D7VlGIRDzNfWtgJQXuAm3zO8ol4jirz4IkS3RxfTrb3a3ILBID5/OIrA19NtrmsS8kOS+W4pz7uOM3aqcrfW72wEIC8/n0KvNUFQIyMqmAeIEd26wC7XRbfWNiyzOd1WbLdcR9rWOiJtG7vMjoiyL4js6R5syzApusVGtPvaXEOK7hST6txg0Spsi1TlM9K2ouV0G7a163wTVUVVVRV/UMWJfuPSj+g2enV/vLMFGJ6X22BEsdcML3cTwKN7ugOKS58v1LZ0aVNTwNerhT57iKlYnqQIFKl6+cbdmuguLkxNOHk8Ij3dft3TnR/j6S7L1/5t6YwspCarl2cL0rbWYdjR7tdeA1H2BZHuawyGWr08mwqp2Xn7WIVomiHXyL090mKkp1ucJ8LZYFu7zjeRB8Mf1Oxseo4T5HRDWHS/vXEfADNrSoY9rxFF4fByjxI0w8sNAQgq2/d1aPMFAj5DdAeiB0pSdKfaGx1v7FR4iXr8QXY0NANQVlKckvnFw6hgvre9x4w4KFCjPd1lhRGe7gwXUpM3LqlH2tY6Im0bu8yOiLIviHRfY5DM9SHeOjKnW2xEu6/NNaToTjHpuLG289M7kS5Oop2cIre7SKI70ra+oH5hT8LTbbQN69bDvRdMrhz2vEYUeU3RDVDm0sWcIf5VlR2Nnfp8IeDXPKzuYYpuu+d0f7C1CUdI+01FBampVh6PUZE53fp2yFe1yAJjGxiebq1lmCyklm1I21qHqKLb7vuCSPc1BjKn297bxypE0wy5htwSKcbON9bpQOZ0W4eonu5I2/oD2rxdA7QMA6jUW3wZzN8vFaLbY3pYAUa4NaF59PSx5rIdTZoIVICQX/N0j4yNbM8y0f3OxgbTm6+4vAOsPXQic7p9ek53fkh7yGEIbCOnu7XLH1FITYrubEHa1jqk6LaebLKtFN3Zh4jHVC4hRXeKSfVOLlp+RqInwnZ80ibaE8GsEN26pzuZQmpGeDnAuIp8xpSlIKc7xtPtDmoC++TZ49EnzI6msKc7pHu6J5XHzDNJESiK6N7S0BnOW+8n5H+4GJ7ufe299Oqi26tvAzO8PLJ6uZnTHZNTbzHyxsU6pG2tw7Cj3R94G4i0L2SjbfsT3TKnW0xE0wy5Ru7tkRZj5xvrdCBSGJZoto1EFNEde+Lv1T3dbqX/Pt0QDi8H+NKk4Xu5QRPdIRwEVW1+joCWT6wYucNqiIZ2TVArYHq0J5TGiO4kRWAq864TjZ2Ki2ttS7dZyd0M6baASE93r17Qzhsywsv1QmoFeiG1qOrlmfN02/1YEw1pW+sQ2dNt930hnm3tTDLXh3jryJxusRH5vjYXkKI7xUjRLcPLrST2wm/X+Q7k6XYpRk534hZgkeHlX0pBaDloohuI8nZDOKRaUdBKmOt/K7rYG1sSK7qzy9Nd29IdLhZnYXj5qBKtkFpHb4COgLbtPQk83S1dmS+kJtK5QRSkba1DZNFt931BlGuvgQwvt/82sgKRjqlcRIruFJPqsBbRDqB4xb7AnnMWMQwnNsTNruFTe9s1kRTb/9OoXu4xw8uT83TP368iJfMaoff+9sWIbodbE3yOiM3vUMChe7qrC2PsnEUtw9p6/LT3BCLCyz39f2AYFHqc5Ls1sd3Uo83ZFerRv1fbF8oNT3eXDzVDhdREaWUkItK21iFbhllHrG3tOk+DobYMk6JbbETTDLmGvc8aAmKVN0sUYSg93dYiytP21z7fC0BDR29My7BYT3finO4xZfkcM3UkZ84Zw9jy1FTUrij0oChxPN26uFMA1JD5t0PVhGiJOyakMEnPq5XHb6pCM3e3aCH2xS79N1ro6VYUhVElel53T8yb+jYozdfEd0gFv573nalCaqKcd0VC2tY6RPV0i7AviHLtNUjm+hBvnWzq0233bWQFIh1TQ+Hhhx9m4sSJ5OXlMX/+fFasWNHv+vfffz9Tp04lPz+fcePGce2119LTE3vzkT4S3/FKhoRVhdREEYaykJq1iHLh7/BpD1y6egMUaBHFUTndriT6dDscCk9cNC+l83I5HZQXePAHYkS3W/OuaqI7HF7u0kV3cazozqLw8tpmTXSXeVXoxVJPN8DIIi/bG7to9TkgcvPrYj/PrXnDu/1BenHhAVlILYuQtrUOUUW3CPuCKNdeAxlebv/7OSsQTTMMhmeeeYbrrruORx55hPnz53P//fezaNEiNmzYwKhRo/qs//TTT3PjjTeyZMkSDj/8cL744gu+853voCgK9957bwZ+gfR0pxy73linC1lIzVpiw8vtOF9VVU1x3eMPxM/pZuCcbqsYVezFrybydIdvNBTAQwCXQyFPie3TPfhCanY9Nxie7hK3/tstFt2Gpzs2xD/ye422YT0hff+QfbqzBmlb65Ci2zpyRXTLQmpiI9IxNVjuvfdeLrvsMi666CJmzJjBI488QkFBAUuWLIm7/rvvvssRRxzBueeey8SJEznxxBM555xzBvSOW4kU3SnGrjfW6UKKbmsR4cLf4w+hp27T6w8lEN0D53RbxeLjpuDx5kUtU9wRIdURnm4PfkYVe3GEYsLJs6hl2K40hpcDjCrWbO9TY7Z9xPeW6nnd3SFdmMtCalmDtK11xD6UjVxmR0TaF0R44B2J9HTbfxtZgUjHlEF7ezttbW3mq7e37/2Vz+dj9erVLFy40FzmcDhYuHAh7733XtxxDz/8cFavXm2K7C1btvDSSy9xyimnWPNDkkCK7hRj1xvrdCFzuq1FBNHd2u1H0QK16fFFe7p9ugfcmUSfbqs47aAaqsqLwwsUJ4pTm4eqhkD3dhue7smjivp6tpMspJYO0T3c3K3dLVp+U5HLaONmcXh5cfwK8pGpBoanu9sGnm47H2siIuJ5VxRE9nTb/TgT4dobSTLXh3jrSNEtNiKeX2fMmEFpaan5uuuuu/qss2/fPoLBIFVVVVHLq6qqqKurizvuueeeyx133MGRRx6J2+1m8uTJHHPMMfz4xz+25Hckg8zpTjGpPiGLWGHbwO4VVO2W0727pRuXQzHbKsVDhKftrd1+zU0M9AYCqKqeLx3P091PTrelRH6vyxu2o6piWFRRtH7iU0YU9PVsD0F0xy4bLqnL6dZadhU69W1isafbEN29fUR3+HuNtmGdQUN0S093tmDlMZHriCy67X6ciSq6h+rpzoZCana877QaETXDunXrGDNmjPl/rzc19yBvvPEGv/jFL/jd737H/Pnz2bRpE1dffTU/+9nPuOWWW1LyHYNFiu4Uk+oCDiJdlAwcDgehUMj2F347nZw21rez6P63CKkwriKf+751MHMn9m2TFXvht+NFpaXLZ4punz/6CbpPjzsPe7rTn9OtTcAT9Xfk0/2yAjf1hNuHTan0QNfQRLeVLXxS1W6nVg8vz3MYxe3S4+n2kTi8vEwPL+8I6L8nzeHlIrUyEg3R2lqJhGi2Fek4y5WWYTKnW2xE1AzFxcWUlJT0u86IESNwOp3U19dHLa+vr6e6ujruZ2655RbOP/98Lr30UgBmzZpFZ2cnl19+OT/5yU8ycgzb+6whIFaGl9tZaEUiytN2O52cVm9vJqRf43Y2dfPPj3fHXS/2KbQd7dra7QcMT3cwumWYEV6u6gIvAznd2gQihGWEp1tVVSoKtGeRRoj8lApPX9E3yJxuK46FVIRm+gIh9rZrv8UsFmd1IbXigQupleltwzoN0Z2h8HI7PJDLNkS5PoiIaLYV6TgT2dM9lJZhUnSLiYiaIRk8Hg+HHnooy5YtM5eFQiGWLVvGggUL4n6mq6urz2/P9P4tPd0pJtdzuiH+hd+O2Mm2O5q6ov5veIRjEeHCr4WXa3/3+oPROd3BzOd0axOIEPvOWNFthMNrb0+qcMHmGNE9yOrlVovuoe6/da09qCp4XY5wG7c0hZf3V0itXN8G7f7MeLrtdG7INkQThiIhmm1FOs5EuPZGIgup2X8bWYFIx9Rgue6667jwwguZO3cu8+bN4/7776ezs5OLLroIgAsuuIAxY8aYOeGnn3469957L3PmzDHDy2+55RZOP/10U3ynGym6U4xdiyWlExHyjsFeJydDdFcUemjq9Jm5z7GIcOGP9HT7AtGiW/tdKk6jZVjGcrojPd2eqH3BKOJl/IaRefQJJw/4e5I6ecYeC5HLhstQ9l9VVXnp0zpau/2cM2+cGVo+piwfxRC2Fnu6Kwu9OJT+C6mV6tugLcOebjucG7INK4+JXEc024p0nIlyX2Mgc7rtv42sQETNkCxnnXUWDQ0N3HrrrdTV1XHwwQfz8ssvm8XVduzYEeXZvvnmm1EUhZtvvpna2lpGjhzJ6aefzs9//vNM/QQpulNNqndyERvdi5B3DPYKw9nZrImfySMLaer0ERBYdLfFiO5QKLp6uVlEDWyS0+2NsmtFoSb4jJJqStDXJ5y8q6sLIwPJHwzhdsbfb9Lh6U724tra7efG5z7hP2u1Sp9FeS62NnQCMKY8P5y3brGn2+lQqCzy4utIXEjN8HS3+AzRnTlPt52PNRERzRsrEqLZVqTjTIRrbyTJXB/irZNNnm673ntaiYiaYTAsXryYxYsXx33vjTfeiPq/y+Xitttu47bbbkvDzJIj9/ZIi5Hh5eI8EbaTbXfqnu79RhQB4A/Fv+CJcOFv7fabodmoKr5AeD/wBWNFtx2qlyfwdBs/Iug3RV9AP2V2dmsPSdbWtnLIz5byk//7NO7X2Cm8/I5/rjMFN8Cd/1rH797YBMCZh4wJh3A7rRXdoOV19/ZbSE17r9Wn/x4ZXp41iCYMRUKU1C4DkY4zEa69kQzV053pnNdUYPd9yUpEOqZyESm6U0yqPaYiHkAiFPsC+1Qvb+/x09SpiYr9RhYCmAXHYol9oGHHJ7laTrf+tBWV3kB4rv6AGiO6M5XTHe3pjrRjRR/R3WuK7m6lAICe7i5CIZWbX1hLe0+Af3+6J+5NSrxQvUxUL1dVlTc27AXg4XMPYWx5Pnvbe+kNhDh8ciVnHDwmHMLtsja8HDTR3V8hNePBR0tvxDZIIyJVVRYNK4+JXCfWtna3q0jHWTbatr91pOgWExE1Qy5h77OGgEhPt/R0D5adTZrXtKLQY4bVBgT3dBvh5ajgj/B0+2M93bbo0+2JsmNpviEGI7yseni5z6mL7p4env1wF2t2tgDQ0uVnd2tPn6+xS073ln2dNHb68LgcLJwxip+cMh0Aj9PBz844UPtcMD053aAVU/OriUV3ab4RXi493dmGaHnHIiHKtddApOMsG20rPd3Zh0jHVC4ic7pTjFU53Zn2xg4GmdM9OIwiauMqCnC7tDkJX0jNnJZKrz9WdEf8NiVD+0aCnG6APJf+pD+OpzvgKoLAXrq7u/n1y58DmkNcVeGz2lbGlOVHfU26wssH2h9Wbm0C4OBxZXhdTk46sJpff+MgRpfmMXmkltJAmgqpAYwqzovTpzuiZZju6e41hLlsGZY1yPBy6xDh+hCJSMeZqLYdbMuwbCqkZtd7TysRUTPkErm3R1qM9HSLc3Gyi22NfO7xFQW49BPmQKLbzk/boz3d0TndvYFQuDWVwx0O4U43Cfp0Q3i/9bh0wRcIi+6QRxOodc1t7OvwUVno4fSDagD4bHdbn6+xS073im2a6J43scJc71tzx3HU/iPDKwXTU0gNNE933/Dy8Pe6nQ6KvK6wMM9gITVRzruiIEW3dYhy7TUQ6TjLRttKT3f2IdIxlYtIT3eKkaJbnItTJm2rqioPvraJ1dubKcnXhMX4inyzCrZf6D7dgSgx7YsNL1cy3KMbYvp0R4eXGw803C4nENIKqenh5YpXE92+Xi0lYMHkSmaPK+PFj3ezbk9i0Z3p8PJV25oBOGxSRfzBQiEI6Q9D0lRIra+nO/p7ywrc+HzGgw8purMFGV5uHSI8lI1EpOMsG20bbx2Z0y02Ih1TuYgU3SlGim5xLk6ZtO2jb23h3qVfRC0bV16A26l9v6gtw1RV1VuGGQtC+ALhuUbldGcqnxuS8nQrigNNdIc93c48rVGYR/fWL5hcaYZnr8uQp3ugMLL6th52NHXhUOCQ8WXxB4sM305DIbWasvx+C6mBJrp7WjIbXi5CKyPRkJ5u67D79SEWkY4zUW2byy3D7L6NrEBEzZBLSNGdYlJ9QhYxPyNWdNs1ryZTOd1L19Vz138+77N8fEWBWUDNJ6inu8cfwhcMheelqmaovKLE9OnOVI9uiPbmxni6Tdsa+0LAZ4pud4Emut266D588ggqizSxWNvSTXOnj/LCsHjMZHh5MKTy4se1/OvjPQDMqCmhOC/Bg47IPuRp8HQfNLaUn5wyg9Abbhwh/SFNjOguL/Cw3fCGZ9DTHbtMMjyk6LYOu18fYhFJIGSjbfvzdMucbjERUTPkErm3R1pMqtteiHRRMpAtw/rnmZU7ADhv/nhOO2i0uXxcRQGuJD3ddn2g0ap7uSPn5Y9sGRZUw4XUMtWjG2Kql0e3DOtj22CvKfryCksB8OJndGkeEysLKMlzM75Cq2oeG2JujBEZSmssa+r00d7jZ6jE80gYy7Y3dnLWH97j2mc+ZtnnWquwYw4YlXiwYMQ80hCBoCgKl315PxyRIeUx4eWl+W58ZiG19Ipu2dbKOvo7JiTDI9a2dreriC3Dssm28daROd1iI6JmyCWkpzvFWBleLsrTO1GeCGfq5LS2VhNmZ8wZw5iyfN76ooEir4vRpXnUt2ltpxIVUrP7Aw1DdOd7tFOLihoVXu4LRhZSy2ROtyfq7/493b1meLNXF91ugiyYXGl+bmZNCTuaulizs4Ujpowwx0rk1evxBzn+njcoznPz5g+PGdJ2TDR2MKRy/p9WsKOpiyKviwsWTGDepIroommxGOHbTk9UPr7lxGyHSMoLPOG875BfyztP07lPemOtQ9rWOkS59hokmyJjB0S17WCrl0vRLTYiaoZcQoruFCNzusW5OGXCtg3tvdS19aAoMGN0CYVeF69efzRORcHldOASvJCaKbrdeui4quKPzOkORLQMc9pEdPeb041eSE3ztBqF1DyKnwX7VZqfmTepgv+srePh1zfx5f1HMmtsqT5GfIHR2OGjuctPc5efurYeRpdGtxpLhsRj95o53P+5+ijG6V74fjHCy9MQWh5FP57usgI3/shLVNAHjry0TEsKQ+uQtrUOu18fYhHp/iYbbSsLqWUfIh1TuYh8/JFirBLdIjwJNpCF1BKztrYVgP1GFFLo1QTFqOI8Kos0wWEUUhO1T7chuvM8YbHkj9gPfAG7eLojq5dHi25zvzVyziMKqaG3DKvIg5MOrDY/8+0vTeDIKSPo8gW5+MmV7NUjFmKPBWNZty/8/y0NnUP6CYnG3tuuCejKIm9yghvCvy8NRdSi6MfTXVbgiS62lsZiaolsKxk+0rbWIcq110AkgZCNts32nG67byMrEFEz5BJSdKeYVO/k8XJu7H4AxQpDu4a2ZCIMxxDds8aUxn3faBlmFFSLJWnR/dFT8MRp0NU0nOkOGkN0Hxt4G4ACtcvMT1cUo2WYHXK6Iz3dyRRS0wWfVyukNqnMHVWUzO108LtvH8IBVUU0tPfy+LvbtDH6CS832NLQMaSfkGjsBl10jyoehNc6U57ufsPL3dGiO43F1KQ31jqkba3D7g9lY0kmBNouiGrbXK5ebtd7TysRUTPkErm3R1qMDC/P0MXJ1wm/Oxz+/FWo/yy8/OWb4MFDYd0/4MM/w70z4X8vgvb6jNj2U110H5hAdJd//gzveL/PNwP/jvt+0rZd9SfY9rb2SiOG6D6wdw0Ac5UNYduqKr6gGlG93Cbh5c4Bwsv9XaDqAkEPL49X2Kskz831J04F4JmVO+kNBBMKjN5AWHRvHqanO5HoHjkY0W38nnS3cesnvPygsWWoOPCpRsSBFN3ZgLStdYgqDEW4v8lG28ZbR+Z0i41Ix1QuYivR/dZbb3H66adTU1ODoii88MILUe+rqsqtt97K6NGjyc/PZ+HChWzcuDFqnaamJs477zxKSkooKyvjkksuoaMj2pP0ySefcNRRR5GXl8e4ceP49a9/nbLfkGqPqYgHUGyVz7TMt/4z2PsZbHkDHjkKVjymLV/7HDRugr9fAC9eBW274LPn4eHDcLTtAgZh244G+MvX4LMXBp7P+n/B74+Eh+fDsxdrRaDQPN3zlfWcveZ87T3j9f++Af+5gZGvX89YZR83KY/D2/f0GTY2xC3hftar7/O+roHnmkLM6uW6GV0EmaJodnZ89ER0y7CM5nRHVi/3RNnRPIb1mw9628Pr6uHlibyux08bxejSPJo6fbz06Z6Ehe96/GHBsXmInu5EFbb3tmuh7YPydJvh5RnydCvOPi3kJo8sZExZfriYWhrDy2X1cuuwezFIkYm1rd33WRGrl2eTbftbR4puMRFRM+QStjprdHZ2Mnv2bB5++OG47//617/mgQce4JFHHuGDDz6gsLCQRYsW0dPTY65z3nnn8dlnn7F06VL+9a9/8dZbb3H55Zeb77e1tXHiiScyYcIEVq9ezW9+8xtuv/12Hn300ZT8BunpzlDuU3dL+G81SGj1n1lb24pqLlfAlQ9H3wijZkBPK8rWt4BB5L5sfAU2vwYrkthXVv4R6j+Fhs814d/wOY0dvexu7eFbrjcoalyrvWe8Ni2FDx7RPho6QBtj2R2wc2XUsEk/bffp3lN/ekV3my66e13FAKjASFoAUJq2aOHlNvd0h/db/fToixDF3mJ9pfgC0OV0cO688QA8+e72hMdCdHh5anO6h+TpznQhtThiX1EUjp46MlxMLQPh5TLvOPWIlhsrEqLZVqT7m2y0rfR0Zx8iHVO5iK2ql5988smcfPLJcd9TVZX777+fm2++ma9+9asA/PnPf6aqqooXXniBs88+m/Xr1/Pyyy+zcuVK5s6dC8CDDz7IKaecwt13301NTQ1PPfUUPp+PJUuW4PF4mDlzJmvWrOHee++NEuex9Pb20tsbvtFub2+Pu55VOd0iFUXISE53jxa2jbsQ/J20NTfw9QdfY0Oevs2ueA8KKqFoFLjz4dXbUPya2Ek6p7tbz4/ubh54Pl2NMZ9tZm2r1iqsxtMNQeCIa2DKQi10uXYVbFxK54Tj+ObSGTzj+RnzHZ9D02YYd5g5TPKiWxeK/u6B55pCOnu1ImkOvUK5qkJQ1eao+LvwB0M4TdFtl5zuBOHljlhPtwJuvTBZP6HOZ80bxwOvbWTNzhZ8e9qjx4zj6d7d2k2PP0ie29l3sH5IFKa718zpHkSl70wXUnPG/96jDxiJb43Rqzv9hdRkCHTqES1MVyREs61IRZ9Ete1gW4ZlUyE1u0cjWIGImiGXEGaP3Lp1K3V1dSxcuNBcVlpayvz583nvvfcAeO+99ygrKzMFN8DChQtxOBx88MEH5jpf/vKX8XjCN3mLFi1iw4YNNDcnFlN33XUXpaWl5mvGjBlx15Oe7gxdnHpatH/LJwDg8rdTii6qFSeMnKYJboD8cm1evrDoTsq2XYMQ3ZEedv0zG+o00V3l1oXw2MNg0lGw39Fw1PVw8csEDr8WUGhTdXEXiBYaST9tz5Cn2ygA59RF99b8GXwSmgSA4u/E7w/iNsPLMym6I6uXD1BIzRDdLm9YlPbjdR1VnMddZx6E1+WgucsfPab+Pd0Rnm5Vha37Bu/tTmlOd6YLqSUQ3UdMGWGGl+9ubEvXrKTothDRxItIiGZbke5vstG20tOdfYh0TOUiwojuuro6AKqqqqKWV1VVme/V1dUxatSoqPddLhcVFRVR68QbI/I74nHTTTfR2tpqvtatWxd3PSm6M3RxMkRumSa6C0KdlCuaWOpyFEHkHAzR7de8wUnbdjCebmPdysnmZzbt1b6vTNG90PllfT7m0auXm1WbE4jufm0b8EFIE3vpFt1GqzOHLqwdTg+9qvZbFDWIK9gV4ekenGc3pfTj6e7TMswQ3U5PWJQOUNTrG4eO5dnvHm7+v9evbY944eUQDjFfvb2Ze17ZkLBlXCSJQqD3DqV6eaY83f2ElwMUeV043Np7736xO12zkuHlFiJamK5IiGZbke5vstG28daROd1iI9IxlYsII7ozjdfrpaSkxHwVFxfHXU+Kbnt4uh2KyjjHPgD2+vP4oj4iHUAXu4peZCx50a2L7UBP/2HbAV84vLt8kjk/Q3QXhgzRXd7noy69T3cvuvgJDkF0R+Ygpzm8PBDUn5br1b5dToWQ7u1XgLxgO27FZuHlCT3duug27On0hD8X8pvF8RJx4JgSXPpDlCZdCCcS3ZsbOlBVlev+voYHX9vEPz8eWGDG88YCw6xebq/wcoDCfC3q45+rt7JyW3pa4ElPt3WI5jEUCdFsm0wItF0Q1ba53DLM7tvICkTUDLmEMKK7uroagPr6+qjl9fX15nvV1dXs3bs36v1AIEBTU1PUOvHGiPyO4WCV6BYpPyMjT4QN0V00iqBTy2U9uEhb1kohz324K7yuGV4+SE93ZM/ryMJtCeYSQqHZW6N9R1ez3hpKxeNvjZpHJC697LdPHYan29cZ/+80EAiFABUnWm630+nUqqmhBRsUBtvN0PPMFlKLrF4+QMuweOHlEI4mSICiKBR4td+4r6PHXAbQG4gWylsaOvhsdxvbG7UHQSu3DRxNEU8YdvqCZuj60MLL7eXpBigtLgTAqfq56PGVnHT/W/zwfz+29KZQim7rEE28iIRothVJIGSjbfvzdGdDTrfdt5EViKgZcglhRPekSZOorq5m2bJl5rK2tjY++OADFixYAMCCBQtoaWlh9erV5jqvvfYaoVCI+fPnm+u89dZb+P3hG+alS5cydepUysv7iqDBkuoCDiI2uo9tGZaWYhaGCM4rpUevnD09TxMurWoRL3xUS1DPNzbErkMPLw+FQkkWUmuO/3cMjfv26N9byKdNmujqad9Ha7efIqUHJRSImkckiqLgdioRbZJ8fd6HxLZdW9vKjc+8H16QZk+3L6jiJIRDn6fb6cBQ3Q4FCkLtuHVBntmWYQP36XYY8zParznd0Z8LDFzYq9CrbcdGvY2Xsb0MT/eYsnwA3tm0j2dW7jQ/t3r7wB7deDdHjZ3aea3I66LAMwj7ZrplWD9iX9FD+vcrd9PRG+Dzunb+d/UuM3LECmTLMOsQrfWSSIhmW9kyzDqG2jIsm3K67b6NrEBEzZBL2GqP7OjoYM2aNaxZswbQiqetWbOGHTt2oCgK11xzDXfeeScvvvgin376KRdccAE1NTWcccYZAEyfPp2TTjqJyy67jBUrVrB8+XIWL17M2WefTU2N5nE899xz8Xg8XHLJJXz22Wc888wz/Pa3v+W6665LyW9IR3i53U8kmQkv17zHPncJHWiesQmOBgC6nMXUt/Xy3ma9orjh6VY18Tfo8PLYv2P49wdavn+LWsinTdq26mrRQt2nl+phxa48rYp6HNxOR0ROd0/UewP1uP3Le9tZvy0iNDnt4eUh3ASM8nG4nE6tUhhaeHmx2mFDT7cm+PpEaBjHmc/I6fZGi8Ng/55ugELd090U4+k2RPeJM6uYUFnAvg4ff3l/u/m5L+o7aO3qf/xtulc8EAiYy/Z1aOJ5UPncYNtCaoC5fW48YT+evmw++43Qju+dzdbVK5A53dYhWm6sSIhmWxE93dlkW1lILfsQUTPkErbaEqtWrWLOnDnMmTMHgOuuu445c+Zw6623AvCjH/2Iq666issvv5zDDjuMjo4OXn75ZfLywq1xnnrqKaZNm8bxxx/PKaecwpFHHhnVg7u0tJRXXnmFrVu3cuihh3L99ddz66239tsubDBYKbpFCZlJt+hu6vSxp14rgvfzZbtpDGo5oKOCmse5vHIkAM9/pIeYuwui8niTueD3+IOoUeHl8UV3U6ePFZ9tAqCVIja2a+LO16EJ/hnl+k18XlnC3+NyKPQanu5AfE93Itt+sbedAiXCA+tPc3h5UMVN0Kxb53YqEbbVisi5DE+3bXK6NaHZx7axDwVcHq34m6LneifRwsrwdDd1xuZ0a99RnOfmR4ummevnu52m9/vDHYkf7Gzd18manS0A/OOjcOqEIbpHDFZ027SQGmBuH5fq5/DJI9i/qgiAnU3WPVCS4eXWIVqYrkiIZluRQmFFte1QW4ZJ0S0mImqGXMJWfbqPOeaYfg90RVG44447uOOOOxKuU1FRwdNPP93v9xx00EG8/fbbQ55nf8hCaul9IhwIhjjvjx/wWFczKPDxPoWjXF5mOKGoSxMjE8bUQC288FEt1SV5XLPwADx5ZSgkV0ittdvPafe8wtuBiJv8BKL7z+9tIz/YBg4I5ZXT2qF55VQ9/H3/kgDsIW5ouYHH5cDnN8LLk8/pVlWVTfUdzCPCO55mT7c/FMIV4elWFAd6LTEUMNu4ARn2dEdWL0/k6Y6pru6MEIj+ruTCy/O039gc4+k28q7z3A5OmVXN7HFlfLyzheOmjyLP5eS5D3exansTx04bFXfcT3a1YDzZ2NEYtmlDVnq6jYrx2hzHV2gP1XY0SU+3iIjmMRQJ0Wwr0v1NNtq2P6+oFN1iItIxlYvYytOdDaR6Jxex0X2sMLQytOWvK3eyfk8bpYp2A95KIW16eLlDF8nVVaM5Z954Qir87o3NXPTECtT8ctMbG+8pbyRvb2zA1xGTY9vTQiAYor0nHALsC4R46oMdprAsKB1Jq6rNxdXbAsDEAt2j2I/odjkc4ZzuQXi697T20N4boJBIT3ea+3THeLpDoRBuvThc2NNt9Om2SXh5Qk93zL4QKxCTCC838qr9gegbNSO8PM/lRFEU7vnmbM6cM4YfnDiVuRO1fWNVP8XU1u1uA+PRhhr2xhqe7kEVUYOI6uVpjj5IytMdfSyM00X3zjSIbunpTj2ieQxFQjTbiuSVE9W2Q61eng2F1HIxrFpEzZBL5N4eaTHS052+i1Nbj5/7ln6BgxDFutd6+sSxptA155Nfxl1nzuKRbx9CocfJ8k2NbO30mN7YgWz7zsZ9lCvRRZvUrma++/9WM/fOV3l3k5av/Z+1e2ho72WMVxP7I0ZV04IWCpsfbANgbJ7uhe5HdLtdSsKc7v5su1EvLFWgRHzGl/4+3ZE53aqqmhXZNU93Bx6zZZhdPN2JRHeMp9sVK7oH9nQ7jYt+zP5lhJfnubXvmDKqiHvPOphJIwqZO0HbNz7e1ZKwX/fa3a2ERXfYI9HQYfTozovzqX6wcSG1cG907beNK9dFd7MMLxcR0cSLSIhmW5Hub7LRtjKnO/sQ6ZjKRWwVXp4NpPqELOIBlK4wrD+8uZmmTh+zRyiga+Ibvjaf5UteIDLC2sifPunA0eS5nVzy5Co2d7ijPN2JbKuqKm9v3Md4pT1q+c7dtby6XmtPd9VfP+LFq47k8eXbAJg7CqiDihFV5JeMgF4opgsnQUa5daGg9wqPh9vhCOd0DyK8fKPei7wwg+HlgZCKWwlE2dala1dFgVKlkzalTFuQyZxudz4UjtS81d4SfX4DiO68Uu1fM9w5OgoBVY0QwCq01aJ07tXfit5evYFweHksk0cWUex10d4bYNPeDqaPLon5GpXPdreZmjvf46BNf68h0tMd8MEL39P2geoDtVoGBZUw65vab1j/T1AcMG4e1K3VBkh3eHnhyOh/42E87NDtPbVjBT9w/Z1gUz5qXTlK9YEpn5YU3dYhmngRCdFsK9L9TTbaVoaXZx8iHVO5iBTdKSbV1QJFPIBiW4ZZNd939Wrk35tXCa8B7gImjCpnwtGz4L9/Da8YIXCPmTqKHy2aSuurRVREhEDHs20gGGJ7Uxe1Ld3McUV7jDdt19o7eV0OGjt9HPHL1wDwOB0cUOKHOnAUVPDbi46BR7TPTCsLURDQ5VF/nm6nY8BCavFahn2hi+6CKNGd7vDyEC6COCJFtz5nh6JQTCd1DhtUL3c44bLXQQ2aoq5PSxhXxPy8JXDENdrfRrjzc5dB+QSomgl7PoHtyyEUriQO4KjV7R9zXjDDy90xwh5wOBSmjS5m5bZmPq9r6yO6a1u6aenym2OV57uo19/buk/7vprSPG0+a5/V3tjw7/AAq5+Ayinwyd/62sUCAdsvs76h2XPy8YnX8WjRIqz7BwR6qXnrNyzWN03w76txfn9Vyqc1UJcAydARrfWSSIhmW9kyzDqG2jJMim6xEVEz5BL2PmsIiMzpTl9O9z49lHZCoS5MjYrgsZXBY/5/7vzxdDiKo0KgY+f66FubmXnbf7np+U8BOHhEdJiv29dKVYmXF648gtJ8TYQVeJx8//gpeP2t2kr5FUypLkfVPamPfH1SuABbP55ul1PBpxqqIr6nO94DjS/qNXd/oZK5nG6/kdOt/79PeLnSiVfRbZnJnG6AsnFQPtH8bx/bugvCbcIu/CeMOURbsWI/7d+mzbD5NXj3Qdj6Zh/BjeIw7UCMp9sIL8+PFN2qCktvg4cO47hSrfL++j3RERaA5uUGSvO1hwUl3nAoQW1LNy6HwsHjy6Bbr0NQNh4OuRBmn6sdC7WrNMGtOKC4Rv9Nk+H8F2DaqQPbLZW482H22VDUj6f7kAshvwLqPoW3fgPAe8rB2nvtuxN/bhiIVjRJJKRtrUM024okELLRtv2Fl8ucbjERUTPkEtLTnWJkTndqw7De3byP//uwltu+MpMib/Tuuq9dE9sVzpiQbSMM2CBG4BbnuampqkbZrv0/nm1f/7yB3kCIFVs14TKrIggt0KCWMFJpo9zRxc/PmMX00SUsu/5oWrr8TBpRiNOhwCOGsNa82UpeGfS2MS7fDz0tUe/FQ+vTbXi6kwsvV1WVTUZOd6SnO+iDYCBtAtfM6Y7wdDuV6JZhHkcIgmTW0x2HPrZ158HilZBXEr29vvkk7FoBoRC07oD6z6BsAkw9WQvfNgj6UP4+ThtLjV9IzRsZXv7BH2D5/QBc0HETS7iN9XvaiOWzWu2hTkWhh11AaZ52k+RA5WBlE65xh2kF3PSK+VQfBF95QPu7cTP87VzoqIev/xH2Ow7adkHx6PQXUUuWUdPgsmXw1LegcRMs+gV/+HA/FjSchdPfqaUIpHjuooWSioS0rXWIZttkin3ZBVFtm2zLMAOZ0y02ImqGXMJed71ZgBTdqb043fPKF6ze3syBY0q58PCJ5vLO3oDZdsmoXG6K7Vgvcpye2DMmT2DzCu3vYJwLfl1bdAGz/Yu1StXukVNg34fMKA9y4IwqAEYUeRlRFJEL2xUtuskv08RZd3NYCPUruiMLqSUnune39tDRq3laC5XouRPoBmdxwu9LJYGQiqs/TzedEYXU7CXy4noyyif0XdFbBJOPG3jAUNgOHjW6T3d3ZHj5jg/gs/+DFY9qK+eVUthTz0OeB1i8584+wxqe7gp9nyvWPd0KcLLzAzonn6StaDzgidz/KyfD996DkD+cm142fuDfkmkq9oMr3oPOfVAymsrtq6BBf6+nDQor+/34YBHNqyUS0rbWIZptRbq/yUbbxv4mkOHloiPSMZWL5F7shcVkUnQ3dfpo7Rq4jZHVpEp0q6rKhjotvPZT3btn0KgXjMpzO/D6dW9gvPByxQnevoJzzOga0xvb2uWPsq2qquxp1bznfzj/UJ68eB4VitYGrGzMVAAchniOhxFCXmCI7vLwcuO9OA8CDLSc7ujiUebPSWBbo4haTWletKcb0lrB3B8M4YkopBYKhXA7ddGtQLHSTZ6i76N293QP9xh2OFF0D6wH7TerRIeXlzd+DEtOhA9+r+WXzz4XLl2GqjiZ7/gcZ8ceM40CIBhSWbOzBYCKQl10655uRYFyOlgweYS2cqJUBocj/VXKU4HTDSWjARhTUUyHqldoNx4upBDRvFoiIW1rHaLZNhlvrF0Q1bbJtgwzkKJbbKTotjdSdKcYq0T3QOFXvYEgJ973Jif99q2EbYbSRaqeCNe2dJve27UxottojTSiyIvSq78XL7w8rxTifL9SUG56IVu7fQQjLqQtXX5TFB19wEiOPmBkOD+2YrL2b2+rFrYdS6AX/J36fPoR3f316XY6wjndSXq697RqQntqdXF09XJIa153INjX020UVTOWVRi1tjOd0x2DFTdVit4Kq9ipHQvGbUyv7uku6NIK8lE2Ac76f/DVh2HE/igFFdpipZPPI/K639vcSGOnj/ICNyP0XtyuCPtWOjqYM75MW2A8GOrnAY+ojKsooA2tdRg9rf2vPAREu8EWCWlb6xDNtiIJhGy0bX+iOxtyuu2+jaxApJSNXESK7hST6gIO8apLxjuA6lt72dfhY09rjxl+anwu0lOWDmKrlw/VFoaXG7Qe1EYeLISLqI0o8kaIizjh5YkKluWXm2IwEAxR29xlztUQsJWFnnB16S5DdE8KjxHvZt8Q1YoDvMZ8Bie6PU4l6Zxuw7b1ejh8dWk+RY6Y7Z3GtmGBkJbTHVm93Nhvu9FEYpmq281mnm4rqtM6XNp2LDJFt+7p1luGeUP6A5JRM2D66ZoXGsz9o0zpiMrr/seaWgBOnjUaV0zBG4cC4/K6w/usWT+gbNi/w26MqyigTS3U/mOB6BatUrFISNtah2i2ldXLrWMw1csjBXY25XTbfRtZQbxIBSm67UPu7ZEWk6nw8oYIYb1SL/4FcN+rG5l756ssW1/f5zNWkaonwhvqw6I7GFKjRHiU6I7NXfUUaWHlkctiyS8PO8BVlW37Os251rVpIrW6NC+8viGWi0aZfZ3NZZFEho/HCCja68Je5/483Q5HOKc7yerl9W3aelUlXoqUWNGdHk+3qqp69fLoQmrGfttFPgBloRbtTZvmdKfU0623I8t3alERQVXbl/1BzSbukP5AxFMQ/UF9/yilg/V1muju8Qd5eW0dAF+dXRNnvlDljtjWSdQPEJXpo0toQxPdDfv2pnx80bxaIiFtax2i2VZ6uq1jqJ7ubBLddt9GVhBvm+aiHeyKFN0pxkrR3d8T1ob2sNBasU0T3c2dPh57awsQ7mmdDlImuuui2yWt3R32aBmVy0cWe8KeLsPTrSiJi6oZ5JWZ4c4KKp29fnOuu1s07+Po0vzw+kZ4eX55WMjHyyXtiljPwJhD8zb9Cx1h4R4HtyuyT3e0gE7UP3iv7umuKsnrW0gtTaI7GNL2/djwcuOY6FS0hxilpqe7b4/qTGJFoRxDdOcpmugOqWpUxIbH8HS7Y0S3vo+VKZ2s3t6MPxjijQ17ae8NMLo0j8MmVvSdL1CiRhwzWRxeXprvxqkfV59v25Xy8UUrmiQS0rbWIZptRRTd2WRbWUgt++gvZUCSeeSWSDGZ8nRHhpCv2tZEKKTy/97fblZJ3t6YvrzeVF2cDNE9tUorhLa2Nhxma/zeysKI8PJ4YeWJBEdeqTkvr1PVeiTrc61rNUS37ulW1Yiw8Irw2P15uvWcXO0zugBv2mJ+N/2cBN0OJSy6kyykVt9uiG4vBWi2CTn0YmxpCi83vbcE4j4s6lC1hxhevZK33VpUWSK69ZxuN4boDlcuB3AFDE93YfQH9X1mtKeb7Y1d3PXS59zzyhcAfGV2DQ6HEme+oHQ3a63MIKvDywFKyrWK5TtqU9+rW7QbbJGQtrUO0WwrUv6pqLZNpmWYFN3ZQ7xtmot2sCtSdKeYVB/syTa6j/R0N3f5WbenjSff22Yu29mUftE9nNwnfzDE5gat7/TXDx0DwMc7W/hoRzOtXX4aO43wck/81kjG34kEh8OJ4ikCwK2ERXdkTrcZXt7bBiG9aFpBRVhEd+7r44mO8ogbGH+3bO/7XhxcTgWfOrg+3UZ4+ajiPPLRhFyvV2+jlCZPt1+fk1sJe7oj99suJT/6AzbL6bYmvFzLY3erenh5KNyj2+NyoAT0bRPr6db3kVOnaPvgkuVb2bi3g6oSL985YmL8+QKoofDxkMWeboCqUVrLvpbmfbT1pLZrg2ihpCIhbWsdotk2GWFoF0S17VCrl2dDIbVc9PDG235231dzCXvd9WYBdvB0A1z9t4/Y1+GjyOuiozfAjqYuVFVNy8GXiovTtn2d+IMq1Z4evuZdxVrHJ1APjz/yMjNrSpjqD+JydHLC509D7WrtQwURvXrN8PLEAlfJKwHqyVN7KKGZ7YBSu4qJxXV8xdHBvI5d8PFqWPWn8Fju/PCYL14F/7gCpp2mvQI9sPKxvt8bO4cBRLfbGZHTrQa1Kul6pe94tg0EQ+b2rypyk6d7krs9FeR370lby7BAlKdbn35keDmxotuenm4rcrqdqiYKgyp0+/Qe3S4H+PRK9wlyuqcU+zn5wGr+s7aOEUUenrr0S2baQ7ycbkBLcfAWg689aqxso6RMa41WpHby5oYGTp9dk7KxRbvBFglpW+sQzbYihpdnk21lTnf2IXO67Y0U3Skm1SfkZC9Khqd7RJGXfR29bG7QbubvPONArv37Grr9QfZ1+BhZbH1/3lSEYRlF1H6Z/xdG/ud1HvBEvLlP/9cD7ARQYN7lMObQ8DqGAM+PCPOOnWe+JszdgXamOzo10b1qCVcd7NHGXq2/QCvOduq90WOHdO/a+he1l4HDBVNP7juXRP+PQRPdEYI02Nuv6N7X4UNVwelQqPSE25h1usupgLR5ugN6qzpPgpzuVoqiP1BcnZZ5JYuVolsJGX26YadeKT/P7QxvG3f88HKlp4XffHM2h02s4Pjpo5hQGV4vrqcbtGiLyAiPyBZ62YT+u0qVTt6QolsYpG2tQzTbStFtHUMV3TK8XGyk6LY3UnSnmFS3Khisp/ukA6v4f+/vAOCHi6Zyxpwx/Oa/G6ht6WZHU2daRHeiYl+DwcjnnqJoLZJ8Iw+ky1nC53XtprgDmH3AJIqP/wHUzIke4EtXaOL3wK8nnuehFwKrCDm8NIW8QCvqyGm8pxYQDKkcOqGcfLcTysbBMTdB6Vjtg3Mvgq5G2P9EqJoJHz4JjZs1V+OYuXDYpVAyOvxFI6dp86n/DJweOOLqfn+72xmR0w1aiLme8xv7QMPhcJjtwkYVe3Ho4cpBVaHTqYutdOV064XUPI5Q3JZhy0JzeTrgoc1ZxncvuBAmfTkt80oWS1qGubXwcDWg5+YrDjbt1dIm8tzOcBRCAk833c0UeV1cfOSkxPPV+8UbNqerKfywyVNsu37oKUMX3SV08UV9+wArDw7R2gOJhLStdYhmW9kyzDqG2jJMim6xkTnd9iZL78YyR6Zyuvd1aDf1px1UQ7cvxPiKAq44ZjIA4yryddHdxaETEnt+U0U8YThYjMJv5SGtMJnnaw/hqZnDw3/6gLc37jPXW/P1E6DA03eAsYfC2D/0P8+JhwOgFlSyXqkGVrNl2uXc3jAegM8vPAnccSpsV8+Cbz0Z/n/Nwf3/GEWBk+7qf50IXE4HQZyEcOAgFFVMLV4UgSm6S/LMcOVO8uhW9QcsafZ05ynxw8sbKeXHgcsgAN+dfGxa5jQYLCmkpud0BwN6dXwwRXe+2wl+Pbw8gac7brG+2Pn6e/X/6290NULhCH2csmHN39YYolvpYktDR0rTZ0QrmiQS0rbWIZptRfR0Z5Nt4wk0I7xc5nSLiczptje5t0daTMb6dOvh5aNL87jnW7O5euH+5nrjKzQv2o7Gbq1gmMVPMFMRhtXQ3otCiHy/XpisSCuadPC4MnMdt1OhNH/oecGRti3waIfC+j2ax6yi0KN5IjOA26nNJaAYxdTCLcDi2bZe3/ZVxV7waYKuizy6MaqXp6mQWjDs6Y4MLzfmahQAu/HkaWmZz2CxspBayPR0KxGebkeEpzuR6G4ZeL4BzdMdFV4er6J/tmF6ujvp9AXZ2947wAeSR7RQUpGQtrUO0WybTLEvuyCqbZOpXi5zurMHGV5ub6SnO8VkQnR39gbMNkQjivqGjxt5oLsa2+DRo7WQ0+/8K8I1llpScXHa295DOR04VP0JbOFIAGaPLTPXqSz0DsvOUaJbF9jr69phLFSX5A153OHi1uOEg4ob1F4I9PV0R9o2skc3Pq2tWqeaR1covS3DAvqcvEowrqf7jDljueJbsxhbnp9oiIySDk83KGzTozi8kTndfcLLy7R/k/F06/uHOd+upvDnsrRyOWCK7nKntn9vbujQjoEUIJpXSySkba1DNNtKT7d1DNXTLcPLxUaKbnsjPd0pJhOi2/By57udFHr7PkcZp3u6exq2wZ6PYfs7WgiqRaTK0z1SadH+U1Bp9nQ+eHyZuc6I4jhh5UOYZ6Sn22i9ZfbozgBulzYXv6L/vmDYgxfX090W7tFteE678NKh6p9Pc/VyT4KWYQ6Hg3EVBba9AFjq6dbzrlGgqVMTyVpO9wDh5b6OqIcucecbEboOaMd2lvfoBqJyugG26MUjU4FoXi2RkLa1DtFsm4w31i6Iatv+xJcspJZ9yJxueyNFd4qxSnT3F35lFFFLVCTNCC/3t+wKL2zenpL5xWO4T4R7/EHaegKMVFq1BXpoOWiefMNTWlk4vKJwkRf8/JhQ8hNmVMX7SFpw6Z5uvxle3r+n2+zRXZIXFV7eEUpveLlPz+n2OuJ7uu1+4rdEdLt10W3ev4THzHM5Enu6IyuOGwI6dmxjvrqgN6cbGV6eA55ur9qDmwCbGzpSNrRoN9giIW1rHaLZVkRPt4i2jV0W+38purMH6em2N1J0p5hUF3CIV4Eykad7RFF8z68hur1ddeGFzVtTMr94GHMeaiE1Mz/daYjuUVHvz9bzuuOF0g+GSNvmuzWbVhTl8dgFczl73vhhjT0cPK7kc7ojq5dXRRRS61K9tAf0z6crvDzC0x2verndi5pYUr1crxxuim4lPGZU9XJ3QewHw8I7QV53uHq55umOql6eC55ub4n5ZzFdKfV0i1apWCSkba1DNNvK6uXW0V9l8v7WkYXUxEYWUrM3ubdHWkwmwssNT3ciEVpe4GZEkZfRSlN4YYv1nu6hPhFu0H/PRK/uuSqK9jqfOms0LofCgsn997sezDyNGd7x1QMz6uUGcOknTb/RNixOeHlkFIFRQEoLLw9XL281Pd2pEyP9YfbpVuL36bb7id8ST7cxZhynQX6Up7swzgp6iHmCvO4+nm7jjcicbmOMbMTp0upTACVKJ1v2SU+3CEjbWodotpWebuuQ4eX230ZWID3d9kYWUksx6RDdsU/vGvR2YYnCyxVF4Ztzx1L1ToToTkN4+ZBFty4ix7rbIUAfT/cps0Zz/PRReF3Dqy4ez7ZOZ+afQ7md2rx8ccLLY59ihtRwjnBVcUR4uZpHW0A/vNPdpxuxw8tTWkjNGFO/f3E4wmMWu/yA/kaspxs0wdy8bUDRHTSqlxvnhVwJLwfNk+9rp4QuPm3upscfTEnXAdGKJomEtK11iGZbEUW3aLbtL7e3v3Wk6BaT/orjSTKP3BIpxkrRnUjEhsPLE4dbnzd/fLSnu3lbSuYXj+FenAzPbbVTq8Qd6+kGhi24I+dltwu+0TLM14+n29gXfLqaczkUygrc0Z5uf3pFt+Hpdsd4ukXzDFgiuvWHzq6IhzpFjogCaYlENyQhuvVCag79mMiVQmpghuBXe3tQVdjemJr6BaLdYIuEtK11iGbbZGrW2AVRbTsY0a0oihTdgiMLqdkbKbpTTKoP9kjP5oDh5Qk83QBjywuYWtAeXpDG8PKh5nSPoEVbEEd0p4J4DzTs8ESwj+gOJBbdAd27XOh1ae8ZOd14afLrn/elJ7zc7NNNwPR0i3AzZZCO8HK3M/ywyBTdrnyIt98lG14e0m+YDNEdCkDLTu3vbPd066J7SrFmgy0pKqYmWiipSEjbWodotk3GqWAXRLXtYMLLFUWJmxMsGna6n0s3Mqfb3uTeHmkxmcjp7ujRwktL8939jlWjhNuEqa27IBTsZ+2hk6rw8rKQ7pmPCS9PFXb1dLuM8PJBiO4io1WcHl7eqebRmubwcqNPt0vmdPcdUxfdkZ7uQoe+XWMrlxsMNqfb4dQEPEDrjugxshVddE8s1Lz9m/ZK0W13pG2tQzTb2vUaHA9RbTtY0S093WIjc7rtTcpyun0+H3v37u3zdGz8+MxVgc4EmRDdvQFNPHtd/TxDCfjw9OwLjxsKQFstlKV++ww3DMsQ3UV+Q3Rb7+m200naY3q69cOzn/ByI2y50Kt7OU1Pdx7d6JEPaa5e7iZI0Ka27Q9rRbduG6eDPHopoYtC9OrbsT26DQwv9YCiO+I4K6jQjmuDHAkvH1ugie6NUnTbHmlb6xDNtna9BsdDVNtK0Z1bSNFtb4Ytujdu3MjFF1/Mu+++G7VcVVUURYnKK8gFUh3Wkpzo1vsj9ye62/cAmpCrDVUyyVGv5XVbILpjw1sGL7p78ODH62/VFlgkupMJ3c8EpqdbHbhPt77pIzzdmujuVvLoUg3RnZ7wcqNPt5sAqm5GEVrBGFjSMswYU79/KaSbt73XUEYHq3rv0BYO5Ok28rO3vwvblsNhl0BBRcR89dZ8Tqd2rJiiW4GC4VX4tz266B7t0R5MpUp0i9YeSCSkba1DNNvKlmHWMZSWYQ6HQ4puwZE53fZm2KL7O9/5Di6Xi3/961+MHj065zeuVZ7u/oRhWHT3U1ysbTcADUolO9VRTKJeq2A+KSXTjDvn4Xi6K9GLqDncluWl2vUpu5HT3aPqh2ecPt2GbSNzugFTdAddBXQH9JZhoQAE/eDsP/1guBiebhdB/Poyu9m2P9IRXl7cU8tIpQiAqrZPtYXxiqhBWHQ3boJnvg3r/6n9f/2LcNzNKCv/qI1tHmcOWPQLWPP/tAc1478U7vWdrei/b4RbO0Y2N3QQDKk4HcPbdqJ5tURC2tY6RLOtXa/B8RDVtkP1dGdDTrfdt5EVZMP2y2aGLbrXrFnD6tWrmTZtWirmIzxWFVLrb2wzvNzdz5NX3fvV4hrJTr+eI21RMbXYg34wT4RVVaWho5cZih5SWzQqfpGpFBCviIsdnl4bLcN6zerlfT3dpuhW4+d0h1yF9BBRWM/fBU5rBZiR0+3GT08cT7fdL4DDfVjU75j5lUADiuKgRS2kTOmkuFv3SMfr0Q1h0b37I+2lOMBTBHWfwNPfQuns1eer53QrDpiwQHvlCrroLlr3N57wfMINvkvY2dTFxBEJbJokVuwLEg1pW+sQzbYiim7RbDvY6uXZ1KfbDvdz6Wa4kaYSaxn2Hjljxgz27ds38Io5glWe7v7G7vUnEV6ue7rbPaPYqY7UljVtTckcYxnOE+GWLj/+oMpIxdrQcrDvBd/wdPeanu7EOd1GxfBYT3fIXYgPF6pR0swf9pZbhT/C021YUYSqtAaWiu4C7UFXsOpAlgYPBaCwS68wnsjTXTgi/PeYQ+G7y+HilyG/Qhtbfyvo0/YPJQdvMNj/RCifhBLs5RjHR5zhXJ6SEHPRbrBFQtrWOkSzbTKRfHZBVNvKlmG5hWgRGbnGsO/SfvWrX/GjH/2IN954g8bGRtra2qJeuYaVojvRQTSY8PLuvCq2qtXasqbNKZljLMM56Bv09mfjPXp7sxwU3S5dPPWEjEJqg8/pVryFgILq0JeHAtZOmnCfbpdsGZZwTLfTSbNSBkBep+HpTiC6aw6Bwy6DE38OF78CVTOgaiZc+QF89x2UAs0THtJz/nNSdI/YH77/EXzpCgBGK418Ud8+wIcGRt64WIe0rXWIZlu7XoPjIaptZSG13EK0/TTXGHZ4+cKFCwE4/vjjo5bnaiE1K0V3oiesSVUv18PL/UWj2WKI7n2bQFUhxQflcJ4IG5XLJ3g7oBfL2oWBfS/4Hpc2h+5+crr79umOrl4edGlCTlWcgD89olufi0sNyJZh/YwZKhwFPeAI6ZnviaqXO11w6t19lxeNgqJRKHmlQJ2ZL64oOSi6QTt/VewHQLXSzH9T6OmWNy6pR9rWOkSzbTyBZ9c5i2pbKbpzC9EiMnKNYYvu119/PRXzyBpSfUKOzEmJN7aqqmFPd7853Zqnm5IatqsjCeHA4WuHjnoork7JXA1iD/rB5NXsbdcEZo2rLa2i20453VGebgfJh5erqpnTHXBqQi6kuHBCWkS3X/d0Owmaz3Gk6O475ulHHgyvRqyUyNM90Nh6PrMpum2w72aM4tEAVCtNbNwrPd12RtrWOkSzbTLC0C5ko23j/aZ4Vc9Fw073c+lG5nTbm2GL7qOPPjoV88gaUl3AYaDwcn9QxXgg2X94uebpdpSOwYdKg6uKqsAe2Lcx5aJ7OAf91gbNUzte2astKB2X0rlFYteWYW49YsH0dPdXSE2/LhZ7XZo4V7Xlflc+0K17uoGQ9REnZvVy1Y8jjui2+wXQ0pZhEWOOGTMheqVEOd0DjZ2v9fkOmfbt5/jPdko00V2lNLNpbwehkIpjGBXMRWsPJBLSttYhmm2TaWtlF7LRtrJlWPYh2sOhXGPYohugpaWFP/3pT6xfvx6AmTNncvHFF1NamuXtauKQjvDyyBOnEVoO/YSXB3qhvQ6AvMrxwHZ2KmOoYo/WjmjSUSmZa+ych3LQG0WQqoNaX3EqJ6d0bpHYNbzcrYuFnpALnER5umMvkv7IlmG+cD/uoFMT3SFTdPuxmihPt77Mbrbtj3R5uvvUKUhUvXygsfPLtLFNT3cui+4xAIykBb/fz67mbsZXDu1hBsgbFyuRtrUO0WwrPd3WIcPL7b+NrGA4kaYS6xn21li1ahWTJ0/mvvvuo6mpiaamJu69914mT57Mhx9+mIo5CkW6c7qN0HLoR3S37ABUcBdSVKndnG4J6d7txk0pmWckw7k4fVHfjosAJT16OLyeq2kFthXdRp/uflqGxQ0v10PLceXjdGmfDSnpK6RmzMWp+oUOL7ekennkmIUjo1caoqfbCC8PyvByzaaKE5cSYgStvLt5eB01ZF6cdUjbWodoto2db+Qyu5GNto33m6ToFhvR9tNcY9h3addeey1f+cpX2LZtG88//zzPP/88W7du5bTTTuOaa65JwRTFwso+3fFErCG6PS5H4u9s2qL9W7EfFYVa7+Z1Rq/ufRtTMs9IhvqkzRcIsa2xi3FKA4oa1MSInqtpBfFEtx2eCrr0Pt2+JAqpGZ7uokhPt6cQl+4tT6foNvp0OyMKqYVCIeE8A5aL7vxycLjDKw01pztfF936c7ec9nQ7nGaaTLXSxCvr6oc1nLxxsQ5pW+sQzbZSdFvHUEW3zOkWG5nTbW9S4um+4YYbcLnCkeoul4sf/ehHrFq1arjDC0e6W4b1+pOoXG6K7klUFHkA2BDQxWzj8EX3k+9u4/q/f0wwFP3bB3vQb93XSTCkMt3ToM93P7DwhBGvkJodTlCGp9tneLoDyXq6I0S3PoYZXh5Mp6dbFlLrd8zYEPNE1csHGju/XBtbhpdrmMXUmnln4z46eoe+z4sWSioS0rbWIZptZXi5dcSz7UDrSE+3+Ii2n+YawxbdJSUl7Nixo8/ynTt3UlxcPNzhhSPtojuZHt0Rnu5CjxOPy8GWkC66m7dHibrBoqoqd/93A899uIvPdrdGzW+wB71RdfjQ4iZ9vpOGPK9ksHt4uc8ouRAcuHp5kdcZDi/3FJl54eGc7vT16XaE/DKne6AxiyJCzIfq6dY/F24ZZm/7Wo5eTG1GUSe+YIg3NzQMeSh542Id0rbWIZptpei2jmTmK0V39iHafpprDFt0n3XWWVxyySU888wz7Ny5k507d/K3v/2NSy+9lHPOOScVcxSKVIe1JJvTnZynez8URaGiwEM95QTdhVq16+ZtQ55fS5efdt2jVN+miUPjtw82DOuLek00Tvfo+ZgV1hVRg+htZKeTtNOhoCiRnu5+RLd+r1LkdYO/S/uPpxCnLrqD6RTduvpzqAFZvXygMaM83UOsXu7Utq0huu1uX8vRi6kdVqmlY7yyrm7IQ4lWqVgkpG2tQzTbyurl1pHMfPurXi4ydrqfSzeipUHkGsM+a9x9992ceeaZXHDBBUycOJGJEyfyne98h2984xv86le/SsUchSJj4eX99eiOEN0AFYUeQKGraKK2fBgh5jubNaHnwc/spWfD8/8TPugbt/aZb39s0j3d47G+cjkM/EAjk7idDnr7Ed3GfI1blUKvM354OekT3f5gCAfay65RBP2RVk93ZDG1oVYvN8aWnm4NPbx8aoF2Hnnls3o21g+tZ7f0FliHtK11iGZb6em2juF6ukFcb7co9xxWINp+OlgefvhhJk6cSF5eHvPnz2fFihX9rt/S0sKVV17J6NGj8Xq9HHDAAbz00ktpmm1fhi26PR4Pv/3tb2lubmbNmjWsWbOGpqYm7rvvPrxebyrmKBTpLqTWM1B4eTCgVy/HFN2Vel53S4HeL3gYxdR2NXcDMEvZwqjmD+GTv6Hohb9CQX+f39Afhqe70rdLn2/6RLfdnl67HQq9qlG9vK/oDqP9v9ATUb3cU4hbL8YWTKPoDgRV3AQiZiWm6La8kBqkxNNtji2IfS2npAaAiuA+5k+qoNsf5OInV7Kvo3eAD/ZFegusQ9rWOkSzbTLFvuyC6LbtT3THK6QG4hZTs9v9XDrJ5kJqzzzzDNdddx233XYbH374IbNnz2bRokXs3bs37vo+n48TTjiBbdu28eyzz7JhwwYee+wxxowZk+aZh0nZHllQUMCsWbOYNWsWBQVD748qOrYrpNa6UxNcrjzTE1ReoInueq8uuus/G/L8djZpnu4Zju3hOXdqB8BgPHD+YIht+zpxEyCvs1ZbaGG7sNh52e0E5XY5wjndcQqpmf8HCjxOHA4lwtNdlKHw8lBYdEeEl9vNtonImOgepqfbrF5uc/tajn5+U9r38PtvH8qEygJ2NnVz2z8Gf34T7QZbJKRtrUM020rRbR1DFd3S0y022ezpvvfee7nsssu46KKLmDFjBo888ggFBQUsWbIk7vpLliyhqamJF154gSOOOIKJEydy9NFHM3v27DTPPIxr4FX6cuaZZ/LEE09QUlLCmWee2e+6zz///JAmJiqZK6SWQHQboeXlE0F/AqaFl8NWz1TmAuweej91I7x8uhIhuvd9oc13EKK7vq2HQEhlqnMfihrSKjrrLYCsws6i2+VwhHO6+/N0K4pWuRyiwsvdRl59GluG+YIqLvSLt74sFAoJcwFMa3h5ZCG1YXq6ZXi5ju7ppm0PFQVu7jvrYM783bu8sWEvgWDITLlIhmy+cck00rbWIZptZXi5dcjwcvtvIysQbT8FaG9vp62tzfy/1+vtEynt8/lYvXo1N910k7nM4XCwcOFC3nvvvbjjvvjiiyxYsIArr7ySf/zjH4wcOZJzzz2XG264AaczM91ehuTpLi0tNTdkSUkJpaWlCV+5hpWiu99Cau4EO1BMPjdApS661ytTtAWNm6C7eUjz29mkhZfPcIQr2Cs+7eAxxUAS4+xp1ULSZxcZlcutbRcG9hbdHqfSb053GEXr0Q1R4eVGn+6gcYjrof5WEgiGcBO9j0ZetO1i20SkV3RHerqHKbr1bWx3+1qO7unG3wm9bRw8tozSfDedviCf7W7r/7MxiHjjIgrSttYhmm2l6LYOKbrtv42sQLSIDIAZM2ZE6ca77rqrzzr79u0jGAxSVVUVtbyqqoq6uvhFU7ds2cKzzz5LMBjkpZde4pZbbuGee+7hzjvvtOR3JMOQPN2PP/64+fcTTzyRqrlkBenO6e4NaAdWnuHpVlVYeis0fK55N43Q8QjRPapEe4K0od0N5ZOgeSvs/ggmH9d3Amufgzd/Dd94HKpm9Hl7Z3MXDkJMVXaayxSiw14doYFbkhmie6a3AXqBSmtDyyH+Aw275AC5nA58ahKiW9GLqEF0ITWfIbqN8PIgVhOV0+3U5i5FdxKie6h9uo2xHW6g1/b2tRxPAeSVQU8L/PUcHHMvZt6kiSxdV8/7WxqZPa4s6aFEu8EWCWlb6xDNtlJ0W8dQRbfM6RYbEXO6161bF5Vnnap6YKFQiFGjRvHoo4/idDo59NBDqa2t5Te/+Q233XZbSr5jsAx7jzzuuONoaWnps7ytrY3jjosj4rIcK1uGxc/pjvF0130C7z4AG1+Bza9BRz2gwKQvm5+ZMVqLQPhsdxvqmEO1hbWr40/g42c0Ab/+n33eUlWV2uZuJil7yFd8dKle1JpDzXZRQVX7QzHEYD/UtWoe8/0c9doCi/O5IXob2e2poNuphHO6Q35IdAJVHBGe7r7Vy9NZSM0fCuFS9IcXuugWoRWMQVpbhpWNhxFTYcIR4BzSs0/hWtikhSkLtX+3L4f/+y5HTCgC4P0tjYMaRtrWOqRtrUM028qWYdaRipZh0tMtHqI9HAIoLi6mpKTEfMUT3SNGjMDpdFJfXx+1vL6+nurq+Kmoo0eP5oADDogKJZ8+fTp1dXX4fAM7A61g2GeNN954I+7ke3p6ePvtt4c7vHBYebAnldPdrOdWV0yGM34P578A162HAxaZnzmgugi3U6Gly09L+SxtYW2CvO623dq/Rph6BA3tvfQGQmZo+efqOLrHHWlGhYccmphQ/B0D/rbdLZqne2xod3j+FmPn8HK3MyKnGyCoHWPx5hdfdGvrBTJVvdylzUkED4ZBWj3dTjdc8R5c+K/Uj53LnPkYfO89yCuFkJ8jy7V0lZXbmgkEk/faSNtah7StdYhmW+nptg4ZXm7/bWQFou2nyeLxeDj00ENZtmyZuSwUCrFs2TIWLFgQ9zNHHHEEmzZtijq/fPHFF4wePRqPx2P5nOMxNBcL8Mknn5h/r1u3LiqmPhgM8vLLL2e0LHumsOJgVxQFVVXjhkAb4eWm6G7Vw7xHHwQHnxt3PK/LydTqYtbWtvG5c38WgObpVtW+edTtiUW3UURtXl4tBGF9aAKlE76C4rwH8IdzTZPydGuie4Rfr1xucY9uEEF0RxyegR5w5/V9Wq1EFlIzcrqLInK6DdFtfU63PzKn26E9MBChKq2BFblQ/Y7pGF4hDxFztyzH4dDSYEbNhB3vsl9oByV55bT1BFi3p42DxpYlNYy0rXVI21qHaLaNnW/kMrshum37E92yenn2ELtN7R6RMRiuu+46LrzwQubOncu8efO4//776ezs5KKLLgLgggsuYMyYMWZO+Pe+9z0eeughrr76aq666io2btzIL37xC77//e9n7DcMeWscfPDBzJkzB0VROO644zj44IPN16GHHsqdd97Jrbfemsq5AtqOdMsttzBp0iTy8/OZPHkyP/vZz6JODqqqcuuttzJ69Gjy8/NZuHAhGzdG96JuamrivPPOo6SkhLKyMi655BI6Ogb2yA6EVaI70f97Y/t0t+o9rkvH9jvmgTVaiPn7XWNAcWph6IZX28DfA116WKYuuvd+9BI97z0GHz1FnR7mcaBLE/rr1fHUuiagHPQtAIJGITVfhF13fwSNm7W/fV2w4WUIBtjT1oObAEXde7T30hBePlCRukziiiykBgk93Qrxq5e79BNtIJ053SEVV0xOd9RcbWLbRPR3nNlpzHSMLTx6/QlHwzrmTaoEBhdiLm1rHdK21iGabfsTgnZDdNsmY+tY0S16Trfdt5EViLafDoazzjqLu+++m1tvvZWDDz6YNWvW8PLLL5vF1Xbs2MGePXvM9ceNG8d///tfVq5cyUEHHcT3v/99rr76am688cZM/YShe7q3bt2Kqqrst99+rFixgpEjwy1wPB4Po0aNsqQk+69+9St+//vf8+STTzJz5kxWrVrFRRddRGlpqfn04te//jUPPPAATz75JJMmTeKWW25h0aJFrFu3jry8PADOO+889uzZw9KlS/H7/Vx00UVcfvnlPP3008OanxWi2+FwJAy/Cud0689PWvQq4qXj+x3zwDGlsHInH9X5tBvUuk+1PEhdMAPQHt556drHZ8v/xcyl55mLxo3+JvA1JgW3AbAuNIGD28MFnUJ6TrfDCC9vr4M/LYKCSrZ8+33Gf/RrXO8/BCf+nD0t0xiv1KMQAk9RdKEpi4h3cbHLU0G30wEoBB1unCG/WUytb063Eje83K2Hl/vVdIaXh/CYorvvqcXuJ38purOIUdO1f/eu59AJF/Lq+no+rU2+grm0rXVI21qHaLaVots6hiq6I++BRPd02+V+Lp3E/ma776eDZfHixSxevDjue2+88UafZQsWLOD999+3eFbJM2TRPWHCBCD9T8LeffddvvrVr3LqqacCMHHiRP7617+yYsUKQDvY7r//fm6++Wa++tWvAvDnP/+ZqqoqXnjhBc4++2zWr1/Pyy+/zMqVK5k7dy4ADz74IKeccgp33303NTU1Q55f+j3dseHlyXm6Z43Ri6nVtqIevgil7lOtUnki0Q3sff13zAS68FJAL5UtH1PEIsoCDQBsUsewt70nQnTr8zU83fVrtZ7T7bu58v6neLzkX1QDoc2v0dAxiQMVo4japL5h7hZg7/By3YYOjya6g4mLPhR64oSXm4XU0tcyzB/Zp1t6ui0bMx1jC88ovdPC3vVMn1sMwPo9UnTbAWlb6xDNtnafXySi23Yonm7RRbfdt5EViLaf5hrDfgx01113sWTJkj7LlyxZwq9+9avhDt+Hww8/nGXLlvHFF18A8PHHH/POO+9w8sknA5oHvq6ujoULF5qfKS0tZf78+WYD9ffee4+ysjJTcAMsXLgQh8PBBx98EPd7e3t7aWtrM1/t7e1x17PCYzq48HI9p7tsXL9jTq0uxuVQaOz00TDpK9rCTa9CZ0QIZky4+eF+zTbPBI4BYFTPVg5QNJHf4a6kjUL2tvWavz2oq27F36WFkjeG88JPcq6guneb9p+dK0ENMcVpiG7r87nB7tXL9aqiii5eddHd6YsJE1cccVuGOfWc7oBxiKejenkwhFvRvsfh6lukwu5PnWPnl8rq5akcMx1jC4/h6W7dyYwK7VjY0tBBjz+5NAtpW+uQtrUO0Wwr0nxFmiskN99460jRLTZSdNubYZ81/vCHPzBt2rQ+y2fOnMkjjzwy3OH7cOONN3L22Wczbdo03G43c+bM4ZprruG887SwZ6OgW38N1Ovq6hg1alTU+y6Xi4qKioRN1u+6666o5u0zZvTtWQ2ZzOl2aMLWyMEu7V9057md7F+leYA+6q6C6oM0Ybbu/wDo9gX5YuMXUZ/x6oLqueBRhNwFuFU/C51a1fP2okmAVtG8b3EOtLD3iGJsFzn/a/7t8LWxv1LLdK/mMU9HPrc2LxvndBsPLhy6eNXDy1u7Y8SzAsV5yYSXpyen2yykFie1xC62TYT0dGcR+eVQrEUsjezeTEWhh5AKG+uTq9shbWsd0rbWIZptRZqvSHMF6ekG+28jKxBtP801hi266+rqGD16dJ/lI0eOjEpoTxV///vfeeqpp3j66af58MMPefLJJ7n77rt58sknU/5dkdx00020traar3Xr1sVdz6qc7kiic7r18HK3Ixxa7inWWuYMwIE1JQCsrW0Nh5V/8ncAlizfytsfahXqQ0pYQLVTwDp1Ij3l2oOWUx1arkRvmeadjgovN7z+CtC8DZo2m+OUKF1RcznU8QWTjR7daahcDvbO6fa49AcXets1Izy8j+g2CqkFA1qFc9Crl8cWUkuTp9vM6e7r6bb7yV+K7ixD93Yre9czrXpwIebSttYhbWsdotlWpPmKNFdITU636IXU7HI/l06yPac7E4RCIb744gveeecd3nrrrajXYBn2Hjlu3DiWL1/eZ/ny5cuHlRudiB/+8Iemt3vWrFmcf/75XHvttWaJeKNJen8N1Kurq9m7d2/U+4FAgKampoRN1r1eb1Tz9uLi4rjrZbR6eWRoeRLff9C4MgA+3tUKB34DHC7Y+QGsfY51e9qoVjSv+XbvAeZnPnPNIISDtpL9AZjg0OyojJwKRHu6zTxpgH1fhKuWR9BSMBGAQx0bGaMalcszJ7rtcoIyPd2KLl6Dmqe7pSc6N3uio14T3f6ItmyeQrNlmF81wsutz+kOyJzutIyZjrGzAiPEvP4zpo/WHjCur5OiO9NI21qHaLYVab4izRWkpxvsv42sQLT91O68//77TJkyhenTp/PlL3+ZY445xnwde+yxgx5v2KL7sssu45prruHxxx9n+/btbN++nSVLlnDttddy2WWXDXf4PnR1dfV5kuN0Ok3RNGnSJKqrq6MaqLe1tfHBBx+YDdQXLFhAS0sLq1evNtd57bXXCIVCzJ8/f1jzy2ghNUN0D1BEzeBgvWftxztbUIur4ajrtTf+fT1djbVUK80A/LdjivmZzfmzAdjpiQ4Bzx+t3eDWtnTTpecdh20BG9e8RbB5GwBbQ+EHG/8pPQuAkxwrqPTrof0ZCC+320nayOkOxOR0t3RFi+dvO5cxomdHOLTc4QKnxyykltbq5aGIPt2yerllY6Zj7Kygaqb278o/ctm26xiv1EtPtw2QtrUO0Wwr0nxFmitI0Q3230ZWINp+ane++93vMnfuXNauXUtTUxPNzc3mq6mpadDjDbl6ucEPf/hDGhsbueKKK/D5NGGQl5fHDTfcwE033TTc4ftw+umn8/Of/5zx48czc+ZMPvroI+69914uvvhiQNvBrrnmGu688072339/s2VYTU0NZ5xxBgDTp0/npJNO4rLLLuORRx7B7/ezePFizj777GF75zOa073PEN3953MbTBtdjMfloLXbz7bGLiYd9QPY8B+o+4Rv9T5KlS663w9N47v8E4DdZYdAA3ziH8thEWONnHQg8ybWsmJbk+Y5j5wvMHrv2ziVIEGnlxcDC7ja8X9sCVXzbOfBnAMUKXpo9KHfgWLr24VBchehTGHkZAcN0R3w0eMP0t4bnZutKDDV2wQ+PZ3AUwiKgsvI6TYLqVmb062qKv6gituhh5fHKaRmF9smQoruLGPqyTBlIWx6lep973GRs4Tf1o1FVdUBbSVtax3SttYhmm1Fmq9IcwUpusH+28gKRNtP7c7GjRt59tlnmTJlysArJ8GwPd2KovCrX/2KhoYG3n//fT7++GOampq49dZbUzG/Pjz44IN84xvf4IorrmD69On84Ac/4H/+53/42c9+Zq7zox/9iKuuuorLL7+cww47jI6ODl5++WWzRzfAU089xbRp0zj++OM55ZRTOPLII3n00UeHPb/053Qbfbqd4ZzuASqXG7idDjOve83OZnB54NR7ATg6+AFVaKL789B43i85CaaeQlflgQC80xpRiM5dgFIylptP07zdW/ZF52s7FMUU1fXO0fxv8Gg2MY4/Bk9lzT54xbOQ9aFxfHjEI3D6b5OaeyqIt43skgMUz9O9aW/fIlAK4PG3RbULA8Lh5aH0tAwzKtW7FN3T7ZDh5VaNmY6xs4L8cvj2c3D6AwDMcGynpctPXVvPgB+VtrUOaVvrEM22Is1XpLnC0EV35L8yp1s8ZE53apk/fz6bNm1K2XjD9nQbFBUVcdhhhw284jApLi7m/vvv5/7770+4jqIo3HHHHdxxxx0J16moqODpp59O+fzS3zIsXnh5cqIbYPa4Mj7c0cLHO1v52pyxMHYugcLRFHRq+dUhHOxTygic/jDsP4KyVzcCsHqvym61ghqlCSqngMPBQWPLOPOQMfxxafR8g658QBNln/WMZJc6il9OepzV25sJdvm5vE2LUnhx+hHJGyVFKIoS9TTXLicow1MdUIxCaj52t3SjZ8ibOBSgpyWqcjmEc8L9aSqk5g9qNjQKqTlcfUW33S+AsmVYllIzB4Dpzl2AytraNkaX5vf7EWlb65C2tQ7RbCvSfEWaKwy9ZRiE74ukp1s8RHs4ZHeuuuoqrr/+eurq6pg1axZud/S97UEHHTSo8YYkus8880yeeOIJSkpKOPPMM/td9/nnnx/KVwhLusPLe/wR4eVDEN0H68XUPtrZYgzOvppjqN74V+2/RaP4+KZTtGJdQGWRFjbc1hPgc/d4apxNMCJcaO2mk6fz3EMeIruYd7nLAS33YXNICx0fX1FIjz/EO5v2AbD/qCKmVscvTmcldhXdnlhPd6AXnzOkxZNHoChAd0sf0W22DAvp61stuvWHTWb1chlebtmY6Rg7qxg5FRQnJWoHVTSzfNM+TpjRfwqLtK11SNtah2i2FWm+Is0VUuPplqJbPETbT+3O17/+dQAzhRnCukFRFLPdcLIMSXSXlpaaG7K0dODWVLlEOkR35NNJM6fbAbTt1hYmWUgNwqJ7/e42egNBvC4nm8uPpBpddJfUmIIbYERRWEy9F5rBcc41MP5L5rKRxV5Onz2Gh98Of0eLowxDdG9XDdGdz+jSPN7ZtI9TDxrNXWfO0iqwpxm7nqDCOdlhT7efELGebgUFupuhRy8Q5dUeXDj18HJfmvp0B3RPd7h6uRTdVo2ZjrGzCpcXRuwPDZ8z3bGDdzaNH/Aj0rbWIW1rHaLZVqT5ijRXkKIb7L+N+uPVdfWUF3o4dEL5oD7Xn16QDJ6tW7emdLwhie7HH3887t+SzFUvzw+1h72ZRckXIhtfUUB5gZvmLj/fWbKSrx0yhgbXbA5RPeQrPiiJLixXUeg1//5T8BQmzv8q5809MWodo3K2QUMo7MHepmqVyydUFnLM1JGcMWcMI4u9ZAq7XkiNnG4/4Zxun5rA093TAl1axAAFI6I/b4pua3O6A0H94Y9i9OmW1cutGjMdY2cdo2ZAw+dMc+zkjb0d7G7ppqYscYi5tK11SNtah2i2FWm+Is0VpOiGzG+j3kCQf328h2MqGqnc/ALMuxxKRg/4uedW72Ll8/cx2dXAjNNOIP/A07QaJUkg2n5qdyZMmJDS8VKW0y3RSHshNd3TnYfWxxmnB+IInkQoisLRB4zkhTW7eW9LIx9sbeTEGdW8EzqQE5wfQnH0CaIywtMdwoGjegYMULhha0+R+fc2vV3YuIoCFEXJqOAG+z4VNESzL0p0970AKqCFl3caorsSiPCUq+kKL9fm5lH0fueyT7dlY6Zj7KyjaiZ89jwLCut4pBVeXV9PQ3svh4wv59hpo/qsLm1rHdK21iGabUWar0hzheGJbuM+SBZSGx7/u2oXN7+wlmfyf0ml+gl88gyc+3eoPjDhZ9bWtvK7/3uVZe4/agv+/Q9Y93e48J9JfacspDZ8XnzxRU4++WTcbjcvvvhiv+t+5StfGdTYQxLdc+bMSXpDfvjhh0P5CmFJp6dbVVV8Rni5nkuLK4/B8utvzOZbh43jx89/yrbGLpZ9Xs/u0Nc4eKTCyEPOj1p3RGG0SB5Z1Fc0x863Xc3nr4FjOWK/MvZ8UYGiwNjy/gsZpQu7XkiN6uMB4xAN9OJzJPJ0t0JXo7agcETU532q0TLMWtFterodenh5nJxuuyNFdxaj9+ye4dQ6PNzxz3UEQipjyvJZfuNxfVaXtrUOaVvrEM22Is1XpLmC9HRD5rfRym1NlNDBIaHPNA9JWy0sOQn+502onBz3Mzc9/yn7hbYDsE8toUzpwrX1Ldi5AsbNG/A7RdtP7cgZZ5xBXV0do0aNMltNx2MoOd1Degx0xhln8NWvfpWvfvWrLFq0iM2bN+P1ejnmmGM45phjyMvLY/PmzSxatGgowwuN8YQtHTfXhpcbwGN4uocguj0uB4dPHmEWF/IHVT5RJ7PxlL/D/2fvvOPbKO8//j5tb8dxbGfvvXdCElbCCnuVvUdbZgmF/gItUKDMsgqUvUfZewRCIAkJ2SF778SJneF4D637/XF3smTLS9I590j3fr0ClnR69Oiru+fuc9/VfmjItulJtoCgA8gO46mu990liSdcN5F96asM6pjB6UM64LK3fv52OIy6QGme7ppATrcHt9ev5HAHEfB0V9bxdKt3O91+1c6+1qle7tBahtXxdBvFro1hiu44JmcAAG2rd2DDi1eNzMgvrqLaU/+kadpWP0zb6odothVpvq091+0HK1i2syji95ui+8jvTyt3F3O8ZQV2ycdWf3v2pvQHdxksfyvs9h6fn3X7Sukt5QPwq38wn/omKi/++mSzPrOp373S7eWW//3O9M9WsedwaHtfEwW/309OTk7g74b+tVRwQ4Se7nvvvTfw97XXXsstt9wS0idb22b37t2RDC802sHeGi3DgkW3U3Yrf9hbLro1ju2bwyu/1hYNCJfzKEkSWSkO9pcpIj9ceHj97y7RrW0yyQ4b39w8KeL56YFRQ3Hs9Qqp1eCR6nu6Ay3DKlRPd53w8hq/BFb093T7tZxuH8hgqePpPtJhXs3BbBkWx2R2AUcaFncZY9MOsbSqPTLg9vrZXVRJ79zQzgmmbfXDtK1+iGZbkebb2nO9+s0lbD9Ywf+uG8f4nm1b/P5oW4aBKbqbg9vr55eN+zm2b7uQYsDFlW52HKrkTvtSAL71j6VY7sc9rIc1n8Hk++qlZu4trsLnl+nrUER3VWZvXj04iD9Y5yBt+h4K10HugEbn05TofvLHTXy1Uim6/OmyfC4b35Ubj+tFVop40YlGYvDgwXz33Xd07tx496ioV42PP/6Yyy+/vN7zl156KZ9++mm0wwtHa+Z0a0XUJAls/sg93RqjurUh2VG7aLTPDD9W26CQ8uBq5nXnF/QEXdumRDwvPREnp9sTSCUIRpJQPd11w8u1QmqtFV5ex9NdR3Qb5WZGY5ie7jhGkqC90k/zpQllzLnjOHrnKLUmdh6qf7fftK1+mLbVD9FsK9J8W3Oubq+f7QeVNqD//HptIH2rJZg53a1zPffMrE388Z1lPPTt+pDnV+0pwYmb46yrAPjJP4r3ivrhd6Qq7X33LGbZziLyi6sC79HORf1t+wDo0mcE2+QOzPCrYeXznmpyPo05klbuLub1+duR8HNn23nMtN5C1sKHOe6xWTz382Yq3fpeJ8YzO3bswONpumBx1HtkUlIS8+fPr/f8/PnzcbkiF4Ci0po53TVBPbolb/Si22mzclRPRbS1S3M22MJLE9rpLlvYbcKJ7m5tkyOel54Y9aSvVYB3y0E53b4wOd2ghCuVFyhP1PN0a9XL9V1M3epFQUB0W8zwcr3GbI2x45JBSr/N1DXvkZfupJt6I3DHoYp6m5q21Q/Ttvohmm1Fmm9rzvVAuXo9h8wJB95i0efPtngMM7xc//3J75f5fLnilV6zbC6e/10G/7sIvr2drZs3cIl1FklUQ3on7J1GUIODHe2UGiKHf3ycza9cxdZnTqXm7fPh/Qvo9MuttOMwXWWl9si4sUcxdXAez3mUYl3+NZ/C4R2Nzqmx3/2er9Yiy36+bfssN1T8l66W/dxo+4p/+x9n6U8fccG/P2d3USUUrlW+x6qPY2UqE5Woq5f/5S9/4c9//jPLly9nzBjlbsyiRYt4/fXX+cc//hH1BEWjVUW3VkTNZgVvtfJiFKIb4Ni+7fhpfSGdGyl0poWhNFR5vJ73WJI4tm/9CsFGwKgnfS28vEauDS93h+vTrT2sLlH+r7UMU+92emkd0a15ul2ScqdPcoTuP0axa2OYojvOGXw+/PgPOLgRdi2gS9ssAHYVmZ7u1sS0rX6IZluR5tuac91fqlzP9ZLyud3+Cb41n1Ez8SScef2aPYYpuvXfn5btOszeEuW3ukb+HPvGxYHXLuUt7HbV8zn8Uo7x57BsVzHf+CdwC1/RZs9PXGgDZGCbslkP4J/2AzjlGrA6sbXtzn8utHDnJ1bmrhnM0dbVbP38AXpe9G9wZQYuAKvcPl6bt429hQX8oVPoTeTgyNiVu4sZIm1nQMUisDph5BXIy97iBJZxgnUZuGHdCwOR5a1I3mpF4A85Xz8Deqqa3ibOiNrT/X//93+89dZbLFu2jFtuuYVbbrmF5cuX88Ybb/B///d/sZijkLSO6Fa8ik6bpVZ0R5HTDXD+qE788ege/N8p/Rvcpq1awTw7TOXycPO94bheDO6UEdW89MKoJ/2whdR8/rqau+5DSFaEhFUV7T5aK7xcuQGUjLIfSo7QyAaj2LUxTNEd57jSYbDi7WbZm4Homx1B4eUPfrOOi15eiKdOOKdp29hh7rf6IZptRZpvq4putWbOxGxFQFnxU/j1/S0awxTd+u9PX6u50W2S7fSVlBpW/7OdySJ/P+wogjt/2F/gmL9xTJ92ALy+twuVGb1xy1Y+803ibt/13OG5ntXdrgRgqlUV7tm9wWrDZrXw7/OHsqnPdQD03PUJPNpNqYLu97NqTzHHPzGbvF+m8dDG03B+cF7IHCW/Mo/CEmWf6mZTUxE7DIOpjyNd9R0MOBNPVh/8SAzwrFUENyiiW699IH8ZvFy/c0i8E5M+3X/4wx/4wx/+EIuhhCZ4gWpVT7fdUnvHyBZdKy6nzcr0qQ0Lbqj1cOekhxf4dedrNUiedDiMetIPiG4tPNxbg6cxTzeAKwPUquG1Lcdax9Ot9el2oRT0kxyhOfxGsWtjmKI7ARh5JSx/G9Z+Qfd+yk3hXWp4+f7Sal6bvx1ZhpGWUO+3advYYe63+iGabUWa75EQ3b2cJVCmPNcp/zvYvwFymuftjoXoFj2nW8/fyOvz891qJff64TP60vXzQgCeLj+BQv7AsZaVuC3JvDb1FrBYGNwxg6wUB0UVbsYf+jter5dTR/WmT24aD367nm3lKbwpfUKaXK58QLva39likbjmkstY+8wnDCyZqzy5eyGs+5z75uZhLd3FOY5f1e8cOk/Jp1yTabnj/ZJKwQ1kdFI26DQK/vA2duDnhUtZ/PUrVEjJ3G97A8lTCRUHIDW2karr95bQ67vpUFMc03FFICZqqLi4mFdffZW77rqLoiKlxcHy5cvJz8+PxfDCELxA6VmluH5Od3B4eXjvcyw5Y1gHThvSnqsmdAv7ulmRNHq0nOzqQHi5O2xOtyX4oRpaDrWiPSC6dW4Zpnm6k9TWdZIzVHQbxa6NYVYvTwA6jICcgeCroc/BmQDkH67A6/Pzw9qCwE39QxWhBVFM28YOc7/VD9FsK9J8m5prtcfHOwt2UKiGhkfDAXWMTpZDtZ+HTMl39zV7jGiql2v/F87TrV7nxKSLkCw36uVdsO0QB8vdtEm2MzmnBJvkp1ROZsKwQfw07Vgmn34JN1xxGUlqcWKLReLqCd2wSFDiteO2JvPnY3tx6pD2APyeX8HXvvG1H9Au9OaKZLHQ55YvuTDvW57yKBFblT/+ixW7irjM9jMWSWaubzCvDPlf6Pt8yjXZXlV093AcVl5I71jvOx0/bhRfpv6Bd7xTcKco86Joe73tomHl7mKee/4J7PmLkK366xWjEfUKt2rVKvr06cOjjz7K448/TnFxMQCfffYZ06dPj3Z4oWh9T3e48PLoPN3NoWNmEs9dPIIRXdqEfd28ex09Ds3THSy6vWE83cEPUmpFtybGW83TreV0a/3i7ckhtjSKXRvD9HQnAJIEwy4CIGPTJ0y1L2OF/WpKf32J79cUBDY7WO6u8zbTtrHC3G/1QzTbijTfpub60pxt/OPLNTz1wvO4P7sB3jwN/nuU8u+n+5SN8pfB22ey9L37uOrFn1i5uzjsZ2me7lxZEd2L00/EK1vI2PE9bJsdk/k2to2Q4eVzHoMH2sJjPZFrlPCAiPcnrxteOApemgS+8BWpv1qhhJafMrg99kObAEjuNIgnLxxOr5xULhvfjYm9s0Pec9PxvVl574m8c80YvrhxAt2zU2ifkUSPdin4ZfjEM6F243Z9632m3WrhuUtH83XyWRTLKSSXbuU667dc4pgDwDu+EygoD+0drRVZ1kR3R+1GTkb41lbDu2QCcNDeQXmiicJtLeWjhVv4m/V9AFa1PzemY8caj8fD5MmT2bx5c5PbvvTSS+Tm5ja5XdSie9q0aVx55ZVs3rw5pFr51KlTmTt3brTDC0VriW7t7l1tITVLq3q6myKeTqRHCi08vFoLL/e58fj8YeYb9CC5bdDzEnarhK+1WoapUR4uWRPdSabo1mnM1hg7rhl8PkgWpD2Lecz6IilSDd61X7Joe1FgkwNlNSFvMW0bO8z9Vj9Es61I821srn6/zK+Ll/K9YzqPVN2PY9V7sONX2L9W+TfvKbwHtirie9tsRm1+imf2XcqiV27il59nQMFq8NauOZrozvIdACB10Cm84ztBefG7OwNCcMmOIpbtPAyVRYpQbOZ8m9pGONFdXQrzn1H+rjyIrNqiwf2prFApqvl4b5hxl/LcvKfg331g/3rYOQ/2r1N+ly2zFI/3wc2wbxWU7qXG62PGWuUm7RlDO8CBDQDYcpsO/U9z2ZnUux0DO9TWOZrYSxHny+XebJB6gj1ZCfsOQ3aqkwcvPIqXfacBMN3+P1J9JVQl5fGzfzj7S0PPXRafog32liiiu51f2acC4eV1GN5ZcaZt8yk56LEU3VVuH2VrvqeL5QD75Uzu2jU6ZmPrgd1uZ9WqVc3a9uKLLyYlpenWyFGL7iVLlvDHP/6x3vMdO3akoKAgzDvilyOW022zgkcT3fp7upsiXk6kRxK7TTk0A+HlXrf6ezfi6Q4S3aD06m7t6uVOtIgL09Ot15itMXZck5YHPZUCLqmolV4PbcHnlwM3sQ6Ynm7dMPdb/RDNtk3N10iir7G5Ltx2iDEVv9Dfsoty2cUb3pN4OXs6h8/7CLocBcCXz96Of7uSd7vF34F0qYrrLV9z3NwL4MWJ8H5tXSQtRD29RrmG7tGzD8/6z+OQnKZ0X1j+Nuv2lnLBSwu4/9WPkJ/oB59e0+z5NrWNcKJ71YfgLoe2vSE1F23Wdb+f2+vnzbkb2f3E0fDbf6BiPyx5BaoOw4L/QnkhLH4ZNnwXeE/+3DdY+dGD8NwoxfP95AB2fvUwZdVectOdjO6WpQh1gHaN10NqCK1VL0g8lvs43LwM0js0ur004Rb+4z2LCkkResWDr8KHlcI6olvy1YAsk1+s7FOZbiX3vEHRrXq6V1WokawxFN0z1u7jKN9SABYlH80ht72Jdxx5Lr30Ul577bWYjRd1ITWn00lpaWm95zdt2kS7du2iHV4o9BLdDed0q+Hl9thVL48FDXnmjYhR56q1/ArxdMthqpcHP04JDWWyWaRWE91an26nbIaX6z1ma4wd9wy9CLb8hB8rFnzk+A+QRDXHDerGd6sLTNGtI6293+YXV7GpsIxj+7SL+99RtDWhsfk+/N16Pv89n7euHkP/9umtPbV6NDbXj5ftYZR0EIA9fa/i4bXH4N7j5+UvrPxy4qWk7fqNcy2/ALDQ359LPHcz+ywPJbOeJrtmF7lSMdK22XBoK7Ttyf6yGiT8OKsUgeRq24VunST+m38G/7C/h/z7u9y3fAB+Gc6Uf1aE1fqvoHg3ZHZucr4NfqeaMvC6A9dBQhRSk2VY/Iry95jrYMHzqHVd613PPfL9Bn7/7UeudBZQKidjT2lDUmU+zH5UEeCAvPYL/FaXduVE9p5ZtOEXkKDSnkWyp4g+qx7nTtsZHBr8f1gtEhzYqGwcJiS8OYzv0RaLBH4Zctq1a1Rwa9x+0kB+6vIolbn/JqVyG442Q2HuLA5VhobDS7IPSvawt7gKJ25cbjWiqwHRPahjBjaLxIbqLHDQpOj2+vzM23KQlbtL8Pj83Dy5l+IIBDw+P/O3HGRs97a47BY+XryLp6wrAOh39PmkfFRfOxoNr9fL66+/zk8//cTIkSPrebOffPLJFo0XtcI444wzuP/++/F41P68ksSuXbv429/+xrnnGjteP9YcOU+3JWZ9umOBSCd+o87VblPmUeUP6tMdztMtBR3CdT3dVglfK4nuwL4oq/uhwxTdeo3ZGmPHPQPOggl/YfW4fyveI+DmIRL/PGMQACVVoceLadumkWWZh79fz79/2Njodk3tt1v2lzHhkZ955qem8+iawx0fr+SqN5bw/C9bYjKekRFtTWhovrIs8/GyPewvq+G2D1eo574jS0NzLav28P2afXRQRXe/vv35+uaJdM9O4WB5Da8dHEAFtddln/kmct7IrnQeew7bT/2AcTXPs1Aaqry46iN8fplD5TW0pQyL3w1IkNaecT3a8oVvIj6sSHuXc3DHGiT8TLUuqp3U6o+anG+j36lwFXx2bcOebr8fDm7Rp42UtwYK14aE2dd+rg8ObIKKQ/Vf2zFP8f7bU2DohWB1BKZX9/t9v2YfIyxK/vUifz9+dk1Rhl/0UmAbqaoIa/le/LYk9lg74ZQ8JEs1LPL34wzXG7iPuxeAG2xf8eeK/yqdg4rURts5kXm6M5LtDO6UCUCXtsmNb6xisUicODCPdtntoMtYslKduOyWsCmI8oEN7C2uor2k2s+eAkltwo7rslsZ0CGdXbJasbwJ0f3Qdxu48o0lPPXTJp77ZQsfLd0TeO0fX6zhyjeWcOHLC3jsh42U7lhGrlSM355M79En8eVNE5v1XY8ka9asYcSIEaSlpbFp0yZ+//33wL8VK1a0eLyoPd1PPPEE5513Hjk5OVRVVXHMMcdQUFDA+PHj+de//hXt8EJhjPByU3S3BKPO1abeoa3yq6La68Yt169eLrnSALVdXHIdT7fVgreV+nRXub1I+HHIqofQ9HTrNmZrjB33WG1wwj8ZIsvU7Pwf7FvMDYP9kOYk3WWjyrRti9lbUs1Lc5SLz8vGdyW3mS0l6z5+5PuN5BdX8eGSXdw6pXfU89q8X2nB88TMTQzskMFx/WLb/sZIiLYmNDTf/OIqiiqUc0mH/XOY+eEKTr345nrnv5by29aD3PrBCv511iBOHJgXk7muzi+h2uOna1IRyEBGJ/rmpXH7iX246f3feXbePjpaxnC+bS6yzcWxp13H5GG9ADhxYC5ZKQ4+qDqK8Y4VsOpDDo34C345qOBVWnuw2hnboy3/nZ3BAmkYE+VlnG2dx4bkUeS5D9dOauUHMHEaSFJkohtg3ZdIPuVmfYjo9nnh4ytgwzfQeRyc+m/IG1zfUOu/gdUfK9ccFQcU733eIKWWxpafYPPM0OuRpCxo003Jn3aXgSMNuoyrrU/kc8PuxVBdXGuPSz6u/exVHyr/H3yu2jbVETa8PL+4in0l1Yx0KDfflvv7MHf/IE61gQXlWvqAnEE7qQSAX/1D+K2mB9Pt/0OWrPzTdyVbDlTwsu909njyecj+Gtkb3oV3toLsA2eGMrcImXZCH179dRvnjgjvgW4KSZLokJnEpoN1f1OJ6r3rqHT3okOgiFrHRo+l4Z0z+WqPuk6W7VX0RQNRtD+tV6IxxrStptPhxexZtAUGX8K2ShcfLVV6l6/cU8LKPSXconq5LT2PB5uTVGdMulbryi+//BLT8aL+xhkZGcycOZP58+ezcuVKysvLGTFiBFOmTInF/IRCr5ZhDYvu4OrlqvBqherlTdFQOLwRMepcterlVX6bEo/icwd6YQdjScokILobCy9voAJnrKh0+0giKCTXnhxiW6OE7TeG2TIs8ZAkCVf7frBvMRxUPCA9s2xcafuQ24O2M22rUO3xccHLC+ncJolnLxoesl5uO1Ae+Hvt3pIGRXdj++2K3cWBi7i9JdUcKKuhXVrkxUE9Pj8HyxXPmSzDbR+tYOH0ybjs1ibeKSairQkNzXf1HkX4dHWW8zJPYtvsx/3uPBznvgjJWRF/3mu/budAWQ3vL97VYtHd0Fz3FlcDMu1RPN1kdgHgpIF5tM9wsa+kmrfkEznLvhD7yCuZOqo2BNlps3LuiI68++soqiUXrsPbKdv6GwB9k0rAhyKQgFFd22C1SHxQcxQTHcv4g30+dAS2w8/SWI6zrkQ6uAn2/g4dR9Sfry80ZQbAV+eSQpaU40KqVuyfv+Rrhh34HCxWpZDYhm+UDXcvhBcnQf/TYeBZoL6P7XNhaZj8122zG666XrZX+QdgdSjCe8vM+tvZkpTr3LJ9sOhFOPN5xSu+7ivl9cFqTrzVHtbTvXRHESAz1rYF/LA/YwhrizqwxdKBXhbl8x/0XckztmcA+KZmKD/6RnF2m230m3gO6auGwrYi/jNrC27f8Rw1oDtnbL0Pdi1QPqBd36huCh3Tpx3H9IkuJbdjZhKbwqQg+jbNBHrR11UMfhoMLdcY3qUNby1Io0xOIk2q4o3v5vLrrhpkv4+OHTvzz3NGYLVI7C2uYldRJVaLxJtpL5JcsRgOg//pR1mSdT2p8gAGdclmW7FMQWk1F7dZD+VAn5Oi+p5Hij17FC9+p06R3RiBKEW3x+MhKSmJFStWMGHCBCZMmND0m+KY1s7prtb6dNstUK2G5BiwermRT/xGnavWp7vKbwmIbrc/jKc7qQ2wT3lQ52LEZg3O6Q5tIxFrqty+QI9u5cNdpqdbpzFbY+yEoq3qTT2ohDOf4lhFb+vWkE1M2ypsLChj5e5iVu4uZkr/XM4aXtvrdev+WtG9Jr+U4/uFb5/S2H77xI+hoelr8kui8kzvL6tBlsFulTjL9TtdajaxZ/YOeo0/C1LDj7vtQDl/fnc5I7u1Yfop/Uhb/qJSwfiCd8CZFvFcWgPR1oSG5rsqXxF9t3XaiC1fuc5xbP0B/nchXP1DROKmosbLr1sUYbx852H8fhmLpfnjNDTXvcVVpFNJkqze/Fb7H9utFi4f341HZ2xgjdyDVZeuYmT3+sfEBaM788qv2/neO5qzrb+SvPpd4Dx6Oouhsna8FKeNIZ0ymLlrJGVyErkcgO1fAPBWzbGMG9yB5I2fw/K3oeMIpNL80PlWHaz32fO2hD63Ru7O9uR2WKT5AOQteRTyg25QSVY4/WnY+gus/UzJI1//VX1jjfkj5PQDZ7pyE2Ljd7DpR2g/BEZfB6mquJRlpXhZ0XZo2xM6DIe9K6BgFdT6qyF3IHQYoQjct06DdV/D1CcUz3lNCaR1gK5KwTqsjrA53ct2HqYDh2jrPwSSlb4jj4aZu/jWP5ZbLZ9D3mAeu+Yf8NovVBftYWb1SEpIxXfxx9AhgymebSzcVhSoX9Nn8uUwph98eKlyMyCn6crletMxM4m6WcMSkJr/KwOlk+ntKlH2qSZE98mD8hi7uC278nMYKO3kqGW3cZVFEZz5RW35uccMThjRh0XbFc/55NwKkgsW48PCLn87unsKuaDwKS5wgbxfouqkJ9hr70LeN2uVGfU+MfZfXif8fj8PPvggTzzxBOXlyjkuLS2N22+/nbvvvrvFmiEq0W232+nSpQs+n74X9KJw5Pp0W5W8EjCrl7cQo87Vrnq6K3025Sj1uXH7/Eh1c7qTMmsf1Akvt1ssuFspp7vS7SNJUu+k25LAYjFFt05jtsbYCUV2H+X/quie5J5LYT1vgWlbqK2qDPDw9+s5YUAuKWqI4LaDFYHX1u4taXCMhvbbQ+U1/Lr5IJIEo7tmsXhHEav2RCe6C9Q2Od3SZB6ufhKbzQvzv4Atb8Gf5oUVb2/M38HGwjI2FpaxbMMOZngfUApVbfoBBp8X8VxaA9HWhAZDtlVP97gqpdr3h95jOdv+G47di2DXQug6vsWfNXfTgUBueGm1l837y+mb1/ybKI2J7o5qPjfJbcFRm5N70ZjOvLNgB+3SXQzvngdhRH6vnDT65aXxduEUzrb+Su6OL+ksTaKrTQ0bDxJIZw3ryMrdxawdejfj9r0HBzexw9KR+f6BrModwLiNn8PK/7Fr2DRWvXtP6Hyr1NZi6s350moP8xcuCNmmnGTOq7gTt+MK4DAb/J3Yax/C1MEdsJbugRGXwYAzYcTlcPQdsPC/tfnMoKQ3jvsz9D4h9Et2HgNT7gtv2DZdldc1Oo1U/oWj6wTlJkRpvuINX/Op8vygcxRvPDQYXr50x2FGqvnc5A3mvHF9+XFTGY5uN0K1C4ZdjNNuh6t/wOn3cMPigzislkBrr8n9c3nwW6VKeZ/cVPrlpUPeFLjiK8UOY/8Ufs6tSIfMpLplf5BS2gLV/Nn2FW2sqmOmgR7dGi67lXevHcuWZ7tD8U76WvYgIyEDHaVDzJj1EvLwf7Nwq1KU7eKUpXAYCrJGc/zem7nC+iPTbJ+QLlUiIZM86y56aZ854jKlg4gg3H333bz22ms88sgjAcfyvHnzuO+++6iurm5xGnXU4eV33303d911F++88w5ZWZGH/cQDrS66NU+3zVJbfMKAnm4jn/iNOle76ul2o7ZU8Dbm6VapE15utUh4W6lPd2Wwp1u96DBFtym6hSBb9XQf2gLVJfQuns/+OpuYtlXYH9S/vLC0hpfmbmPaCcpNi20HakX3mvyGq9I2tN8eVCvGZybZOWVwHot3FLE6vziq+RaUKPMdnnwQW7WXCtmJ1SLhKlyjeK97h6bB+fwy369RIofSXTZGlP+AZFe/c/5yU3TXwat6/WzWyCLEws1XlmVW55fQlhJyDyvthZ71nYXVauE86WdF4EQgun9cVxjyeOnOouhE94r3ofxS8ourAkXU6noQM5MdzLnzOCyS1KhX/dTB7XmioDerXKMYUr2Um61f0EHzWAaNecVR3bhgdGc1PeJm8Nbw6lcb8C7ew6zKboxtPwxp3woKX/0D46vWhs4Xv+IFn/gXft91mH+89wt/K3ue94K2cdmtHKqxUORRrjv+6vkjzrI+bE/pwy1n1KmvkDsAznyuacPFEosFBp2rtPua/WggJYjB59duEya8vLzGy4aCUs63qsUZO48lK8XBJ39WveME9Yt2JCMB1x9d20cboHt2Cj3bpbD1QAWnDwmqLt55TOhNgyOI4umus5+ldwS2MtWymOIadd5NeLpBcfz0HzQC5s2GpDZIF75Pef460n68nRPLvmTR1r8FPN2jymYBkDrqQpJn2HnXewrZR9/EnyZ2wfbxZcoNkkOblciH4+9p5FONx1tvvcWrr77KGWecEXhuyJAhdOzYkRtuuKHFojvqWNrnnnuOuXPn0qFDB/r27cuIESNC/iUSR7SQmoFyukUSA0adq+bp9mj3xXzusBVcpbRcSM1V+kPaQ6teKoXU1Pf7PfpUHFWp8nhrRbfdFN16jtkaYycUmV3BYlfW0MWvYPPX1HOAmrZV2K96urNTHQDMDBIywTnd+cVVHK6on0MKDe+3hyuV7dskOxjSSbngXbWnYY95c9ineroHOJR5rpG786FP6dHOgmfrbb9o+yEOlrvJSLLzf6f05xzrr7Uv5i+Lai6tge5rgrcGFr0M394OX9zI9LdmMuKBmZQufFspoNVCws13d1EVJVUeTrUvRZL9yB2GU5PamZfcJysbbfgGdi+BmrJmf47H52eWWivgpK5wrGUFnlVfNO+c6PeD31d/roWrYc4j7C2uooNWFTqMB9FutShtpRph6hClANc/y84E4Bzrr3SvXqO8mN4xZNuQegQ2J8O6Kl1L3l64i0dLFQ/zaGl9+CJpqz7E75e544Ml3Ff1MO0th0O2yVHrMGhmuXpCdwBenLOVSre+N+6bjSawC1eDrwa6TYL2Q2tfD+Pp/n3XYfwyDHCot1PDFYBrBv88YxB/GNWJy4/qFtncdaZjm6R6jpkij43fUyZikWSy3GrKQZ19qkHG/gmOnQ7XzoKuR5E2+hIqrBl0thzgh09eYc+hUgZYdpFashmsDjKGn8O3t0xizp3HcdPkvticSUruvRaFecydtakFglBUVES/fvVTB/r160dRUVGLx4va033WWWdFO0Tc0Nqi26PeZbbbJLN6eYQYda5aTneNXNsyzOML4+l2pMAfl4PFVu81u1WqrV4OIPsJFDyJMZVuH8mSKbpbY8zWGDuhsNqUfMIDG2D+M4BSxVZJflNIJNvKssy3q/fRNzeN3rmhnsDCUuUYP2FAHv9bvIuNBaWBi/G9Jco5qE2yncOVHtbtK2VCr9DoG2h4vy1WRXdmsp0B7TOwSIpnvbC0usGibE2hhcP3kBTvdb6lI6/UnMTlrh+UvsgFq0MuwL9dtY8R0ibuzlhI9/zOZGnhqAD7VioFKa32iObSGui+Jqz6EL6/I/BwhDefzb7jSJ9xj1IE6/92tejGf7j5rlKjG85y/Q4ekAacxfEFOXy4tIataaPpWbYEXpuifN6V30Hn0WFGDmXF7mJKq72cl/w7j+9/Asnhh3xgeSaMvKLhN1aXwEtHgyMNyXdOnbmCvPxtZM9wOjYiuptDz3ap9G+fzrJ9PZlpGckJ1mWkuQ+oYzYukCb3y2FA+3TW7SvllUNDuNSZTSfpIHK3ScB3QfO1wP51rFq5hCtKX2KUbRNbHKkola0U8jKS2Kl9OeD0IXn8qhbLmrmukDOHNVOs6UneYEVo71oAk25X/gXvR2E83VrkTTuHG6pRqpxHwMTe2UzsXX9NMwodM5PqpSDmF1dzWdFV3G21c5FNrcTdtmfzBkzLhWP/r/axPQnf8Ctg6X+4t/ox7g1elnufCEmZdKt7+KflwhVfw+5FSlqCYAwdOpTnnnuO//znPyHPP/fccwwdOrSBdzVM1KL73nvvjXaIuCG4enlrFFLzqyuLVZIM3afbKMXJwmHUudotdTzdsh+v10vd0CHJ7gBnatgxrJagPt2ghJhbdBTdAU+3suqaotsU3cKQN1gR3TWlyJKVzzkeggIvRbZttcfHpsIyBnXIaFbhqLmbD3LT+7/TtW0ys/96bMh331+mnGeGdc7g5w1OCktrWJNfSrJDWVfaJNsZ3yOLrPXvsm+DBL2m1hu/YU+3Rx3DQZLDSp+cVEoLd7BqdzEntLDStMY+9UZAR59SBMiX1Ys9e9uxtd1keu3/EX57Ds5RevR6vT76r3qUBxzfYCmWoVi1h28wE5J2YHWXwf71SjEog/DWbzv4ce0+Xu/wFU6bBYnQPsGx3G83FJTiW/ILA4HyrIGkFq3lHOu82nxmn1sJ923f/AvRcPuCJpC6qTdK6DyWKVm5fLh0N3eVnMXbmQdwVuxTvJxrP2+W6N5zWLmBdnLSOqQKP+Wyi1SpGu+8Z7ANv0wJWw7HgucDfYqlPaG1i6Ssnki+PVzp/4JMSRWuzQjbbYhzhnfkX/tKucV7E9PbLuHC9NU4UjIhr3F7tklx8O0tE1m+6zBzNx2EnFegYBbWwdfCDT1q5+tKA3z4Zz/KxVYlHNhyymPwRK0QSnHaGNcji0/V30WWZc4c1oFnf97CF7/nG0N0SxJc8olyPRPu2idMIbWyamVtSdE6vRi8IGKktM9w0SEriT1Bz7l9MuVyEiuG3c9FI4uhujSq/TT9mJvwrf8Qa0VQuoZkhdHXNvym3AHKPwF57LHHOPXUU/npp58YP15Ja1mwYAG7d+/mu+++a+Ld9YmZwli6dCnvvPMO77zzDsuWGT8MSw+CPd2t0TJM+zhLsOhuoJdea2LUNlzhMOpcLRYJq0WqzekGLH53vXQdSyM5/HaLBU+w6NaxbVhVsOh2pChzM1uGmS3DROGE++Gkh+HM/yL9cS6FWaNCXhbZtvd9tZYznpvP5a8vZm9xVZPbf7NSaZ+z81BlvfBuzdOdk+ZiWOdMAFbsPhwootajXSonJ63nQfsbjF39j7DjN7Tfan2Z26Qooet/TprJb65b8C97ozlfMyyapzu7RukXm9FJEaXvWdT8vDWfQKnyfb//4Rsulb/GIsn4B54LPSdTbMnkee9ZHEwfqGy/d3nEc4k1h8pr+Nd363Ftn4lzyX9hwXNY6lSojtV+u7e4ilP/Mw93/ioA7jt0AuvogVPycLR1de2G+ze0eOzgc67FYlFTGGTSvWroZloex/Vtx4RebVnk7s6kqicoOlGJSGHH3GZ9xn51v+2kVmt4O/lySuRkbIe3KpW1w7Bj106882tzli37VoS8bumjVGC+wPoLw63blSejEDNXT+zOG1eO5qe/ncLlf3kYx9XfwAXvKpE4TSBJEiO7ZnHbCX3oNPR4OOlfWFLbhmxTY88EYETJT1glmcNdTsTSc1Lod7JYeOXyUXTMVKLVFNGtCO25mw9yqLwGQ2B3NehswGqvF15e6VZumLj88S26bVYLn/ypfhepnu1SuPeMAdD9aOh/WnQfkpaL9fb1MD0f/rYT7tgG/7cTeh4X3bgG5ZhjjmHTpk2cffbZFBcXU1xczDnnnMPGjRuZNGlS0wPUIeoVec+ePUyaNIkxY8Zw6623cuuttzJ69GgmTpwY6GmWKLR2eLlPvZ0nSQR5us2c7pZg5LnarVKtpxtwEMbTbXM0+H6bNYynWycq3d7aPt2mp1vXMVtj7IQjvQOMvwGGXwJ5g2ifExpCGGxbWZZD1noj4/H5+Xa14jGct+UgZzw3n5LKhm++eX1+Zq6v9WBo79XQPN056U6GBkR3cSCfu0d2CoMdyns6VG9lxdb61wBNhZe3SbaD38eU4o8ByN4zq3lfNgz7SqqR8JNasQOA3gOVOjNv7syiPHeMsiYueol9JVXsXPQ1ALtyp2A5/3W47DOeH/k9i+T+bLCoRaQMlNf9v8W78Hi9/NX2UeA5qXhXyDZRrQmrPoZ3zobSvazaU4Lf76OfRbl58bu7Iy9rOdbBHFjf4o+pe54oqnSTTiU2vyrw0vKwWS28cOlI+uamsb+shv9uVyMfCtYoFbmbQCsAmONVi+R1Hca7PrWI3m//qbf9kh1F/PLqdGzeCoozB0L/M+rXeWjXm6J2Y3BKXjpToDwZYXg5KJFpx/XLUYthRU/d3/6ALwW/pFxP+JHIPPW+sMdimsuO3aZIA1mW6ZWTyuCOGfj8cr31wJCECS+vUkW306+mDDkaEOxxQJIj9CZNpzbJvHnVGJIdUQc212KxKjc9kjIhpW3c3sTQ6NChA//617/49NNP+fTTT3nwwQfp0KFD028MQ9Si+9prr8Xj8bB+/XqKioooKipi/fr1+P1+rr22kXCDOKS1RLd29zoQXm4Jzuk2q5e3BCPPta6n2oG3/nytDf/eVkudnG4de3VXuX0kSWZ4eWuM2RpjJzo9slNCHgdSevwyF7y0kFOe+TUgQI3Mom1FlFV7yUpxkJvu5GB5DUt3NixSFm0vojhIlH+7al/gvObx+TmkeqNz02s93St3lwQql/fMSaWbRfEmWiWZF/73OQfKQr1jTYWXZyY7YMssUqoVIdO9Zj0V1S2P0vH7ZfaX1tCeIqzeKrDY6N5rAOeO6IQsw1OVaq/YZW/w2FfLGIPixe006tTAGEM6Kd9xflUX5Yn8I+PplmWZN+Zv54vf86GyCO/Sd+j46//xhP1F+qtCGEAq3hnyvojXhMoi+HYabP0Zfv4XW/aX0VUqJIkaZJuLytSufOsfR7FNKYr0k2+48r4oPd2SJHG4wk07qVh5wpkROJ+ku+z87ZS+AMzaI0F2X0CGnfOb/Iz9ZTVY8JPpVvapEUOH86b3JNyyVck1PVCbu78mv4Sr31jEmdIcAO4vP4sfut3BerlryJjlNV6WdL0u9IMyIxfdsabub19U5WchgwDYnHMKUu7ABo/F2mhK5dg/c5giML74PbT3tyEJU0itwu0FZBw+tcNCQ17yOKDub9o1O5XOWckNbG3SFDNmzGDevHmBx88//zzDhg3j4osv5vDhw428MzxRi+45c+bwwgsv0Ldv38Bzffv25dlnn2Xu3OaF/sQLeonuhkKgA+HlYOjq5UYOzTSycFHuNkv4VWHtwENLPN12qwUZC7L2Hj093Z6glmF2RbCYotsU3aLSPTv0okyz7b7SahbvKGJDQRnXvrXUOBV9G+DHdYrIOHFALuN6KOGmGwoarvqstcs6a1gHkh1W8ourWLG7GICD5TXIMtgsElnJDoZ0ykSSlErlv25Wij71yE5BCurb26lqPS/M3hryGU17uh2w/K3A61lSOWvWrmzxdy+qdOP2+elpUb1zbbqD1c7/ndKPNJeN1w/0ozS5C1SX0H3jKwyXtgBgCQqTHKqK7u+KVK/G/nVKTmQrs2DbIf759Tre/OgTvE8OwvbNTZwt/8Q5VuVicIekhDXHTHT/9izUqN9z5f84vGcT/SXFiy7l9Oeta4/iqqP7YLniC/5qu4vXfGrufow83Tma6E7LDdl2RJc2AGw/WEFVJ7Xd0/ZfaYoDZdW05xAW2QtWB/379MHZpgNL/ep16/Y5gW3v/2Yd7d07yZLKqcLJV+V9+ePnu3nXF9p7+vfdxSy3DGKRX61sbHXWVmk2AHV/ez/wj+pL+Nh5Dp0vfjrsNtrjgGNHrVN0xtAOWCRYvquYnYcqMDKyJSglT/0eVW4fTjzK7w9x7Zk1asqkqNxxxx2Ulipr4erVq5k2bRpTp05l+/btTJs2rcXjRa2GOnfujMdT/y60z+eL2P0uKnqFHDYVXm7Fq1SmBtPT3UKMPFebWvRIO4nYJS9Wa13R3fDvrb1fCynDr19Ot9Kn2wwvb40xW2PsRKdbA57u7UG9qFftKeEfX4T2wjUSsiwHWnqdODA30JO4IdHt8fn5Ya2y/ZnDOzKlvyJ4bnr/d95esCOQz90uzYml+jCp3/yJz5Mf5k37o6RX7aZb22SlWnmQ6B5q2Rbo5arRlKc7TzoMG78HoMSm3CgoWNe0N7MuBWoRtcEutU2Q2o+9XZqTO07qi4yFxyqV/MabrF9il3zQphtkdQ+M0TkriTbJdvb42lCd3k05z+6Yh65Ul8Dcf8OBjYGnXp+3Awk/D9hfx+atYKu/PS94T2dJznm87D2V22uUqEKpeEfIUE2uCX4fFK6DDd/Byg+Uf7+/B4uU4nKkdwTZx5j8N+lnUUPXcwfRJzeNu6b2J73zIA51PI5NfjWX+fBOcFfW+Qx/o/VE6nu6PeSgepBSQ0V3ZrKD3jnKDbFNScOUJ7fPbTKKa39ZDV3UCAwyuyBZbZw6pD0L/Gpxpx2KcN9QUMri7UWMsyoe+4qckYG2myO7tgkZc+mOYvYUV/OU9zzlxnbeoIYLsh0B6v/2EgecXRl7/fMkZ+aG3aYhT3dOuivQieDLFXt1nHX0BIvu4JzuFIIik+I4vNy8Logt27dvZ8AAZZ349NNPOf3003nooYd4/vnn+f7771s8XtQrxOOPP87NN9/M0qVLA88tXbqUW2+9lX//+9/RDi8U2gIV6528oYNICy+3y0H9UM2c7hZhZGGo9eqWrYo324EXa927mE3kdAP4tYrlOnm6fX4Zt9df2zLMYXq69RyzNcZOdJKdoW2hNNtuO6jkLndqo6yz36zaS7VHv7SNaFiTX8q+kmqSHVaO6plN/7x0ADYWhPfUfrViLwfKamib4mBCz2xuOK4n2akO8ouruOfLtbzyqyKmc9JdStXo1R8zzLeaY60r+Uv6HD7+01Gk2GQoqQ13HiJtY/2+UsprateehkW3ch7rVLYSZB+0H8rBzkresBRBLrUmuvurPbo10Q1wydiuDOyQzvvVR7HJ3xGLpN4w73FcyBiSJNVGCCSNVJ7c9kuL59IiVn4APz+A/OoU2D6XHQcrmLWhkPOscxls2UGpnMQf3Pewe+TfGPGnV3nReRWr/D3wWxxInlDB2+Sa8NHl8MJ4+OAi+PyPyr8vbwBPBRXZQ+A8pYjd8dWzONmyRHlPnR7H/dqnc4gMKqyZgAzznoR/94H1Xyt9vV8+Bh7qCG+dDq+eAA93hhcmwHd3QtXhOnOUKK/xkiupojutfb0pj+qmiN9fqlUv9YH1cH+W8l0acHwcKK2hk6S24GrTDYDTBncIiO6KTXNYl1/MOwuUSIEzM5XCaG0HHsc9pw3ggbMGcf6o0NDxokoPs9YXstA/gHnHfawUPTMQdX/73AwXb149hi5tkxvcpiHRDQQKqn2xIt/QNS3Ci24vKZIWDZqsWxcXI2BeF8QWh8NBZaWyrv7000+ceKKSlpSVlRXwgLeEqEX3lVdeyYoVKxg7dixOpxOn08nYsWNZvnw5V199NVlZWYF/8Y4WihPrcOqGDhpNdDvkoJw5A3i6RQpvCZ6r0eZpD4hm5SQSTnRbGqlWb1O3lbVibDrldGshtq46LcPM6uVm9XJRqWdbNUpEy12eOrg9uelOarx+lu5oeV5Xa6B5mCf0ysZlt9KvveLp3nqgghpv6Frg88s8P1sJr752Ug8cNgv98tKZ97fjuXC0IjZmrFFC1XPTnIE2Sv5kJaf3jLb5tEtzKoLb7wV1zepmKSRVLmelGqIODe+3Wi55hl+1Z5vutO2rhBB3rFxHSVXLInUK6vTopm2t6LZaJB44axB+LDzh/UPtm8JU4D1xoOIV/LJMFXlbf27RPFrKxg1rAJBqSuHdc1ny7Wt0oYC/O5XCcl9lXMp1p4zlX2cNwmqR6J2TigcbxWm9qdsRzlLeSOGrwztgwzcA5Cf3Y0vaaCo7H8NCaSgzfSO5rfIqvB1HU9lzKnbJR1+LWhQvd2DIMP3UCIodFlWUzn0cygvhm2kw72koWKW099o+F/YsVsLWC9fA4pfgy5tC9gctFSq3gfBygJFdlWvJX/fKMOjc2hfWfRk2v7vK7aOsxksXSfV0q6J7UMd0knuMoVJ2kuIt5rbnP+CjpbsBmcE+JYJF6jqBqyd257JxXbFa6wg1SaLao1zzpXYfpRRjNBB1j7Me7dIC4fkNbaM9Die6TxqYi9NmYduBikBbNyMiW8N7utO0dmFx7OUGU3THmokTJzJt2jQeeOABFi9ezKmnKjU/Nm3aRKdOLe9WEHU5u6effjraIeKG1vB0B/+t9SK0a2HDNhf1SmweAUQ66I3sjbWpnm6/RfN0ewLPaTTH0+2z2MCHbi3DtMqgAU+3XbmTbmTbhsP0dJto1LNtTQlAbWus7BQm9W7HJ8v28OvmA0zsbZxcTo2D5YrnuHMb5XjMS3cx0HWQa3wfUfLD7+Scendg2+/X7GPbgQoykuxcOq5L4HmX3cpFY7rwwZLdgXSmnHQnqFWyLYPOhsUvYy1cpXg1i9TWSW17KR01Dm9niGU7S3ccDoSnVtWJDJAkCb9fDuR0p3hV0Z3Sjsze42EGDJJ28O+Z67jr9CHN3s/3lVQh4ae7Wy2SlRPaJ3ZElzZcN6k7b/0mUdT1ZLKq86Hn8fXGOb5vLlaLxKdF3bknyYp0aAsU79alaNaew5Vs3baZvhIclDPI9pVw7rZ7ONGRRIa/ErL7cumf/hVyc71PbhqLthexw9kHiRUh40mbf2z4w1Z+AMC6pBFMLfqr8tyBoNeL4MOlu+k65F76bPmtNs+6nuhWIihWutszUApqH1axH2Y/BMB/LRexpyaZCtlJ1/6juXWwG+tXN8GGb5DwB97iU6+hOtpLQQZS6/dnH6WGea/eU0L1ta+w/+gnqPhiGv33fkbBd4+Qce2XJDlqBbJWyE8r8KeJbkmSeO3qCZS9MorkgvmMk9ax0deZY7PLcJQfAKsDOo6stWWd/e7Uwe2Zi3IDp4sBC1U15/zQlKdbcyQBpLnsnDAgl29W7eOLFfkM7pQR6ynHhEA6HaGiu4MWXh7H+dxgXhfEmueee44bbriBTz75hBdeeIGOHZWIj++//56TTw7TwaEJohbdV1xxRbRDxA16ie6GvLGap9sma5XLj3yPbqj//Y3sgQueq9HmaQ+IbjWnO1x4eSNzDuSEo294udYDM9Wi5XSbolvPMVtj7ESnni2rigHYroaXd89OIclhVUX3Qaa38vxAER1dspLJSLaHfT2kBRcgLXmVL5iO3epBXvIbnHg7ss3Jp8vz+edXimfvqgndSHOp4+38DT77I4On/JPs1LSAiM9Nc8FWNb+3+9Gw5jOoPAj7Vtbmc2d1VyJeDm9nmLQlUDF93uaD3PDe7yHzlCSJsmpv4CZyklutrp7SDrJ64LZn4PSUsHDBXJ5LSebmyb0JhyzLPPbDRtqmOLhmYnd+3nCAPtIekn1lSnHH9kPqveeuqf258+R+2K1TG7RzRrKdcT2ymL9FZn/aQHJLVykh5iMub/A9kSDLMtM/W83NchFI8E/PZUxwbOZC6QcyqERuPxTp4o/rRbP1yVU8d6t83ciss9tKRVuVHPRuE0Nf8PvxLHsXO/BiyXgcNgvtM1zsPFRJmtPGeaM68cb8HTw1czOXjuvCK54/8objMSw5AyAp1FvaKyeV3jmprDvUEdRdp3zkn0ld9gIA6/2d+Xf1qXTKSmXP4Ur8a2GXvTtPHfM3pF8eRAq6GaztA+0tJcqN4rT6ortr22Tapjg4VOFm0mO/cKCshi7SJH5xfE7e/l959j//5M/jsrHtXQYWKweHPQBAD9tBpZqYKrpBOcdmDZwMBfP5W+4SxtvsjEkvgi1Ax1FKP2jNlnXWhGP75XL9pLFUeXy0TT3yEYZ1iUZ0a9dCdcPIzxzWkW9W7eObVXu5a2p/pXOOwQhbSM3jqw0vj+PK5SBWpKkIdOnShW+++abe80899VRE4xlLZQhOa3u6Nc+Dza+KHYOKbiMf9EYWhlp4uU9Sw8slb31PdyNz1rb1Sa0lutWLJ7OQmq5jtsbYiU5dW1aVFVHt8bHnsHLh1qNdKhNVz+26faX12mLpzber9nH6c/O4/eMVDW6j5UhnpjgUMfz937CjHKMSfji4if/O3spfP15JWY2XkV3bcM3E7sqbZRl+uBtKdmGZ+XeO710rtHLTXQFPN5ldofMY5e/di2o93Vk9AkLvAutsVu06xA9rC7jmrSXUeGu9Z6DYukida7LDirVKLbyWkg2ShKPbWABGWTby5E+bKCwN36pt2c7DvDB7Kw9+u563ftvB+n2lHGVTi5F1HgPW+jcnJEkK3NxsjBMHKOJvrk9pucSWn5p8T0tZu7eUXzcfVArJAXvltvxf9eVM91xDwcBrkK78Nmy4de9cxXP3VXm/eoJcAtj0Q8hzHp+fTz59D3vZbkrlJGZLY3jx0hF8f+sk/nnGQD744zimn9KfLlnJHCyv4b+/bGWOfyjvDnsfLv203udbLRL/uWg486XhHJLTWN7+Qv7tv5SZvpH4sDCv1+1cOr473986iRcvHYnNIvHFir08WTUVuk1ColbY+VTParsGCqmB8psd00dJazhQVoNFgi49B7I641gAbi5/BttP/4B1X8CaT7Gt/wKATrKa258Z2vqLHsr7kovWcfL+V8na8pnyfLcJ9T637uOJvbM5YUD9ORqBWHi664ruo/tkk+6yUVhaw+LtTfdHPxLIUv3w8ooaL6map9therpNWobf72fTpk3MmzePuXPnhvxrKTHslm7S2qJbWw8DhdQaye9tTUQ66I0sDLWLQZ8aXm6nZaLbHqherolufXK6qzyKmE+RapSQQLOQmq5jtsbYiU5dW3rKD7HzUCWyDGkuG9mpDiRJYlDHdNbklzJzXSEXj+3CrkOVyMh0bZvSwMgt57etB/l0WT53ntyX3HQXHp+fx35Qqiv/uvkgNV4fTlv9wkCHKxSBnZXsgDn3g+xjb/YEdu8/xFjLBgq2/M4zP+UAcOvk3tx8fK/a9WX7HNir9qQuzeei3ov5CEWs5CX7FM82QGYXRdBu/A52L669sZfVHYZciPzzv+hceYDj3L/yx3fUE1aY/fZwcLuwCjXGOUURVnSdAJt/5ITkzbxRdgrLdh5m6uD6BbZ+Wr8/8Pd9X68D4IzMHVCmjhEFJwzI5d6v1vLO4QGc7wA2fAuHtkLbnlGNG8yuokpADuQz2zI6QLFEUb+LyTt/VIPv66OK7uUlaSyr+RPwSOA1SaI2z7m6lNINv7Dw+3c4o/pnkGB52nF8fsUUerZTvH9XHNUt8N7HzhvCJa8uwu1ThHCb7sMgvb7dAfq3T+fa049l5OcvwnYJacdO3pVv5b3z+3LdkH6B7U4cmMdD5wzmzk9W8ezsHQy4+EUkRx/wKDezrJ5yPEAbLa8/jKcb4L4zB3L60A5kJNvp3jaFNikOOPg45W9fyK5iDzvkXIblWOhwaCEZu2eRwsVkyEqKCG3qiO6OI+G0p2D/emUe3mrl5sWY60M2E2291UN0O21WThnUng+X7uarlfmM79k2llOOCbIlNLzc55ep8fpJtWqeblN0mzSfhQsXcvHFF7Nz5856x4MkSfh8LbuuNj3dMaT1c7o1T7cWXn7kK5eDWAe9kYVhbcsvrZCap354eSNz1rat9XTrk9NdGcjpNluGtcaYrTF2olPXlnZ3CdsOKKHlPbJTkPw+mHkvj/MMj9te5LUvZjDx0Z85+vFfmPLknEBv60jw+2VmrNnHJ8v28P6iXVzx+mI+Xb6HF+co/a4/WrqbnYeUaqo1Xj+r95SEHUcTsu29e2CVkr9bMu5ONvqVXORffp2N2+fnuL7t+MuU3qE39H59Qvm/6mkcsvMtulgO4qKGjpIquF0ZkJQJnVRP954lihAFpSe2Ixlp3J8A+LPtK1IcEpeP7xpoRaYhSVIgFD4z2V5fdHebBMAw/1ok/CzbGb5w3az1hXWekRnoVYqS0fWosO9pLh0yk+iXl8Yqfw8Kco9Rbi7Muj+isbbsL6eipn7U0b6SatKpwKm2XvzLOZM4dUh77j19YL1tg8lKcXDjcT3JSXNSJdepug+wd4VSNO2ZIaR/cTkn1szEIfk4mD2aY//4dEBw12Vcj7ZMP6VWMPfKaTws9+IxXfjHaQOxSIpDYFDnbMYM7ltvuz+M6swfj+4BwAPfbkQKEkkOTwlJVJMkq1XYw3i6AdJddo7rl8OILm0UwQ2Q3ZvUacv4aNT/uMHzF/64V2kHl3dwAT0ltc1VUpay39Zl1NUw9XE48zk491U483lIzQnZRLT1Vg/RDXDmMKVg3HerC3DXiVoxAnWrl2s1JFJIjPBy0fZTo/OnP/2JUaNGsWbNGoqKijh8+HDgX1FRy6M9TNEdQ7SiE62V010/vNwYeUVmTndscNiU+XjVk4izpZ5urfp5K+V0J2EWUmuNMVtj7ESnri1T5Up+XqtUb+7RLlXJ6Z3/NP0PzeR821z+avs4EHru8cnc/tGKFrcSK632UFBSzfXvLOVP7y7nrx+v5K7PV+PxKev8N6v2UVHj5ZmfNgOQohaLWtRAmKcmurtveUvpL93nFLoPnURRiuKdzanajstu4f4zB4V+3z3LlErTFhtc+hk40rAe3MBcxy0sS72NHm7l88lUC651GK5sW7YPDqpFy7IUUcXo65AdqfS17GHZRTbuP3MQwzpnhsxTkqRar3yKA8rriO72Q8GRSrKvjH7S7rCie9ehSjbvL8dqkbhsnOLJPKZtGY6q+gWxIuX4fooIeyPpCkBSwpf3tKyV2bKdRZzw1Bxu+3BFvdcKS6trq3YntWF8n448f/EIOmQ2fTP9jpP6seiuyZw/MrS4m5SUpbRf++pmqDrMITmd97yTWXvSh2Tf9FPYcPVgrpnYnVsm9+by8V0DVcobQpIkrpnYnTeuGsMpg/J45NzBDa5Jf5nShw4ZLvaWVAf2bwCXt6y2yrg9JSKv5D2nDeCe0waw2dqDQjkTp7+Sh+yvKS/m9G/xeBqirbexyOkOLqSmMbZHW3LSnJRUefh5w/56rx9p/HVyuivVG1ypkhZeHt+i28zpji2bN2/moYceon///mRmZpKRkRHyr6VErDK6dOnCTTfdxI8//ojXq8/FvGhodwX1bBkWPrw8tFXTkUakg97ILcM0T7eW023HG3hOo7F9LVC9XGfRrVUvd9UR3WbLMLNlmKjUtaXNIrNilRJu3T07BQrUKs2q8DwpeSOvXDqcX/56LO3SnGw9UMGTMzc1+/Oe/HEjQ+77kXEPz+Kn9ftx2CwM75JJssPKVRO6kZls50BZDTe9v5z9ZTV0zEziFrWgWLjcSlmWOay24EouU/OsB56Ny27lxgvOAGCoI59/nz+UznUrL897Uvn/4D9A3iA47i4lD1KykOItRlr2pvrd1TBdR3JQoS5ZqTidoYq/pEykwecB4NqhtNoa2DEz5OMsFkvgBkE7lwzuMuWFFLUivNUGXcYDMM6yjrV7S+rd0PhJ9XKP7taGu0/tzx0n9eXhkeo4dQpiRcpxquj+YGca/qEXKk/OfrhFY/ywthBZhl0bl1Hz9R1weGfgtX0l1Y32p24KSZKYOiS0bZUlW735sV3JPXzYexEzuv+NgeObV3VXkiSmndCn/o2ZRjimTzteuHRkoKp5OJIcVu4+Vakm7w7SdVZJ5gLrbOVBWm69VITmYLFIXD2xOzdP7sss33AABll2KC8e+38tHq92XLHW2+Y4P1rSMkzDapE4Z4TSKun9xbtiMtdYUje8XHMKtLFq1cvjW3SLdnPI6IwdO5YtW7bEbLyIV4133nkHp9PJjTfeSHZ2NhdccAHvvfcexcXFMZucaLR6ITX186w+VeyYhdRajJG9sVpOtzdQSM2DtQWebi28PJDT7dPX0x0Q3Q7T063nmK0xdqJTz7ZAT5QQ1R7tUmC/kjPM8MvAmYGlpoQTMvfSPTuFh84eDMA7C3bi8TUv/DI4H7lHuxQ+/dNRfH7DBNbdfzL3nj6QUwYpua2/bFS8wLed0CfQpmzZzsN463xOabU3EAllr1YLk6UqnmN7e0XsZPv2c1qfOheg+9ervZslmPgX5bnxN8Bde2D8Tcrj3QuV/2ueboAL3oMrv4MrvoE/z1eEcuALHaf8f9svyttS6hT7kqRAj+6ODiWEH4s9NAxYFfVHOzbi8cmsyQ8NqZ+1QRHdU/rnKjcWju5Kh41vKy92n0QsGN45k3SXjZIqDw+Vn45PlmDLzNobMM1gwdZDHGNZyce2e3EuexnmPBZ4raCkilytgFgDucxNUVdElSXXer5L5GS+9o1n2gl9Iho71kwdnMeILpkhz0nAH6yzlQdh2oW1hHNGdORneUTg8b5uZyvV9iNExPW2qXNwJOHloKQSAPy6+QC71FQXo1A3vFy7PsmwqNcnzoZvBsUDIu6nRubmm2/m9ttv580332TZsmWsWrUq5F9LiVh0H3PMMTzxxBNs3ryZ+fPnM2zYMJ599lny8vI4/vjjefrpp9m2bVukwwtJa4ju4JNqbU63Wb08UowsDOuKbqWQWmjBpOYUUvNq9RJ1Cy9XxnXKZnh5a4zZGmMnOvVtC5d1OcTRfdpxdJ92UKi02CJvCPRQL+S3KqJycr8c0lw2qjw+NhaUNevz9hxWLlxn/GUSs6YdU68H7hlDOwb+7p2TytnDO9IvL50+rsP82fcum7dsDtm+OKgauEUreqaFaye1gTTVI3pgg/r/jTDvKfjuDuVx/9OgXZ183Lo9rINFtzNVqfbcfVKth1qj+9GApHxW6b6w+63m6c6zltfONXg7VXRP5He+cPyd6vkvBF5ye/0s2aGI1WP7qnm4C1+AwtXKd61TECtSbFaL8tsDr66Fb/3jAJB/bV7rmJJKD959q3nN/jhpWvuiLTNBDeEtKK0mV1KjFiLwdEP9/XZZeVbg7098x3DGyB4M79ImorFjjSRJXDmhO2rmOQCyJNWGATcR+t4U7TOS8Hc9hn1yFgVyG4on3hPVeCKut3qJ7i5tk5nUOxtZhv8tMZa3u77oVq5P0jXRHefh5SKld4rAueeey/r167n66qsZPXo0w4YNY/jw4YH/t5SY/BoDBw5k+vTpLFy4kB07dnDRRRcxa9YsBg0axKBBg/j2229j8TGGp/X7dCv/DxRSM2j1ciMf9EbO6dZysgOe7hbmdAdahmmHuc7h5Y7AfmgWUtNzzNYYO9EJ5+meUPItb186iHSbXJu7nDugnifXYpECecu/7y5mz+FK/v3DRkqqwhcyLK32UFqtHJtdspLD/o5jumfRUc3tveOkvlgtEtbqw7xte4gbbV/BLw+GbK+FlrdNskKl1oIrqDCUltu6f51Ssfndc+Gn+2DHr8rzE6fVn2iX8aE3duu2XmqI5CzoMEz5e9vset/PLxPwdOdYStW51hHu7YdBah522cMwyzbGbnkKKhWBunZvCW6vnzbJdnq2S4GSPbVh3yc+WH+sKNDyugFek89U/lj3eW0BOe07+WVmrivkiR83Uq7mlC7cfoiTLYuxSX4W+AZQIbugvBAKVuH3yxSW1EQVXg7199s5+bDF34Ea2c6uHhfy8DmDIxpXL04emIcUlDK1W+qIX1YfR+npBjhtVE9OqHmME2oep21Oh6bf0AgirreRiu7Gcro1LhmrHP8fL91tqIJqgcg+1JxuraWp2afbJAK2b99e79+2bdsC/28pMW8ZlpeXx3XXXcd1111HZWUlP/zwA06nMQp86U2rVy9XVbfV9HRHjJGFoSaaPeph2mLRreWEo3P1co8PO16sqHmWpqdb1zFbY+xEp55tXRlQVQQr3lMqYfu9SphiRudaD/DuxVBTBs40RnZKY/7m/fy+6zBzNu7np/X7qfb4+PtpA+p9Vr5agK1Nsp1kh03xfGoXTlWHwVONFXj3gs7sKLdz3MA88Hng4yvJ8+YD0LFgFgeKy2iXqRSeOlyhnBM6J9dAjXpBnBzU3ienP2ydBYXrFK9wyW6lUnTvE5Qc6I61YbkB7C7lu29VcrNDPN1N0eNY2Pu7Iro7hbbZ2n6wkqocZb5tJU10twt9v9UG1//CltUL8P5wL/0suyla+C5Zx98SKKw2oksb5Xf7/V3wVELnsTDskubPsRmcNqQD6/aWMrp7Fou3d+fnRcM43roCZj8C574CKEXdrn17CZsKFa+9zy9z58n9WLD1EFMtSoTEbOcxlNYkc5J1KWyeSVF6f9w+P7n2YuWDIgwvD3dD4yLP37loaCZ3n3dKvfPHkcZhs+CyW6lQHxfJqfzdew73Z36DrW/z8s4b4+RBeTz1U1vsVgttU6O7DhVxvdXL0w0wuX8OOWlO9pfV8OO6Ak4bEt1NjVjRUHh5WoIUUhNxPzUyXbs28+ZyM9F1BU5OTubss89mypQpen6MYThSLcOsgZZhpuhuKUYWhuFyuu0t8nRr4eU69+l2+2orl4MpunUeszXGTnTq2baTWv16wXO1Obw5A5QQ6Kzu0KabclPr/Qvgk2u4efGx/M/xIAs2H2DOJiUP+/s1BWEvYjXR3alNMrx1BjzUAT69Dt48DR7tBk/2gyf70f2tURz39UQ4sAkWvwLb5yDbUyiV0kings8+eS8wphau3cWpypmkrNA867whyv+Xvg5zH1f+PuEBpVXSqKsaNkxwiHlm54a3q4sWDbBpBpJWqE1lVX4xBaXKOSzDX6w8WVd0A6R3oOdR57CgjVIIrmLBa8h+P7/vUt4zoqsaNr32C+X/I68MDVGPAQ6bhb+fNoCTBuZx6biuPOH9g/LC6o9g30pkWebuL1azqbA8cNNzhvq7L9+Sz3BJKciT3Pc4fvEPA6By3fcUlCjfv6NN+S6x8nSP75nNWzefzrQLTw10wzAaLnvtfumX4X3fZKpuXqvcqImSZIeNGbcezfe3TsJqiW5fEHG91VN0260WLhitrAHvLzJOiHn9QmpKpElyoGVYfOd010WE/dTovPPOO0yYMIEOHTqwc+dOAJ5++mm+/PLLFo9lzFVYULRQnNaqXq6FlwcKqZnVy1uMkauXa+Hlmqfbjrden+7Gq5erol33lmHeWtFtsYHNUW9uRgvdD4dZvdxEo55t2w9S8oMP71C8mqCElmuMvlb5/875sOYTrL4axlo2MKJibqAlUn5xFWvyS+t9lpbP3S1dgu1zwFuliDgt1NtiU/5JFqWy95c3wpxHAZBO+hfufmcBkLXjW5bsUEKui1RPd0e7KrrritiBZ0O/05QbBZ5KJXx78PlNG6bXCYAE6Z3C9ztuiM5jFQ9TdTGWDd+EvLTlQCXbDlRgtUi0s9SpXF4HSZI47vwbqZbtdPbsYN6cH1m+S/F0D++SCfs3wIH1SiG2vlObP78I6J6dQlavUXzpU3uA/3QfP63fz6+bD+KwWpg1tYyZzjuZXvJP1n75JFkHl2CXfPjSO3PpycewJ1vJU3cVLKf9Byfymv1xOqLm30couuvut2cO78SADsYWGSHed0nCbpVIdcYuCDPFacNpsza9YROIuN42dQ6OpHp5MBeO6YJFgt+2HmLbgfJopxsTZEvtbx3s6U6WEyO8HMRzeBiZF154gWnTpjF16lSKi4vx+ZT9KTMzk6effrrF4xl/1RCIIx9ebowwfpHuCBt5cbIHwsuDcrptzb+hUT+8XL/q5UlSaBG1unMzmm3DYXq6TTTq2dbmhKPvVB4cVltw5QSJ7qNuhltXwbF3KX+PuAKAv9g+xYIfh3osz1i7r95n5RcrF4P9U1RBbk9WKoUf/3e4bS3cc0j5d+tK5bU9i6G6GNr1h+GXkT1WaV91onUpr8/ZCNTmSOdZGwjXtjnggnfh7Jehzylw1n9rQ9obI6cfXPYZXPS/prcNxu6Ciz+EY/6GlFRXrEu47BaevmAY6b7i8PMNolunjmzLOQGA/XNeZl9JNVaLxNBOmUrvbFA88kmZLZtjBJw9vCP/9p6v3Bjd+jMLv1QKvF0zqTtdVz9Lb2kPJ1iXM2jF/TxkfxUAa49jaJvm4r9/Po0t1p5YJJm2ZRuYbP2dLDm66uUirgkhc5Qk2iQ7DDlv0W0biae7sZxugI6ZSYHihf8zSPswP6Gebq3mTJImuuM8vBzEu/YyMs8++yyvvPIKd999N9agQsajRo1i9ermd67QMEV3DGn9QmpayzAtvNwYnm6zkFps0MLD3YGc7haGlweql6vv8emT013l9pGMKbpba8zWGDvRCWvbMddD+6G1T+YODH1Tm65w7N+U4l0nPkClJY3elnxOsyzg1ilKT+1wIeZ71PDyng5VcGV2gZP+BUffARmdajfM7ALH3V37+MQHAj2svUntyJAqcWz6mt1FlRRpfa8bKkymfCkYegFc/EH979IYPY+H9kOav71Gt4lw3F1IdVp4XZ0yny8v6cLpQztAxQF1vg2LboBeJ98AwMnyPFKool9eGilOW21o+cCzWj6/CDhhQC77re15zqMUVbu1+iUGp5Zy01ALFK7GL1l50XsaAB20yuRq26p0l50tEx7ncc8feNzzh6BRJUjNIRJEXBNC5yiRleI4YnNpDNFtG0khtaY83QCXjFVqO3yybA/VHn1S2FpC3ZzuCjW83OVXo36caUdiWq2KkSM4RWP79u1hq5Q7nU4qKirCvKNxYqYytmzZwg8//EBVlXIB0ZyDNd44Yn26TU93xBhZGDrqeLrteLG1IHTf3mrh5T5cqPtgUIqDkW0bDlN0m2iEta3VBqf/RwnztjpDPd11cWWwudeVANyS/COXj++Kw2ph24EKttYJw9Q83R0tqihL70iDjP0TjLwKJvwFeqm1UixWbGOvA2C67X3+9+vaQMuwLLkBT/cRROoWKro7enbQd7vaU1sT3U2ITkePiVSldSNFquFU60JGdGkDRdtaLbRcI81l5/h+OTznO4vl/l6kS5W8nvEaKZu/AsDXZRKP+y/hHW9QXZugXtEjRk/ged9ZPO87i899E5QnU9qB1U4kiLgmhJ4noEOmMZwHdRHftrHN6dY4tm8OHTJcHK70MGNNQTTTjQlaTrf21arcPqz4sMvqNUoCiG7Rrr2MTPfu3VmxYkW952fMmEH//v1bPF7UovvQoUNMmTKFPn36MHXqVPbtU8LnrrnmGm6//fZohxeK1hbdciCnO7RV05FGpJOTkRcnTWDXyMpJxCm1rHq5VjjGK+tbSK3SY4aXt+aYrTF2otOgbTsMgyu+gUs/bTJ8ecBpN+OXrPTybiatbDvHd5IZLm1mqdpTWkPzdOf61XzeYO92Xaw2OP1pOOGftVeVABNuoSqlM3nSYXKWP83eYuWckC4XK68bSXR3GhX6GGDj98oJrULrKd5Emy9JImnsVQBcaP2FkwflwY75ymudRrVKaLnGGUM74MPKXzw3UoWLdoeWwJzHALAPPourJ3Tj05wbqRp0EUy6HdJr87Vz0lwMVdvL/dNzOftyj4UJt0Y8FxHXhOA59miXxt2ntvxCtjUQ3bZ6iW6rReKC0Yq32wgF1QKiW31c6faRohVRAzO83KRFTJs2jRtvvJEPP/wQWZZZvHgx//rXv5g+fTp33nlni8eLWnTfdttt2Gw2du3aRXJy7QX3BRdcwIwZM6IdXiiOVPVyi8+sXh4pRl6c7LbQ8HJ7C1uGBfp8B/p06xVe7q0NL3eYolvvMVtj7ESnUdt2mwB1QqTDYU/PxaJ5o5e8yiPF0/jcefglSKsAAQAASURBVC9V638IbFPp9gaKnmV49itPZrSgKnjgw5JwnvEEAJfwPXt3K3nnqV5V4KcaSHQ7Qm8OS5IFincqPcObGV4OwLCLkS02Rli2MCFtv1LEDpS2Zq3Icf1yyEpxsJtcDky6X3nSV6NERPQ7jbtPHcAXtxxP0nkvwuR76r3/hP6KV7+YNLaf+BocdVPEcxFxTQieY4fMJHq2M6YoEt22eolugAtGd8ZqkVi8o4jNhWWRTjcmaOHlEoAsU+H2kop6jWx1BAq9xjOiXXsZmWuvvZZHH32Uv//971RWVnLxxRfz4osv8swzz3DhhRe2eLyoRfePP/7Io48+SqdOoXfne/fuHSitnii0dk63Ty2kZjF4eLnRcqWDMXJOt12dj1v1VDvwtjCnWw1Pl1uhkJoZXt5qY7bG2IlOzGw79ALl/4tfIrNGiQI7ZtdzSi9uatuFpblsOCqUnttkNBJe3giWviexM2UIdsnHuVal8nmyWxXdRvJ017Vtuz7KH/Ofqb0xmNyEpxsgNQep7ynK38vfPmKi22W38tEfx/PFDRPocvz1SlV4gK4TmnWzY3L/3MDfeenR3TgXcU0Q5TwRj7ZtKqe7qUJqGnkZLib3U24evXeEvd1+VdZYJMDnocrtI1XSKpfHf2g5mDndsaSqqoqzzz6bzZs3U15ezsKFC5k2bVo9zdtcolYZFRUVIR5ujaKiIpxOY4jA1qK1W4ZpNyElWV0YpejbYsQCs2VYbNA81TVBOd3J/tDCDY3ta9Z6nm59RHeV20dXSc3lcmWGnZvRbmiEw2wZZqIRM9v2nRrSF7ZSdtLDtwP3ig8B2KPlc2cmQYkmuiM7mQNUDboYgPOtswEZR80h5QUDie56ts1Ti7itUmxCz+OVaufNQa0Sz/K3oXiXcg7sPDZGM20+vXJSlTBxSVIqwR/3dzjtqWa9t19eGicMyGV8j7Z0yap/LdUSRFwTRDlPxKNto20ZFswl47oC8NnyPYGK4UeCgKdbAnxuKt0+UkmcyuUgzo0sETjzzDN5+22l5ojb7eaMM87gySef5KyzzuKFF15o8XhRrxqTJk0KTAiUH9jv9/PYY49x3HHHRTt8WPLz87n00ktp27YtSUlJDB48mKVLlwZel2WZe+65h/bt25OUlMSUKVPYvHlzyBhFRUVccsklpKenk5mZyTXXXEN5eXR9Bo9UeHlAdBvkJCDSHWEjL05aKHmNX/N0e8isyQ/ZptHw8nqebp1yut1ezrKqXibN84SxbRsO09NtohEz29qTYIhSmVoe80des56n/P3zv8DvD+Rzd8pMglL12G6skFoTdD/mYkXYWwoYKW3CWqXlSBtHdNezbXC/87T2cPZLzR+s5/GKvTzqzcj2Q4+8N8uVAcfcAdm9m7W5JEm8cvko/nf9uHrpQy1FxDVBlPNEPNo2VuHlAJN6ZdM5K4nSai/frNobyXRjQkhOt89NpdtLiqSGlx/ptaGVEOWYEoHly5czaZKSTvbJJ5+Qm5vLzp07efvtt/nPf/7T4vGiVmmPPfYYL7/8Mqeccgput5s777yTQYMGMXfuXB599NFoh6/H4cOHmTBhAna7ne+//55169bxxBNP0KZNm5A5/ec//+HFF19k0aJFpKSkcNJJJ1FdXR3Y5pJLLmHt2rXMnDmTb775hrlz53L99ddHNbfWEN3Bdya18HJJVsWUQTzdIp2cjLw4adXLNU+3Q/KSUd180a21HPPI+rUM8/tl+no30dOyD9mWDP1PDzs3o9k2HKboNtGIqW1PeAAu+wLp5IdZ1+kiKmUnzvLdcHAT29RK5n0yPOCpVLaPQnQ7kzNYka7c7L7SPhPJrd5IbqowWStSz7ZJmdBzMthT4Py3WtYuy2KF4ZfWPm7l0HKjIeKaIMp5Ih5tG0vRbbFIXDRGLah2BHt2y5LmrQd8HrWQmnrtn4CebhEiMoxMZWUlaWnKzZoff/yRc845B4vFwrhx4yJKoY761xg0aBCbNm1i4sSJnHnmmVRUVHDOOefw+++/07Nnz2iHr8ejjz5K586deeONNxgzZgzdu3fnxBNPDHyWLMs8/fTT/P3vf+fMM89kyJAhvP322+zdu5cvvvgCgPXr1zNjxgxeffVVxo4dy8SJE3n22Wf54IMP2Ls3/B26mpoaSktLA//KyuoXi2h9T7f6HNofxji4RDo5GfmEr4nmmqCc7rTK3SHbNJ7TrYlu/XK6q70+zlHzR319Tw25k2xk24bDFN0mGjG1rSMZeh4HFiv9uuTxu7+X8vyuBSzfqeRcj26jCu6Uds0PrW6AmsFKiPlUy0LlCasjJMT9SBPWthd/BLevhy4RhIYPvxS0WsVdJ0Q/QYERcU0Q5TyRCLatK7qbm9Otcf7IztgsEr/vKmbd3tIWvTdWaHPWPN1Vbh9pkrq+mp5ukxbSq1cvvvjiC3bv3s0PP/zAiSeeCMD+/ftJT2/5eTUmKi0jI4O7776bjz76iO+++44HH3yQ9u3bN/3GCPjqq68YNWoU559/Pjk5OQwfPpxXXnkl8Pr27dspKChgypTavpgZGRmMHTuWBQsWALBgwQIyMzMZNaq2dcmUKVOwWCwsWrQo7Oc+/PDDZGRkBP4NGFC/R2trF1ILfJ7m6bYY09Nt5DttRr4jqBVNq1bDy514SKncE7JN455uLbxcv5zuyqoqTrcqx5Vl2MUNzk2Ehd8U3SYaetl2SOcMlspK4TDvjgWsVS9MB6epN3GjyOfWGHP0VHY5emFFvWBOaae6fYxBWNtabUpYdiRkdoGj74A+pyg3NxIYEdcEUc4T8WrbcNto10It8XQDtEtzctLAPADeX9xyL2As0OasFFJzU+H21nq6nYnh6TZyrSLRuOeee/jrX/9Kt27dGDt2LOPHjwcUr/fw4cNbPJ4t2gnNnTu30dePPvroaD8ihG3btvHCCy8wbdo07rrrLpYsWcItt9yCw+HgiiuuoKBAKeiUm5sb8r7c3NzAawUFBeTkhIaw2Ww2srKyAtvUZfr06UybNi3wOD8/v57wbm1Pt6+u6DbDy1uMkU/4WiG1alkJL+8mFbDOF1pIrXmebn1EtyzLbF77O+OlckrlZNJ7HtPg3Ixm23CYottEQy/bDu2Uyev+vgBUb/sNr/9M8tJdtPVGn8+tkeKykzL1r/DFn9QnjBNaDjrZ9vi7ox8jDhBxTRDlPBGvtpUkqd61ayTh5RqXjO3Ct6v38cXve5l+Sn9SnFHLjBZR+10ww8sRYz81Mueddx4TJ05k3759DB06NPD85MmTOfvss1s8XtRHw7HHHlvvuRBh6Itt8Sa/38+oUaN46KGHABg+fDhr1qzhxRdf5IorrojpZwXjdDpDqrGXltYPnQmEtbRWeLm/biE1U3S3FCMvTlrLrxq/8v9kqYa6M2xRTneMRfftH61k38qFjHfAYUsW6XX2PyPbNhym6DbR0Mu2WSkOHF3H4NsrkVq5m3YcZmS3AUilagRLJD26wzHoXPjpXigvhJQW5Ei3AuZ+qx8i2laU80S82jbcNtGI7vE929I9O4XtByv4auXeQJ53axEQ3YDsq6HK7SPZoonulFady5FClGNKFPLy8sjLywt5bsyYMRGNFXU87eHDh0P+7d+/nxkzZjB69Gh+/PHHaIevR/v27et5mPv378+uXUrhBs0whYWFIdsUFhYGXsvLy2P//v0hr3u9XoqKiuoZtiUEwlpaqWWYltNNoGWYMQ4us2VYbLDblLlV+O2B59wpoftnY/uaJtrdOuV0f7t6Hxkonve8vPrpJKK0gtEwW4aZaOhp29PG9GWDrFyIjrJsYlTXNkHtwqL3dANgc8C4Pyt/t+sbmzFjhLnf6oeIthXlPBGvtg23TTSiW5IkLtYKqh2Bnt3Bnm6PuwavXyYJt/KiPbqWfKIQz6L7+eefp1u3brhcLsaOHcvixYub9b4PPvgASZI466yz9J1gE0S9agTnOWdkZJCdnc0JJ5zAo48+yp133hmLOYYwYcIENm7cGPLcpk2b6NpV6RHYvXt38vLymDVrVuD10tJSFi1aFIjFHz9+PMXFxSxbtiywzc8//4zf72fs2Mh7fLZ2Tne9lmEGDS838snJ0Dndanj4Vjoz3zKCd7xT2D82NIyysX1NC093+9VtYii6q9w+arx+2khKLqozvX5LItEWftPTbaKhp21PHtielVJ/AEZbNjKyaxvYv055sU23mH0OR90Kl34Gx8T+PBwN5n6rHyLaVpTzRLzatrGc7pYWUtM4d2QnHFYLq/NLWKYWi2wttDlbJKipUTzcSdQoLyaI6DayMykaPvzwQ6ZNm8a9997L8uXLGTp0KCeddFI9J2pdduzYwV//+tdA668jiW4qIzc3t544jgW33XYbCxcu5KGHHmLLli28//77vPzyy9x4442AsoP95S9/4cEHH+Srr75i9erVXH755XTo0CFwh6N///6cfPLJXHfddSxevJj58+dz0003ceGFF9KhQ4eI53bk+nQbu5CakQ96I5/wNU93ld/KrZa7+If3aqzOpJBtGpuztW71cl/sRHdxlXLnuK2ktiRKalNvGyPbNhym6DbR0NO2SQ4rUlfl5u5E61r6p1aooluKbfVtiwV6TY68QJlOmPutfohoW1HOE/Fq21iHl4OSRnPWcOVa+tVft0U0RqQEh5fX1ChiO9Wiim5HYohuUY6plvLkk09y3XXXcdVVVzFgwABefPFFkpOTef311xt8j8/n45JLLuGf//wnPXr0aMXZhifqnO5Vq1aFPJZlmX379vHII48wbNiwaIevx+jRo/n888+ZPn06999/P927d+fpp5/mkksuCWxz5513UlFRwfXXX09xcTETJ05kxowZuFy1rVjee+89brrpJiZPnozFYuHcc8+NqNF5MK0uurWbkAb3dBv5oDfy4hQohOaTqfEqv7FW0VyjcU+3mhOuQ053caXS8zvXXgkypuhuxTFbY+xER2/bDj/2XNxvPUgfaTcseUl5suMISM6K6ecYEXO/1Q8RbSvKeSJebauH6Aa4dlIPPlq6hxlrC9h5qIKubVsnnzr4OtxdUw1YSLWY4eVGpqysLKROVt0aWgBut5tly5Yxffr0wHMWi4UpU6YEOlOF4/777ycnJ4drrrmGX3/9NfaTbyFRi+5hw4YhSVK9g3PcuHGN3n2IhtNOO43TTjutwdclSeL+++/n/vvvb3CbrKws3n///ajnEvy9gw/2vcVV3PnJKq6a0I3J/XMbenuzaNrTrYluY4RHi3RyMvLipIlmj8+PWxPdtoaLldVFz+rlhyuVk1i2tRK8hBULRrZtOEzRbaKht2379eiKp9dk2PojLHheebLn8TH9DKNi7rf6IaJtRTlPxKtt9RLdfXLTOLZvO2ZvPMCrv27ngbMGRTxWS6jv6U4iRXIrzgGzkJohqVun69577+W+++4Lee7gwYP4fL6wnak2bNgQdtx58+bx2muvsWLFilhONyqiFt3bt28PeWyxWGjXrl2IVzmeefbnzTx6eaeQGw+SJPHmbzuYt+UgFosUtehuOqfb2OHlRsuVDiZ4rkabpya6vX4Zj6/lnm6tkJqP2BdSK1E93dnWCkV0m57uVhuzNcZOdFrDtvah5yuiWzsuTdFtEiUi2laU80S82jbcNtHmdGtcf3QPZm88wIdLd3Pz8b3ISddfFwTndHvc1Siiu0YR3fakRt8bL4iW071u3To6dqwtIlrXyx0JZWVlXHbZZbzyyitkZxunbWbUolsrYJaovL9oN3LyKu49Y2DtwW6x8MNapd/3ofKaqD8j3KIoy3KY6uXGEN1m9fLYEOjT7fEFfmstz1uj0erl6vu9Ooju4ipFdGeiFFIjqb6nW5SqtBpm9XITjVaxbd9TwOYCbzU40qDT6Nh/hgEx91v9ENG2opwn4tW2sa5eHsz4Hm0Z0SWT5buKeeXXbdx96oCm3xQldauXAyRJiVVITZQbWRppaWmkp6c3uk12djZWq7XRzlTBbN26lR07dnD66acHntM0ms1mY+PGjfTs2TMGs28ZEYnuluQ+33LLLZF8hDBIEny8bA+LthdxZTelqJTHJ7PzUCUARRXuGHxGONEdtEHA022Mk4BId4SNvDhpXu1Kty/ouRaEl2uiW4eWYVpOd5qsFlIzw8tbbczWGDvRaRXbOtOgz0mw7kvofjRY7U2/Jw4w91v9ENG2opwn4tW2eoWXa+PcPLk3V72xhHcX7uJPx/SkbWr0XszGCA4v93lU0a21DDPDy4XF4XAwcuRIZs2aFSiK7ff7mTVrFjfddFO97fv168fq1atDnvv73/9OWVkZzzzzDJ07d26NadcjItH91FNPNWs7SZLiXnQ/d9EIHpt3gF1Fldy7ag0AFW4fDvX1QxVuZFmOascPdwD5ghZDM6c7coy8ONUNJQ/3XGNztqs3Ybxak4KYim7lJJbqV4tfhPF0G9m24TBFt4lGq9n22OlQUwZH/1Wf8Q2Iud/qh4i2FeU8Ea+21VN0Axzbpx1DOmWwak8Jr83bzp0n94t6zMYI9nT7PMp1iktWWoclSni5KMdUS5k2bRpXXHEFo0aNYsyYMTz99NNUVFRw1VVXAXD55ZfTsWNHHn74YVwuF4MGhdYRyMzMBKj3fGsSkeium8edyIzs1obvRvbjtg9X8O0ORfweqnDTXn3d7fVT4faR6ow8kj9c3rE/eDE0WHi5SCcnIy9OmqdaQ5JaVkjNYpGwWqTanO5Ytgyr9AAyST5NdJs53a01ZmuMnei0mm1z+sNln+sztkEx91v9ENG2opwn4tW2jYnuaHO6tbFuOq4X17+zjLcX7OT6o3uQmexo+o0Ros052NPtDIjuxPN0i5AG0VwuuOACDhw4wD333ENBQQHDhg1jxowZgeJqu3btMvz3jTqn2wQykuy8evkobty/lhc/BBkJSQKrJOH1yxSVu6MS3eHyjoM1t+Q3C6lFipEXp7pebbvV0uJ8+SS7FY9Hj5xuN2lUYdVSG8zw8lYbszXGTnRM2+qHaVv9ENG2opwn4tW24bbRrjNi4ekGOGFALv3y0thQUMYb83dw2wl9YjJuOLQ5WyTweWsAuVZ0J0ifbiPXKoqWm266KWw4OcDs2bMbfe+bb74Z+wm1kJiI7j179vDVV1+xa9cu3O7QHOYnn3wyFh9heCwWiXOGd+RFIMlh5Yrx3Zi5rpD84ioOVdTQpW3kB3vY8HJ/sKdbFT6mp7vFGPmEb6/j6XZaLS22rctuwaeD6D5c6SFDUvO5bUlhw7aMbNtwmKLbRMO0rX6YttUPEW0rynkiXm2rd3i5Nt7Nx/fmxveX88b87Vw7qTtpLn1qWASHl/u9Huz4sKJ67M1CaiZHmKhF96xZszjjjDPo0aMHGzZsYNCgQezYsQNZlhkxYkQs5igMWlhL9+xU7jtjIMt2Hia/uCrqYmrhDqDw4eXG8NSKdHIy8uJU19PtsFmoO8WmRbc1KKfbE7O5lVR6aIMqusOEltedm9FsGw5TdJtomLbVD9O2+iGibUU5T8SrbVtDdAOcMiiPXjmpbNlfzqu/btfN2x1cSM3vdZNEde2Lpug2OcJErdKmT5/OX//6V1avXo3L5eLTTz9l9+7dHHPMMZx//vmxmKMwBMJa1NCOrBQlb+WQLqI7+INV0W2Q8GizZVhssFlC59M+09Xi0H2X3RrUp9vX6LYtobjKTRtJbRcWJrS87tyMFrofDrNlmImGaVv9MG2rHyLaVpTzRLzaVs+WYaGfI3HbFEVovzx3G4Wl1U28IzKCPd2y100yarswiw1s+uWSGwlTdBuXqFeN9evXc/nllwNK77OqqipSU1O5//77efTRR6OeoEjUHuzKTt5WFd3RerrDCUN/sOr2Gzu83MgnJyPndNe14yPnDGnx3fYku1WfPt2VHjJNT/cRGbM1xk50TNvqh2lb/RDRtqKcJ+LVto3ldMeikFowUwfnMaJLJlUeH0/+uCmmY2toc7aoojtJUq+/E6SIGhjbmZToRK0yUlJSAnnc7du3Z+vWrYHXDh48GO3wQlFXdGfFSHQ3HV5u7EJqRj7ojX7CP3NYBwZ2SOeXvx7LoI4ZEeV0x7pPd5XbR43XT6Zkiu4jMWZrjJ3omLbVD9O2+iGibUU5T8SrbVsrvFwb9+5TBwDw0bLdrN9XGtPxITS8XPYFeboTpF0YiHNMJSJR53SPGzeOefPm0b9/f6ZOncrtt9/O6tWr+eyzzxg3blws5igM9TzdqU4ADpbXRDVuuAPI7NMdG4y+OD1z4fCQxy0X3VbKAy3Dorv5o1FcpYzTVhPdDYSXG922dTFFt4mGaVv9MG2rHyLaVpTzRLzatjVFN8DIrm04dUh7vl21j4e+W88714yN6fjB4eX4gnK6E6RyOYhzTCUiUau0J598krFjlYPmn//8J5MnT+bDDz+kW7duvPbaa1FPUCT0Ci8PdwBpa6HVIhk+vNzIB71oi1MkorsGtUqoN7qbPxpKj27ItVcqTySZors1x2yNsRMd07b6YdpWP0S0rSjniXi1bWuLboC/ndQPh9XCr5sPMnvj/piOHezpxuchWdI83YkTXi7KMZWIRO3p7tGjR+DvlJQUXnzxxWiHFJYjEV5ukWRAXRjN8PIWI9ri1FLbJoWI7tgULjlcqezP2dZK8GKGl7fymM39LJPIEWkNEw3Ttvohom1FOU/Eq20bE92xzunW6NI2mSuO6sorv27noe/WM6FXdr1uLZGizVmSJCSfmyTU62/T021iAKLey6+99tomG5InCoECDlr18lS1enl57AupaX26Q1o5GyS8XKQqn6JUTtWou4A2Xb3cQrWsVuyMkae7RPV0Z1kqlCfM6uWtOmZD44lgY5EIPtZM28YOkc4PoiGibUVZw+LVtuG20f6vl6cb4KbjetMm2c6mwnJem7c9ZuMGughJIPk9JCVgTrdZSM24RL1qHDhwgJNPPpnOnTtzxx13sHLlyljMS0iORHi5QwpaFA0iukW6IyzaHcHIPN2a6K6u3XGioLhKEd1tUFuGmeHlrTpmQ+OJYGORMG2rDyKdH0RDRNuKcpzFq22PRHg5QEayPVBU7emfNrG7qDIm4waHl0t+M7xchP00kYhapX355Zfs27ePf/zjHyxZsoQRI0YwcOBAHnroIXbs2BGDKYpDQ+HlVR4fVe7IeyQ3Fl5uk4LCf8zw8hYj2uIUVU43xMTbreV0p8ma6DbDy1tzzIbGE8HGImHaVh9EOj+Ihoi2FeU4i1fbHinRDXDuiI6M65FFtcfPvV+tjcmYwYXUFE+3WUjNxDjExDXapk0brr/+embPns3OnTu58soreeedd+jVq1cshheGuqI71WnDoeapHKqIXOyEC3PUwsttlmBPtym6W4poi1PdELGWi+7o87qL1ZzuFL8puo/EmA2NJ4KNRcK0rT6IdH4QDRFtK8pxFq+2PZKiW5Ik/nX2YGwWiZ837Oe3LdG3GQ72dFv8ntqcbntiim4R0iASiZj+Gh6Ph6VLl7Jo0SJ27NhBbm5uLIc3PHVFtyRJMSmmFi4/Q9Xc2DB+eLmRD3rRFqdIPN1ubPhRt4uZp1vG6VNzul0ZYbcT5WJKwxTdJsGYttUHEcWLKIhoW1GOs3i1bbhttGshvQqpBdOzXSoXj+0CwEPfr8fvj07oB2orSWCVvUHh5Ykjus2cbuMSE5Xxyy+/cN1115Gbm8uVV15Jeno633zzDXv27InF8MJQV3QDdEuq4krrDA6XFEc8rhlerh+inPA1Wi66LYCEVwrK646S4io3SdRgQd33nGlhtxPdtnqMaYpucTBtqw8inR9EQ0TbinKcxattj6SnW+PWyb1JddpYk1/K16v2RjVWcHi5JbiQmhlebmIAohbdHTt2ZOrUqRw8eJCXX36ZwsJCXn/9dSZPnpxwP3Y40T295mnus79NxroPIh63MdFtN8PLo0K0xamltk2yK/uEJ4aiu7zGSxpV6gQs4AhfoERk28ZqvqboFhfTtvog0vlBNES0rSjHWbza1giiu22qkz8do7QffmbW5kD6ZCQEh5fb5ODq5aboNjnyRC2677vvPvbt28fnn3/Oeeedh9PpjMW8hKRuyzAObGRo9RIArCU7Ih43rOhWnYy24OrlBgmPbmne8ZFEtDCclobuu1TR7Y5hr+6KGh9pklpp1Jmm3FIOgyitYDT0mK/ZMkxcTNvqg4itl0RBRNuKcpzFq23DbdPaohvgygndyUy2s+1ABd+u3hfxOMGebhuJGV5uim7jEvWqcd1115GZmRmDqYhP4GD3uaHiICx6MfCataIw4nHD53RrfbrVRdEg+dxg5nTrSSQ53QA1Uux6dVe6vaRqnm5neoPbibbwm55uk2BM2+qDiB5DURDRtqIcZ/Fq2yOd062R6rRx9YTuADw7a3PEud3BOd12vAkZXi6aMymRML7KEIiA6N6zBJ7oB8vfCbzmqD4Q8biNhpdrOd0GCS0HsU5OopzwNSLL6Sa0V3eUVLp9pEqa6A6fz113bqLZ1hTdJqZt9UGk84NoiGhbUY6zeLWtEcLLNa44qhtpLhub95fzvyW7IhojOLzcjpdkzD7dJsbBFN0xpPZg94PfA34PXmsSAKnuyFshhBfdyuOAp9sgRdRArJOTaItTpDndNbIWXh4LT7evNqfbFN3NHjOW44YbTwQbi4RpW30Q6fwgGiLaVpTjLF5tayTRnZFk59bJvQF44Jt1bNlf3uIxgsPL7ZKPFIvaOSiBPN2iHFOJiCm6Y0jwwY4zA475G9uOfRaATN8hiHABC3cAaYUm7IHwclN0R4Joi1Ok4eXVmuj2VEU9h4oab1BOtxle3twxYzluuPFEsLFImLbVB5HOD6Ihom1FOc7i1bZGEt0AV0/ozsRe2VR7/Nz6we9Ue3wten+wp9sRktOdFOOZGhdRjqlExBTdMST4YMeZCsfdRXKf4wBw4UauLolo3HAHkBxoGWb8nG4jH/SiLU4t9nQ7FNFdFSNPt88vU+P1B+V0m57u5o4Zy3HDjSeCjUXCtK0+iHR+EA0RbSvKcRavtm1MdLdmTreGxSLxxB+GkpXiYO3eUu76bHWLxL82Z0lSC6mZ4eVHcCYmdTGOUosDgg92zfOcndWGMlm5w1Z2MD+iccMXUlMeB/p0G6gIWN2D3MgFyoLnauR5arTY022rK7qjy+mudHsBTNEdwZixHDfceCLYWCRM27YOpm1jR7wKQyMQr7YNt412LXQkPN0AuekunrtoOFaLxGe/5/PG/B3Nfq82Z6WQmo8kKfHCy81CasbF+CpDIIIPdk0Eu+xWDkmZABTv3x3RuI2Fl1sNGF5utgzTj5be0HA5lNer/DbliahFtxLqld6MQmqitILR0Gu+et7YEc3GImHaVj9M2+pDvLa1MgLxalujtAyry1G9svn7qf0BePyHjewraV5qXG3EqYQDLy5ZveYxW4aZGADjrxoCEXywB4vgUmtbACoORebpbjy8XPN0G0d0i3RHWLTFKeKc7hj16a6oUTzdbazqOK6MBrcV2baxnK+edhDNxiJh2lY/TNvqg0jnXg1R9oV4ta3RcrqDufKobozq2oYqj49Hvt/QrPcE11ZySh5cmH26TYyDKbpjSEghtSARXOnMBqDm8N6Ixg3nKfMF+nRrGxnnpxTp5CTa4tTSKIKkQCG12LQM0zzdGZroNsPLj9i4eo+d6Ji21Q/Ttvog0rlXQ5R9IV5ta2TRLUkS950xEEmCL1fsZemOoibfE1JbCbCiOqYSKLxctLTJRML8NWJIyMEe5On2JOUA4C8tiGjcRnO6MX6fbiMf9KItTi098dutFqwWiRpiU0hNE91pkim6j/S4eo+d6Ji21Q/TtvoQr8LQCMSrbRvL6T4ShdTqMqhjBheM6gzA9M9WN1nNXJuzpe7XTSBPt2hpk4mE8VWGQDTk6fan5gFgrSyMaNxwi6JfCy+3mOHl0SDKCV8jEtsm2a1BojvK8HK1kFpaM3K6RbatKbpNTNvqh2lbfRDp3Kshyr4Qr7Y1sqdb486T+5Gd6mDz/nKembW50W0Dcw7+vhY7WO06ztBYiHJMJSKm6I4hoZ7uWtPaMtoD4Kg6ENG4YUW36uq2YbYMiwbRFqdI5uiyW2Ln6a5RPd1ofbpN0X2kxtV77ETHtK1+mLbVB5HOvRqi7AvxalsRRHdWioMHzxoMwEtztrJ81+EGt9XmXK2mdQIJFVoO4hxTiYhxlFocEBLWEuR5drXpAECq52BE44b3dKN+lvFEt1m9XD8iCYd32a3UaDndnuZVAG0IrWVYsqyJ7oYLqYlSlVZDr/nqaQfRbCwSpm31w7StPsRrhW0jEK+2NWr18rqcPCiPM4d1wC/Dze//TkmlJ+x22pzLnHm1T1aXtMYUDYMpuo2L8VcNgQgJLw/KsU5v1wmATF/TRSDCET6nW/N0Gz+83MgnJ5Fzupu7mLrs1qDq5bHJ6a4V3aan+0iNq/fYiY5pW/0wbasP8eqNNQLxaluj53QH88BZg+iSlUx+cRV3fLIy7E0Bbc5ei4OffMNbe4qGQDRnUiJhfJUhECHh5UEiODNXKQKRRiXe6ooWj9toeLkB+3SLdHIS5YSvEcl8lZzu2FQvV3K6ZVx+U3Qf6XH1HjvRMW2rH6Zt9UGkc6+GKPtCvNpWhPByjXSXnecvHoHDauHHdYW8MX9HvW20OftlmOa5gQ1Zk+GMZ1t5pkcWUY6pRMQU3TGkIU9326x2VMuKp/Fw4e4Wj9tYeLlV69NtoPBykU5Ooi1OkXm6Y5fTXeX2kUwNFi3CwhTdR2xcvcdOdEzb6odpW30Q6dyrIcq+EK+2FUl0AwzulMFdU/sB8PD361m5uzjkdW3OMlBKCj8OfBRGXN7KszyyiHJMJSLGUWpxQEOebovVwiEpC4Di/bER3YE+3VohNQOHlxv5oBdtcYo0vLxGjlH18hofqah54ZIFHCkNbpsItj2S4+o9dqJj2lY/TNvqg0jnXg1R9oV4ta1oohvgiqO6cfLAPDw+mZv+t5ySqtr87mBPNyiRfomGKMdUImKK7hgS6ukONe1hm1JJ0VOc3+Jxwx1Act2WYaanOyJEW5wiFt0xCi+vdHtJk4JCyxuZQyLY9kiOq/fYiY5pW/0wbasPIp17NUTZF+LVtiKKbkmSePS8IXRqk8Tuoir+79NVtR7uOqLbZTfOtXFrIcoxlYgk3t6oI1oBh7qeboBiWzsA5JKWi+7GCqmJ4Ok2coGy4LkaeZ4akYvu2Hi6K91Bnm5neqPbirbwm6LbJBjTtvph2lYf4lUYGoF4tW24bYxaSC2YjCQ7z108ArtV4vs1BbyzcCdQO2dZuRLHmYCebrOQmnExvsoQCO0Om0WCuoXNypy5AFjL9rZ43LDh5epaGKhebiBPt9kyTD8iuUmQFMs+3W4vqZImuhvO5wZxWsFomC3DTIIxbasfpm31Q7QbyaLsC2bLMOMxrHMm/3dKfwAe/GY9a/JLzPByxLmRlYgYf9UQiJDw8jqe5yqXIrodlftaPG64A8gfEPhm9fJoEG1xMkJOd5rp6TbEuHqPneiYttUP07b6IZptRZmvSNc1GvEaXh7M1RO6ccKAXNw+Pze+v5xKt5LfrfnoXaboPoIzMamLKbpjSEghtToiuCY5DwBXVWGLx200p1uAPt1GPuhFW5wimW+S3Uq1ltPtiXFOdyMkgm2P5Lh6j53omLbVD9O2+iGabUWZr0jXNRqJILolSeLx84bQMTOJnYcqefu3HYDp6Q73t8mRxxTdMSTU0x1qWm9qBwBSawpaPG648B8tvNwqgKfbyGFYIofiNXcxdeqW022K7iM5rt5jJzqmbfXDtK1+iGZbUeabSKJbhJzuYDKTHbx46UiS7FY2FZQC4JeV75KIhdRESdlIRMxfI4bUerql+iI4XRXdnkPg89R9a6M0Fl5uDeR0G+cEINLJSZQTvkaknu7a6uXR5nSbotso4+o9dqJj2lY/TNvqh2i2FWW+Il3XaCSCp1tjcKcMnrt4ONq3qfL4ADO8XIT9NJEwRXcMaSyn25aei1u2YkGG8paFmDcrp9sML48I0RanyHK6LaE53VGcSCvcXtKaWUgtEWx7JMfVe+xEx7Stfpi21Q/RbCvKfEW6rtFIJNENMLl/LhN6tlUeSJqn2zjXxq2FKMdUImKK7hiiheKEq16eluSgUM5SHrSwbVhY0a0mrFglzdNtnIXFrF6uH5GEw4fkdCO3ONIiGMXTreZ0uzIa3Va0ECezerlJMKZt9cO0rX6IZltR5mtWLxeDY/pkq38lbni5KbqNS+LtjToSUkitjuc5zWVnH6roLo2B6FbXQhH6dBv5oBdtcYrM0x2U0w3grYrosz0+P26v3/R0G2RcvcdOdEzb6odpW/0QzbaizFek6xqNaD3douR0B5PsUK+FTU93vb9Njjym6I4hIeHldT3dLhsFmqe7tGW9usN5Y/2B6uU+9QXj/JR1D3Ij3xGOxHN8JIlUdLux1T4RYV53pVvZ18ycbmOMq/fYiY5pW/0wbasfotlWlPkmkujWroVE9HRrc3bZreSlu8hIsjfxjvhDtAjORML4KkMgQj3doaZNd9nYK6u5Ji0U3Y3mdGP86uVGPuhFOeFrRJrTDVJQMbXIKphXur0ApJuebkOMq/fYiY5pW/0wbasfotlWlPmKdF2jkWg53VA754m92/HDbUdjtyaezBHlmEpEEm9v1JHG+nSnuewBT7dcuqdF4zYWXh4Q3Qby0op0chJtcYpkvlqfymgrmGue7nSLJrrTG90+EWx7JMfVe+xEx7Stfpi21Q/RbCvKfEW6rtFIZNHtsFkT0ssN4hxTiYhxlFoEPPLII0iSxF/+8pfAc9XV1dx44420bduW1NRUzj33XAoLQ6uF79q1i1NPPZXk5GRycnK444478Hq9Uc+nserlaS4b+1TR7S+OPqfbp6pum6SFl5ue7kgQbXGKZL7JDiW0PNpe3ZU1dcPLTdF9JMfVe+xEx7Stfpi21Q/RbCvKfEW6rtFIZNEtwu+jF6IcU4mIsKJ7yZIlvPTSSwwZMiTk+dtuu42vv/6ajz/+mDlz5rB3717OOeecwOs+n49TTz0Vt9vNb7/9xltvvcWbb77JPffcE/WcGvN0J9mtFEpqVcUY5HTLgT7dWni5cX5KkU5OwXOL15zuNJcqurW2YZ7IRHeFGl5u5nQbY1y9x050TNvqh2lb/RDNtqLMV6TrGo1IRbd2LSRiIbVAFyEBruf0wszpNi5C7pXl5eVccsklvPLKK7Rp0ybwfElJCa+99hpPPvkkxx9/PCNHjuSNN97gt99+Y+HChQD8+OOPrFu3jnfffZdhw4Zxyimn8MADD/D888/jdrsb/MyamhpKS0sD/8rKyuptE9IyrI6nW5Ikyh05yusVBeD3Nfv7NhZeLhmwerlIB7xIc4XIbhKkqqK7So7O013l9gEyyc0U3aK0gtEwW4aZBGPaVj9M2+qHaLYVZb51rw+MPFeNRGwZZnq6xbmRlYgYf9UIw4033sipp57KlClTQp5ftmwZHo8n5Pl+/frRpUsXFixYAMCCBQsYPHgwubm5gW1OOukkSktLWbt2bYOf+fDDD5ORkRH4N2DAgHrbhFYvr29anytTeV32Q3VJs79vY+HlVoOHlxv9gBdprhDZfFOddcPLI8vprnB7SaYGK+rdb9PTfUTH1XvsRMe0rX6YttUP0Wwr0nxFmiuY4eWJimj7aSIhnOj+4IMPWL58OQ8//HC91woKCnA4HGRmZoY8n5ubS0FBQWCbYMGtva691hDTp0+npKQk8G/dunX1tmmsTzdAclIylbJTeRCl6K4XXm4gT7dIB7xIc4XI5uuyW3FYLVRHWb28osZbG1ouWcCREvO5HklM0W0SjGlb/TBtqx+i2Vak+Rp9fnUxRXdiItIxlWjYmt7EOOzevZtbb72VmTNn4nK5WvWznU4nTqcz8Li0tLTeNo316QYlt7aEFJKpgeriZn92uPAfX0B0q15HAx1YIuVJizRXiDwcPtVlo8YdXXh5WbWXNKlSeeBMa3KfE23hN0W3STCmbfXDtK1+iGZbkeZbV5QaHTOnOzERJWUjERHq11i2bBn79+9nxIgR2Gw2bDYbc+bM4T//+Q82m43c3FzcbjfFxcUh7yssLCQvLw+AvLy8etXMtcfaNpHSlKc7zWWnRFa9g1XF9V5viMZyuo3ep9voJyeR5gqRzzfVaYu6enlptbfZlcshcWx7pMbVe+xEx7Stfpi21Q/RbCvSfBNFdJuebrER6ZhKNIQS3ZMnT2b16tWsWLEi8G/UqFFccsklgb/tdjuzZs0KvGfjxo3s2rWL8ePHAzB+/HhWr17N/v37A9vMnDmT9PT0sHnaLaG5nm6gRZ7u8KJb+SyL5uk2w8sjQqS5wpEV3eXVXtKk5hVRg8Sx7ZEaV++xEx3Ttvph2lY/RLOtSPM1RbfxMUW3WMdUoiFUeHlaWhqDBg0KeS4lJYW2bdsGnr/mmmuYNm0aWVlZpKenc/PNNzN+/HjGjRsHwIknnsiAAQO47LLLeOyxxygoKODvf/87N954Y0j4eCRoYS0NebrTXXZKY+Xp9tcNLzfO/RORDniR5gqRzzfNZQvK6Y6skFpZtcf0dBtoXL3HTnRM2+qHaVv9EM22Is3XFN3GxxTdYh1TiYZQors5PPXUU1gsFs4991xqamo46aST+O9//xt43Wq18s033/DnP/+Z8ePHk5KSwhVXXMH9998f9WdrB7tFksKK4DSXjVKSlQdRFlKrDS/XRLdxPN0iteESaa4QOseW5OqkuWy1fbpjldPdBKLlFZktw0yCMW2rH6Zt9UM024o0X21+Rp+nhtkyLDExRbdxEV50z549O+Sxy+Xi+eef5/nnn2/wPV27duW7776L+VxCwsvD5nTbanO6IyykJlp4udFPTiLNFWIUXu6JUHTXeGjfzB7dIN7Cb3q6TYIxbasfpm31QzTbijTfRPF0m4XUxEY0Z1Iikbh7pQ6EFFILm9Mdw0JqZnh5TBBprhBNeLmdmihbhpWHFFIzRfeRHlfvsRMd07b6YdpWP0SzrUjzTRTRbXq6xUakYyrRMI5SiwOa5emOWSE19bEBRXcwRj/gRVucIvZ0u2xUBwqpRZrTHVRIzWXmdB/pcfUeO9Exbasfpm31QzTbijRfU3QbH1N0i3VMJRrGVGqC0hxPd6ms5nRH6emu7dOtJXcbJ7wcxDk5ibY4RRVeLkfn6VZahmk53aboPtLj6j12omPaVj9M2+qHaLYVab6iXNdomKI7MRHpmEo0TNEdQ1rm6S5p9rjh8jPkujndBiqkBvVzg4xK8IJk9LlCNOHlQdXL3eURfXZ5jcdsGWagcfUeO9Exbasfpm31QzTbijTfRBHdZk632Jg53cYlcfdKHQgc7BJhw73Tg3O6ow0v1+qnGbCQGtSvgmlURFucIr1JkOaycUDOVB6U729023B4fH6qPf4W5XSLVJUWzOrlJqGYttUP07b6IZptRZqvWb3c+JiebrFuZCUaYqwcghASXh62T7eNElKVbWMUXm6RjZnTLcodYdEWp8jDy+0UyG2UB6V7W/y5ZdVeANPTbaBx9R470TFtqx+mbfVDNNuKNF9Rrms0zPDyxESkYyrRMJZSE5yQ8PIw4d4ZycGe7hJo5oIW1tMtSHi50Q940RanaHK6C1FFd9m+Zu97GuWq6E4PiG4zp/tIj6v32ImOaVv9MG2rH6LZVqT5inJdo2GK7sREpGMq0TBFdwxpytPttFmRXBnKNrIPasqaNW64kGJtLawNLzfWTynKyUm0xSnScPg0l40COUt54KmEmtIWfW5ptUcZxxTdhhlX77ETHdO2+mHaVj9Es6dI+4Io1zUa0YpukXO6RfmN9CDSNEQT/TF/jRjSlKcbIDU1lRrZpjxoZjG1cELL59cEvrE93UY/4EVbnCK9QElz2ajBURtpUbqvRZ+rhZebfbqNM67eYyc6pm31w7StfsSjMDQK8WjbcNvUOnfE9XSLcD2nF6LVKkokEnev1IGmPN0A7dJdlLawV3fzwsuN9VOKcnIS6YRfl5aGlwO1ed1lLcvrLq/xAjLJpug2zLh6j53omLbVD9O2+iHKuVdDpH0hHm1rhpfHHyIdU4mGsZSa4NSGtWj/qU92qrPW29jMYmqNiW6rbMzq5aKcnERcnCKxbapLEd2FAdFd0KLPLKv2kEwNVu0mj8sMLz/S4+o9dqJj2lY/TNvqhyjnXg2R9oV4tK0puuMPkY6pRMMU3TEkENbSSHh5uzQnpSQrD6LxdGsObtRF0WDh5WbLMP2IJHTfabPisFlq87pbWMG8rNpbG1ouWcCe3OR7RGoFA2bLMJNQTNvqh2lb/YjHtlZGIR5ta7YMiz/iWXQ///zzdOvWDZfLxdixY1m8eHGD277yyitMmjSJNm3a0KZNG6ZMmdLo9q2BGCuHIDQnvDwST3c4YegPfJZP3chYotvM6daPSO+2p9WtYN4Cyqo9pEmVygNnWoORHOHmWfdvo2J6uk2CMW2rH6Zt9SMevbFGIR5t21hOt8iF1ES5ntMDEZ1JzeHDDz9k2rRp3HvvvSxfvpyhQ4dy0kknsX///rDbz549m4suuohffvmFBQsW0LlzZ0488UTy8/Nbeea1JO5eqQPNKaTWLs1JCUFtw5pBozndgT7dxjqwRDk5iXTC14jUtqkuW+Th5TVBnu5mVC4H8Wxrim6TYEzb6odpW/0Q5dyrIdK+EI+2NcPL4w+RjqmW8OSTT3Lddddx1VVXMWDAAF588UWSk5N5/fXXw27/3nvvccMNNzBs2DD69evHq6++it/vZ9asWa0881pM0R1Daj3dUsOF1II93VEVUlMeWwwaXi7KyUnExSliT3dw27AIwstr24U1XUQNxLOtKbpNgjFtqx+mbfVDlHOvhkj7Qjza1hTd8YdIxxRAWVkZpaWlgX81NTX1tnG73SxbtowpU6YEnrNYLEyZMoUFCxY063MqKyvxeDxkZWXFbO4txRTdMaTFnu4YFFKTZGOHlxv9gBdtcYIoPN3OYE93y1uGmZ5uY42r99iJjmlb/TBtqx+inHs1RNoX4tG2puiOP0Q6pgAGDBhARkZG4N/DDz9cb5uDBw/i8/nIzc0NeT43N5eCguZFbv7tb3+jQ4cOIcK9tbEdsU+OQ0JzusPfz8hOdVKqerrlqmKaczg01qfbYvbpjorgBcnoc9WIXHTb2aqJ7vJC8PuafbOmvNrDGMtW5UFabuMb15lnJHM9Epii2yQY07b6YdpWP+JRGBqFeLRtuG3MnG6xES2ne926dXTs2DHw2Ol0xvwzHnnkET744ANmz56Ny+WK+fjNxRTdMSRwsDfi6W6b6gh4uj0VRTiaMW64RVG7ASnJxuzTbVYv149Ib2iku2wcIgO/ZMUi+6B8P6S3b9Z7K6uqOc86V3kw+PxmvUekqrRgVi83CcW0rX6YttWPeKywbRTi0bZm9fL4Q6QbWQBpaWmkpzceQZmdnY3VaqWwsDDk+cLCQvLy8hp977///W8eeeQRfvrpJ4YMGRL1fKNBjJVDEELCyxvwINqtFnwOZefyVR5u1rjhDqB6nm4zvDwiRFucIApPt8uGHwsV9rbKE2XNz+vuX76AdlIJbldb6HNyi+YZyVyPBKan2yQY07b6YdpWP0Q592qItC/Eo23N8PL4Q6Rjqrk4HA5GjhwZUgRNK4o2fvz4Bt/32GOP8cADDzBjxgxGjRrVGlNtFFN0x5CQ8PJGwr39SdnKHxUHmzVuozndGNPTLcrJScTFKZqcboASm7r/taCC+QnVPwBQ3Pt8sNqb9R7RbGuKbpNgTNvqh2lb/RDl3Ksh0r4Qj7Y1RXf8IdIx1RKmTZvGK6+8wltvvcX69ev585//TEVFBVdddRUAl19+OdOnTw9s/+ijj/KPf/yD119/nW7dulFQUEBBQQHl5eVH6iuY4eWxpDmebkDJia0Ee9UBJU68iYOi8fBytZCaKbojQsTFKRpPN8Bha1s6QfNFd2UR4/zLAageckmzP08025qi2yQY07b6YdpWP0Q592qItC/Eo21N0R1/iHRMtYQLLriAAwcOcM8991BQUMCwYcOYMWNGoLjarl27QtIlXnjhBdxuN+edd17IOPfeey/33Xdfa049gCm6Y0hzPd229FwoBJuvCtzlTbZgCpdz46vbp9ug4eVGz30KXpCMPleNSPPls5KVCgLFcrLyRDP7xPtL9mJF5qCcjiuvT7M/T7SFvzVEd6wRzcYiYdpWP0x76kc8CkOjEI+2DbeNWUhNbESqk9BSbrrpJm666aawr82ePTvk8Y4dO/SfUAuJr1/jCBPq6W7YtBkZbaiQ1ep85fubHLfx8HKzT3c0iHTC14jUttmpyj5X5FUrN9aUNut91ZXKduVyEumu5oWW152fCLbVW3TrYQPRbCwSpm31Q5Tzg4iIZluRjrN4tK3p6Y4/RDqmEg1TdMeQ5nq6s1OdHJAzlQcRi271sdmnOypEXJwiFt1piug+6FFv+FQ3T3RXlSkF/ypIwmlr/pIhmm1N0W0SjGlb/RDl/CAiotlWpOMsHm1riu74Q6RjKtEwRXcMCWkZ1ogIbpfm5AAZyoPywga30wgrulXVbbYMi45EahmWnaqElxe6VdHdTE/35t37APDZU1pkI9FCnPRuGaaHDUSzsUiYttUP0VoviYRothXpOItH25otw+IPU3QbFzFWDkEICS9vxNPdIcPVIk93OGEYCC8PFFIzpqfb6CcnEXO6ow0vP+xPUp5opqd79bY9AKRlZLXo80Rb+E1Pt0kwpm31QzSPoUiIZluRjrN4tK2Z0x1/iOhMShQSd6/UgZDw8kY83f3bp3NAVjzd1cX7mhy30fByLafbYAuMKCcnkU74GpHa1mW3kuq0UaYVUmuGp7uwtJrCAwcAyMnOjmiedf82KqboNgnGtK1+iHJ+EBHRbCvScRaPtjXDy+MPkY6pRMNYSk1wQj3dDZu2TYoDt6sdACX79zQ5buPh5cb2dBv9gBdxcYrGttmpDspovqf765V7SZarAUhJy2zRZ4lmW1N0mwRj2lY/RDk/iIhothXpOItH25qiO/4Q6ZhKNEzRHUOa6+kGSGrTHoDq4r1Njhve023snG5RTk4iLk7RiW5nizzdn/+eT6pUpTxwprbos0SzrSm6TYIxbasfopwfREQ024p0nMWjbU3RHX+IdEwlGsZSaoLT3JxugMzcTsq2EVYv99XN6Tarl0eEiItT1KIbrU9346K7yu1j7d5SUtFEd3pE86z7twiYotvEtK1+iHJ+EBHRbCvScRaPtm1MdIuc0y3Kb6QHIh1TiYYpumNI4GBHalIEd+jYFYAk96Emxw1XFEGum9Nt0PByoxezCF6QjD5XjWhO/G1THaGe7kZOqhVuL0Ctp9sR355u0OeiyhTdYmLaVj9EEy8iIZptRTrO4tG24bbRroVE9nSLcj2nB2YhNeOSuHulDgQO9mZ4unt06wFApr+Yl+ds5l/frsPnD7/ANR5ebkxPt9kyTD+iuaGheLrVnG5kcJc3uG1ljbJvpVtqlCecaS36LJFawWjo0RLGbBkmJqZt9UO01ksiIZptRTrO4tG2Zsuw+EOkG1mJhhgrhyDIgbAWmhTBme06AGCXfPz3+2W88ut2lu08HHbbsOHl9fp0G+vAEuWOsIiLU1Th5WlOanDgwa480Uhet+bpTk+QnG4wPd0mtZi21Q9Rzg8iIpptRTrO4tG2Zk53/CHSMZVomKI7hsiyFl4OTRY2szkptyg5su2kEgB2HqoIu2m4AygQXm5WL48KERenaGzbLtUBQKXUdF53pSq60yxK9fKWeroTzbatOWbdsfUaP5ExbasfopwfREQ024p0nMWjbU3RHX+IdEwlGqbojiEhhdSaEe7tyMgD4MQuyuPdh6vCbhcu/EcLL0fzdBssvNzM6daPaEL3s1OdALXF1BrzdKvh5YlUSM0U3SYapm31QzTxIhKi2Vak4ywebdtYTrfIhdREuZ7TA5FSNhIN89eIIYHwcmiW59mRqYjuYVlKzuzuosqw2zUvvNyYotvoJyeRTvgaUYWXq6K72N90r27N050sm4XUjDZm3bH1Gj+RMe2pH6KcH0RENNuKtIbFo21NT3f8IdIxlWiYojuGBMLLm+npJk3p1d2ZA0DLRHeg5logvNxYP6UoJycRF6dobNtWDS8v0UR3MzzdSbK6X5rh5YYZs+7YJrFHlDVMREzb6odothXpPBGPtjVFd/wh0jGVaBhLqQmO36eIlOZULwegw3AA2petBmD34ZaI7jqeboOFl5vVy/UjmtD9VKcNp80S1Ku7pMFtK91e7Hhx4FGeaGEhNRFDnEStXi6KfUXCtK1+mLbVD9FsK9J5Ih5ta1Yvjz9M0W1cxFg5BCFwsEPzRHDnMQCkHViOhJ/C0hqqPb56m4UThvVEt0E93UY/OYmY0x3N3XZJkpS2YXLTnu7yGh8pBNUZcJiebqOM2RpjJzqmbfXDtK1+iGZbkc4T8WhbM6c7/hDRmZQoJO5eqQMhLcOaI4LzhoA9GUtNCYMdBQDkF9cvphbW063Flxu0T7coJyeRTvga0do2O80Z5OluPKc7VVIrl9uSwGqLaJ51/zYypug20TBtqx+mbfVDNNuKdJ6IR9ua4eXxh0jHVKJhiu4YUpvTLanKuwmsdug4EoDJKduB8HndjeV0G93TbfQDXsTFKVrbtkt1NLt6eW3l8pZ5uSExbdtaY7bG2ImOaVv9MG2rH6LZVqTzRDza1hTd8YdIx1SiYSylJji1orsFZu08FoDR1k2AzJ5DZfU2aSynG7NPd1SIuDhFLbrTnJRq4eVN5HQHwstbmM8NiWnb1hqzNcZOdEzb6odpW/0QzbYinSfi0bam6I4/RDqmEo2WxYuaNEogvLwluSRdxgEwpHoZvzhWkTnHDqOXgd0V2KRx0W3MQmqinJxEXJyi93Q7KWhGeHmF20eaZHq6jThma4yd6Ji21Q/Ttvohmm1FOk/Eo21N0R1/iHRMJRqmpzuG1B7sLTBrp9GARKq3iO6WQtrU7IGDm0I2CVtIzQ8gGz683OjFLIIXJKPPVSNq0Z3uokxuOry8ssZbG17ewiJqIObCb4puEw3Ttvph2lY/RLOtSOeJeLRtuG3MQmpiYxZSMy6Ju1fqgN+vtgxrycGelAldxiMjUSE7leeKd4Zs0pCn20LQXUiDhZebLcP0I9obGjnNLKRW4faSohVSi8DTLVIrGA2zZZiJhmlb/TBtqx+i2Vak80Q82tZsGRZ/iHQjK9EQY+UQBNkfgacb4NJP2HrpQn7xD1ceF+8KeTncAeSTZawE3YU02ElAlDvCIi5Oscjpbk7LsEq3j1TUwn5mTrehxmyNsRMd07b6YdpWP0SzrUjniXi0rRleHn+IdEwlGsZSaoITKKTWUgHsSKF9517sltsBUHNwe8jLdQ8gWZaRZbAEi26DebpFOTmJuDhFa9tgT7fcSCG1ihovqUTu6U5E27bWmK0xdqJj2lY/TNvqh2i2Fek8EY+2NUV3/CHSMZVomKI7hkSU062S4rRRaMkFwHcoVHTXDf/R1sFQ0W2sn9LM6daPaEP3s1OdQTndZVqBgHpUun2kaoXUHKan20hjtsbYiY5pW/0wbasfotlWpPNEPNrWzOmOP0RK2Ug0zF8jhkTs6VY5aG8PgKWk8fByn6q6Q8PLTU93JIh0wteI1rYuuxXJla6MgQzu8rDbVdQEtwxLj3iedf82MqboNtEwbasfpm31QzTbinSeiEfbmp7u+EOkYyrRMEV3DAm0DIvQ61zs7ACAvWw3BC12dQ8grV2YGV4ePSIuTrGwbXpaGm5Z3WfC5HXLskxlSMsw09NtpDFbY+xEx7Stfpi21Q/RbCvSeSIebWuK7vhDpGMq0TBFdwwJhLVYIzNrmTMPvyxh9VZBxcHA8/VzupW/Q6qXG8zTbVYv149YhO7npLuCKpjXz+t2+/x4/TIpUeR0ixjiZFYvN9Ewbasfpm31QzTbinSeiEfbmtXL4w9TdBsXMVYOQYgmpxvA6UqikDbKg6C2YfXCy/1hwssNmtNt9ANexMUpFrZtl+bkoJyhPCgvrPd6ZY3S/s7M6TbmmK0xdqJj2lY/TNvqh2i2Fek8EY+2bczTLXJOtyi/kR6IdEwlGsZSas3g4YcfZvTo0aSlpZGTk8NZZ53Fxo0bQ7aprq7mxhtvpG3btqSmpnLuuedSWBgqLHbt2sWpp55KcnIyOTk53HHHHXi93qjmFm1Od4rDGqhgzuEdgefremPrh5dLYLADKxbe2NYgeEEy+lw1YnHiz0lzUiirN3hK99V7vcKtHAtpUfTpFnHhN0W3iYZpW/0wbasfotlWpPNEPNo23DbatZDInm5Rruf0QMQIzkRBuL1yzpw53HjjjSxcuJCZM2fi8Xg48cQTqaioCGxz22238fXXX/Pxxx8zZ84c9u7dyznnnBN43efzceqpp+J2u/ntt9946623ePPNN7nnnnuimlvEfbpVUpy2WtHdiKdb/ZhaT7fBQstBnJOTSCd8jVh5ugvkLOVB2d56r1e6FU93bU63WUjNSGO2xtiJjmlb/TBtqx+i2Vak88T/s3fm4VEU6R//zkzukIQzCXcEVO5DjoiouIoGdXUR1mWRBUTAi3gQ14NVwWMVD0RUUNYDUXdZWI9VV/yhgOKBIMqhIoiCKHIkgMgNuaZ/f0y6M5OZSTJJV1Lv9PfzPPPA9PTUvPlWV1W//b5VFY3ack539CGpTTmNmPo2IFIWL14c8H7evHlIT0/HmjVrcPbZZ+PgwYN44YUXMH/+fJx77rkAgBdffBGdOnXCqlWrcPrpp+P999/Hxo0bsXTpUmRkZKBnz564//77cfvtt+Oee+5BXFxcjWwrj3TXzAlOjovBDiPd9+a3SpxurxnpLusQNVtEDZAzOEnsnOyJdCdguzmV4XB+0OdHC32R7vI53Uwv16nMuijb6VBbdVBbdUjTVtI4EY3a0umOPiS1KachLtJdkYMHDwIAGjf2Re3WrFmD4uJiDBo0yDqnY8eOaNOmDVauXAkAWLlyJbp164aMjAzrnJycHBw6dAjffvttyN8pLCzEoUOHrNfhw4eDzqntnO7ASHf5tmHBke4yp9tVFunWbD43IGdwktg52RXpLjAj3SHSy81IdzKO+Q4wvVyrMuuibKdDbdVBbdUhTVtJ40Q0akunO/qQ1Kachn7eWgR4vV7cfPPNGDBgALp27QoAyM/PR1xcHBo2bBhwbkZGBvLz861z/B1u83Pzs1BMmzYNaWlp1qtz585B51iNvYZzSRrEe/CLtyzS7ZdeXnF+hrlPd4yA9HLd59X4d0i622piT6Q7HvnmnO4Q6eVHC0sQh2LEoWydAy6kplWZdVG206G26qC26pCmraRxIhq1DXWOeS8keSE1KfdzKuCcbn0RfVVOnDgRGzZswIIFC5T/1uTJk3Hw4EHrtXHjxqBzatvYk+JjsB9lEcXjB6zjFTtF8+FjjEvf9HJuGaYOOx5o+M/pNsJEuhviSNkPemo0p1vSVjAm3DKMmFBbdVBbdUjTVtI4EY3ahrpXY6RbNpIeZDkNGT1HCHJzc/HOO+/gww8/RKtWrazjmZmZKCoqwoEDBwLOLygoQGZmpnVOxdXMzffmORWJj49Hamqq9UpJCU63rfWc7vgYHDPifW+KyheGC5deHmse1nAAkPJEWGLnZIe2aYmx+M3dxPfm6B6gNHDl/qNFJWjsKptCkdS4RteYU7WtizLromynQ23VQW3VIU1bSeNENGob6m+i0y0bSW3KaejnrVWBYRjIzc3Ff//7X3zwwQc46aSTAj7v3bs3YmNjsWzZMuvY5s2bsX37dvTv3x8A0L9/f3zzzTfYs2ePdc6SJUuQmpoaMm282rZ5a5denhznwREk+t6UFgKlxb7yKjQga59uzumuNRI7Jzu0dblccDdohmLDA5fhDdqr+1hhKRpZTneTWtlZW1vrEjrdxITaqoPaqkOatpLGiWjUlk539CGpTTkNcauXT5w4EfPnz8dbb72FlJQUaw52WloaEhMTkZaWhnHjxiEvLw+NGzdGamoqbrjhBvTv3x+nn346AOCCCy5A586dMWrUKDzyyCPIz8/HXXfdhYkTJyI+Pr7GttV2TndyfAyOIaH8QNERILFRUAp0eXq56XTrl17OOd3qsGvgb9EoGXt2N0RL/Aoc3g2ktbQ+O1pUgsag061rmXVRttOhtuqgtuqQpq2kcSIatQ31N0nep5tzumVOm3QK4q7KZ555BgcPHsQ555yD5s2bW6+FCxda5zz++OP4/e9/j2HDhuHss89GZmYm3njjDetzj8eDd955Bx6PB/3798df/vIXjB49Gvfdd1+tbLPSy2u6enlcDIoRgyLzWUhZinnFztBKL3eXdYgaL6Sme4OXNOCb2DVfvnnDBBSYi6kdClxM7ViRf6S7cY3Kl6gtnW5iQm3VQW3VIU1bSeNENGpbWaQbkOd4M9ItM5jkFMRFuqvTASQkJGD27NmYPXt22HPatm2Ld999107TbNgyzOc8H0cC4nAEKDxSVl7o9HK3xgupSRmcJA34JnZp2zwt0VpMDYcDF1M7WugX6U6k061bmXVRttOhtuqgtuqQpq2kcSIata2O0y3l7wXodAOy2pTT4CMQG/HWdk53vO8ZiJViHiLS7XK5UPYziLWcbv2qUcrgJLFzskvbllVEuhtzTre2ZdZF2U6H2qqD2qpDmraSxolo1JaR7uhDUptyGvp5a4Ix08vdtVi9HAAOG6bTHTrSbaaXW3O6NUwf4ZZh6jDtrG3aUFWR7toupCZpKxgTbhlGTKitOqitOqRpK2mciEZtQ/1NdLplQ6dbX2T0HEKwGrunhk53nO97Rys43RU7wyCnW+P0ct0HJ4lzX+x62t6iYSLyK4t0cyE1bcusi7KdDrVVB7VVhzRtJY0T0ahtqL/J/17IXJhMClxITWYwySk496pUQO3ndPsi3UeMKtLLy/pAy9XmQmo1RtKAb2Kf052AAvgi3d5DFSLdRbWPdDtZW9Vl1kXZTofaqoPaqkOatpLGiWjUlunl0YekNuU06HTbSG23DIv1uBEX4y6f013oc3rCp5dzTndtkdg52aVtWmIsfvM08705tAMoKbI+O3isGI1cvkwLOt36lVkXZTsdaqsOaqsOadpKGieiUVs63dGHpDblNPTz1gRT3thrHnlOjvPgaJULqZlOd2nZQUa6a4rEzskubV0uF0rT2mCfkQp3yQngl8+tz/YcLkQjcMswXcusi7KdDrVVB7VVhzRtJY0T0agtne7oQ1Kbchp0um2kfE53zWVNjo/xm9N91Dru3zGaq5dbkW4N566Y9uo+rybcPCadsXPgb94oGR97u/vebFkCADhWVILiwmNIdhX6jtPp1q7Muijb6VBbdVBbdUjTVtI4EY3ahvqbOKdbNpzTrS/OvSoVUN7YaxPpjvGLdB+xjvt3jOY+3R6N08u5erk67Hyg0SItEctLe/re/OBzuvccKo9yG+4YID61RmVLWpXWhKuXExNqqw5qqw5p2koaJ6JRW65eHn1IepDlNGT0HEIon9NdC6c73hO0ejkQ6HQbQXO6mV5eUyR2TrZGuhsm4GNvN3jhBvZsBA7uwJ7DhdYe3a6kJkANf8fp2qossy7KdjrUVh3UVh3StJU0TkSjtkwvjz4ktSmnQafbRmyZ0x0f47eQWhWRbpj7dNPprikSOyc7tW2RlogDSMHWuI6+A1uWYs/hE7VeubyifU7UVmWZdVG206G26qC26pCmraRxQrK2VZ1Dpzt6kNSmnAadbhuxZU53QHp55XO6PYx01xqJnZOtTnfDRADACldP34EflmDPocJa79ENUFuVZdZF2U6H2qqD2qpDmraSxgmp2lZmb1VOt9Q53VLqSAWS2pTToNNtI7XdpxsAksKkl/vPuwlOL9evGu2cd6yScIuH6Iyd8+VbNPRda+8c7+Y78ONy7Dt4BA2t7cJqtogaILPjp9NNTKitOqitOqRpK2mckKptpE63/72Q1Ei3lPs5FUhaJ8FpsDZsxI453Q3iq7GQmsH0cruQNOCb2Kltm8ZJiI9xY01Ra5QkNgWKjqDBni8Y6abT7XiorTqorTqkaStpnJCqbW0i3VKdbil1pAJJbcpp0Om2EcMKPNfcCU6Ki8FRw5fyW2V6uel0axzp1r3BS+yc7NQ2xuNGlxapMODGrqZnAgDa/rqifE53IiPdOpZZF2U7HWqrDmqrDmnaShonpGpLp9tZSGpTTkM/b00wXqNsyzBPbSLdHhxFvO9NWKe7LL3crD0NI93cMkwddqfud2/VEADwRWxvAECXY59bq5fXJtItMcWJW4YRE2qrDmqrDmnaShonpGpbmb3cMiz6oNOtLzJ6DiHYMac73Orl/k6s11q9vNT3ocaRbt0HJ4lzuu1+2t6tZRoA4J0jHQGXG1neX9DN9aPvQ6aXa1lmXZTtdKitOqitOqRpK2mckKptbSLdUhdSk3I/pwIn/+26w5qxESu9vBaR7uS4GByx0suPWIWGTC/n6uW1RtKAb2K3tt1b+Zzuz/O98LbsCwA4yV3g+zC1eY3LpbbqyqyLsp0OtVUHtVWHNG0ljRNStWV6ubOQdp06CTrdNmLfPt1l6eVGKVByoqzMEOnlXEit1kga8E3s1rZdswZIivPgWFEpNrX+EwqNWHztbQ9j0L1AmzNqbWfF/+sMnW5iQm3VQW3VIU1bSeOEVG0jdbr939Pploe069RJ0Om2ETtWL0+K95SnlwPWvO4Ap7ss1O0WsGWY7o1e0oBvYre2HrcLXVqkAgAWHD8dpxbOw3VJ0+E682bAE1PjcqmtujLromynQ23VQW3VIU1bSeOEVG3pdDsLadepk9DPWxOMXVuGeeHGcWsxNd+87oA53YJWL9d9bol/p6S7rSYqOtRuLRsCAJZtKgDgQrOU+FqXKelmyoRONzGhtuqgtuqQpq2kcUKqtjVxus37Ic7ploeUhYydiHOvSgV4bXC6UxJ8kcWKi6n5d4zl+3SXed9ML68xkgZ8ExXa9m7bCACw66BvOkM6nW6ty6yLsp0OtVUHtVWHNG0ljRNStWWk21lIu06dBJ1uGzEbu9tTc1kbJ8cBAA4bZU53iPRy83c8LjPSrZ/TLeVJG7cM85HTJQPnd86w3qen1t7plrQVjAm3DCMm1FYd1FYd0rSVNE5I1TbSLcMAOt2SodOtLzJ6DiFYq5fXIvLcKMnndB+znG7ffskBc7oZ6bYNSU/ZTVRoG+NxY/YVp+G8jukAgC4t0mpdJrVVV2ZdlO10qK06qK06pGkraZyQqi0j3c5C2nXqJGq+ShIJwo453bEeNxomxeJISfhId2lZgLs80q3fsxMpjV7SgG+iStu4GDeeG90HW/YeQYdmDWpdHrVVV2ZdlO10qK06qK06pGkrxU5Arra1cbqlzumWUkcqkHadOgn9vDXBmM8DXe7aPctonBznF+n2Od3+KUDlkW5908tVpECrwL9T0t1WE5Wp+263C6dkpMDtrn3ZdLrVlVkXZTsdaqsOaqsOadpKsleSrUDtnG7zPkNqpFvK/ZwKpE2DcBKsERuxI9INAE2T43GkkoXUjIpOt4YNS8rgRMdQHdRWXZl1UbbTobbqoLbqkKatJHsl2QowvdypSLtOnYR+3ppg7HK6AyPdIVYvN31ta59ufSPdujd6OobqoLbqyqyLsp0OtVUHtVWHNG0l2SvJVoBOt1ORdp06CTrdNuK1Vi+vnRPcpEEcjiK80x20kJqGc7q5erk6pKTuS1qV1oSrlxMTaqsOaqsOadpKsleSrQBXL3cq0ex0z549G1lZWUhISEB2djZWr15d6fmvvvoqOnbsiISEBHTr1g3vvvtuHVkaGhk9hxDsWL0cAJokx+Egkn1vju0HEOjEljvdpb5zNF69XPfBSeKcbikdKiPd6sqsi7KdDrVVB7VVhzRtJdkryVbAnjndUhdSk3I/pwIpQa9IWbhwIfLy8jB16lSsXbsWPXr0QE5ODvbs2RPy/M8++wwjRozAuHHjsG7dOgwZMgRDhgzBhg0b6tjycrh6eS154oknkJqaCgA4UexzgmvtdDeIx/dGQ9+bI76LKSDS7S2LqIPp5bWFjqE6qK26MuuibKdDbdVBbdUhTVtJ9kqyFbAnvXz27Nlo2rSpIgvt58iRwOxQJyLtOq0uM2bMwIQJEzB27FgAwJw5c7Bo0SLMnTsXd9xxR9D5TzzxBAYPHoxbb70VAHD//fdjyZIlmDVrFubMmVOntpvQ6a4l06dPDzqWlJRYqzIbJ8dhr1G2T/KRAgBAcnKy9a/3N99HHpe+6eX+9uqMy+VCYmIiCgsLkZCQUN/mVAsp2sbHx8Pj8VgvCTRo4NsqzU5tVZRZF2U7HWqrDil9mESkaZuUlARAhr3StK2OveH6ueTkZBw4cABPPvmkOgMVIqWOVCDtOj18+DAOHTpkvY+Pj0d8fHzAOUVFRVizZg0mT55sHXO73Rg0aBBWrlwZstyVK1ciLy8v4FhOTg7efPNN+4yPEDrdtWTkyJFWp4Xv30PPpHy0zGhWqzKbJMdhb4VI99NPP42vvvoKnTt3xspPtwEA3FZ6uX5O9w033ICUlBTriZTO/POf/8Thw4eRlpZW36ZUi9tvvx1t2rTB8OHD69uUSklKSsJLL72E2NhYxMTI6GqmTp2KHj164Pe//71tZQ4dOhS//PIL/vSnP9lWpklOTg4eeOABXHzxxbaX7XQGDx6MBx54wNZrgfhQ2SaczqhRo3Do0CGMHj26vk2pFunp6XjuueeQkZFR36ZUyTXXXAO3243x48fXtynV4uSTT8aTTz6JTp06hT2nT58+ePTRR9G/f/+A488//3y9Oie1oWvXrjjppJPq24x6o3Xr1nj66aeRlZVV36ZUi86dOwe8nzp1Ku65556AY/v27UNpaWlQP5GRkYHvvvsuZLn5+fkhz8/Pz6+90TXEZUhbJUETduzYgdatW+OXX35Bq1atfAdfugTY9jEw7AWg2x9rXPbm/MMY98Tr+DT+JsATD9xVAPilicx4fzOe/GAL/tXqTQzY9x9gwM3A+ffW8i8ihBBCCCGEELWYftTGjRvRsmVL63ioSPeuXbvQsmVLfPbZZwEPiG677TZ89NFH+Pzzz4PKj4uLw0svvYQRI0ZYx55++mnce++9KCgoUPAXVY2M8JMUzAUnapnuHZBeXloInDgIJDa0Pt9zuBAAkBxX5ohruJAaIYQQQgghhIQjJSXFWhsrHE2bNoXH4wlylgsKCpCZmRnyO5mZmRGdXxfol5csGcOe1cQbJcWiyBWHQ4ZvrpOZYm5ScOgEACA5tszp1nBONyGEEEIIIYTUhri4OPTu3RvLli2zjnm9XixbtixoaoRJ//79A84HgCVLloQ9vy6gt2Yn3jKnu5aricd43GiYGBu0mJpJwSFfpDvJcroZ6SaEEEIIIYREH3l5eXjuuefw0ksvYdOmTbjuuutw9OhRa+2o0aNHByy0dtNNN2Hx4sV47LHH8N133+Gee+7Bl19+idzc3Pr6E5hebis2RboB37Zhew80RHvsDnK69xz2RbqTYpheTgghhBBCCIlehg8fjr1792LKlCnIz89Hz549sXjxYmuxtO3btwfsz37GGWdg/vz5uOuuu/C3v/0NJ598Mt5880107dq1vv4EOt22YlOkGyib1/2bGekuTy8vLvVi35EiAECiWXtMLyeEEEIIIYREKbm5uWEj1cuXLw86dvnll+Pyyy9XbFX1obdmJ4Z9W3g1beC/bVh5pHtv2SJqsR4X4gt/9R1MkLHVFSGEEEIIIYQ4DTrddmKtXm5TpDuE020uopaekgDX/q2+g01PrvXvEUIIIYQQQgixHzrddmLjnO42jZOwF8ELqZmLqDVP8QC//eQ72KRDrX+PEEIIIYQQQoj90Om2ExvndHdtkeYX6S6f020uotY54TfAWwLEJAIpLWr9e4QQQgghhBBC7IdOt53YGOnu0iINe8qcbu/h4PTyU2PLjjXpYMscckIIIYQQQggh9kNvzU5sjHSnJcUiJi3TV9yxfUBpCYDy9PK22O07sUn7Wv8WIYQQQgghhBA10Om2E6NsITWbIs+tWrRCqeGCCwZwbB+A8kh389KdvpM4n5sQQgghhBBCtIVOt53YGOkGgC6tGuHXCoup7SmLdDc58bPvOFcuJ4QQQgghhBBtodNtJzbO6QaALi3TsNcoc7r3/QAAKChbSK3BkZ98xxnpJoQQQgghhBBtodNtJzZHuru2SMPH3u6+oj/4O04cP4oDx4qRhBOIOWoupMY53YQQQgghhBCiK3S67cTmSHezlHi8mvgnFBgN4f5tG3588wEAwCkxZQ53UlMgsZEtv0UIIYQQQgghxH5i6tsAqXi9vkXTdj8zDGiY4DtYcACAF9i9ByhMtuV3umcm4/Z1f8Df415Eg9Wz8E9jIWI8buwwvEByK2DHDlt+hxBCCCGEEEJUs3u3bxcm059yAi7DMIz6NkIiX3zxBfr161ffZhBCCCGEEEKIOFavXo2+ffvWtxl1Ap3uGlJSUoJ169YhIyMDbpu2CCO14/Dhw+jcuTM2btyIlJSU+jaHhIH1JAfWlQxYT3JgXcmA9SQH1pUMKtaT1+tFQUEBevXqhZgYZyRe0+kmUcOhQ4eQlpaGgwcPIjU1tb7NIWFgPcmBdSUD1pMcWFcyYD3JgXUlA9YTF1IjhBBCCCGEEEKUQaebEEIIIYQQQghRBJ1uEjXEx8dj6tSpiI+Pr29TSCWwnuTAupIB60kOrCsZsJ7kwLqSAeuJc7oJIYQQQgghhBBlMNJNCCGEEEIIIYQogk43IYQQQgghhBCiCDrdhBBCCCGEEEKIIuh0E0IIIYQQQgghiqDTTQghhBBCCCGEKIJON1HCtGnT0LdvX6SkpCA9PR1DhgzB5s2bA87Jz8/HqFGjkJmZieTkZJx22ml4/fXXKy33448/xiWXXIIWLVrA5XLhzTffrPT8a6+9Fi6XCzNnzqzS5uXLl+O0005DfHw8OnTogHnz5gWdM3v2bGRlZSEhIQHZ2dlYvXp1leXqjrS6+vTTTzFgwAA0adIEiYmJ6NixIx5//PGg86Ktruqznq688kq4XK6A1+DBg6u02YltSlo9ObU9AfXf923atAmXXnop0tLSkJycjL59+2L79u2Vlv3111/jrLPOQkJCAlq3bo1HHnkk6JxXX30VHTt2REJCArp164Z33323ciE0R1o9bd68Gb/73e+QkZGBhIQEtGvXDnfddReKi4sDzou2egLqt64q9n3m69FHH620bI5T+tdT1IxTBiEKyMnJMV588UVjw4YNxvr1642LLrrIaNOmjXHkyBHrnPPPP9/o27ev8fnnnxtbt2417r//fsPtdhtr164NW+67775r3HnnncYbb7xhADD++9//hj33jTfeMHr06GG0aNHCePzxxyu198cffzSSkpKMvLw8Y+PGjcZTTz1leDweY/HixdY5CxYsMOLi4oy5c+ca3377rTFhwgSjYcOGRkFBQbV10RFpdbV27Vpj/vz5xoYNG4xt27YZr7zyipGUlGT84x//sM6Jxrqqz3oaM2aMMXjwYGP37t3Wa//+/ZXa69Q2Ja2enNqeDKN+62rLli1G48aNjVtvvdVYu3atsWXLFuOtt96qVNODBw8aGRkZxsiRI40NGzYY//73v43ExMSAulqxYoXh8XiMRx55xNi4caNx1113GbGxscY333xTM5E0QFo9bd261Zg7d66xfv1646effjLeeustIz093Zg8ebJ1TjTWk2HUb13593u7d+825s6da7hcLmPr1q1hy+U4JaOeomWcotNN6oQ9e/YYAIyPPvrIOpacnGy8/PLLAec1btzYeO6556pVZmWO3I4dO4yWLVsaGzZsMNq2bVulI3fbbbcZXbp0CTg2fPhwIycnx3rfr18/Y+LEidb70tJSo0WLFsa0adOqZa8UdK+rUFx22WXGX/7yF+u9E+qqLutpzJgxxh/+8IeI7GOb8qF7PYXCie3JMOq2roYPHx6gcXV4+umnjUaNGhmFhYXWsdtvv9049dRTrfd/+tOfjIsvvjjge9nZ2cY111wT0W/pjO71FIpJkyYZZ555pvXeCfVkGHV/P+HPH/7wB+Pcc8+t9ByOUz50r6dQSBynmF5O6oSDBw8CABo3bmwdO+OMM7Bw4ULs378fXq8XCxYswIkTJ3DOOefU6re8Xi9GjRqFW2+9FV26dAl5zjnnnIMrr7zSer9y5UoMGjQo4JycnBysXLkSAFBUVIQ1a9YEnON2uzFo0CDrnGhB97qqyLp16/DZZ59h4MCBAJxTV3VZT4AvBS89PR2nnnoqrrvuOvz6668Bn7NNhUb3eqqIU9sTUHd15fV6sWjRIpxyyinIyclBeno6srOzg1Ixr7zyyoDfWblyJc4++2zExcVZx3JycrB582b89ttv1jmVtbtoQPd6qsiWLVuwePFiq00BzqgnoO77P5OCggIsWrQI48aNCzjOcSo0utdTRaSOU3S6iXK8Xi9uvvlmDBgwAF27drWO/+c//0FxcTGaNGmC+Ph4XHPNNfjvf/+LDh061Or3Hn74YcTExODGG28Me06bNm3QvHlz631+fj4yMjICzsnIyMChQ4dw/Phx7Nu3D6WlpSHPyc/Pr5W9OiGhrkxatWqF+Ph49OnTBxMnTsT48eMBwBF1Vdf1NHjwYLz88stYtmwZHn74YXz00Ue48MILUVpaap3DNhWMhHoycXJ7Auq2rvbs2YMjR47goYcewuDBg/H+++/jsssuw9ChQ/HRRx9Z5zVv3hxt2rSx3odrU+ZnlZ0TLXUloZ5MzjjjDCQkJODkk0/GWWedhfvuu8/6LNrrCaj7/s+fl156CSkpKRg6dGjAcY5TwUioJxPp41RMfRtAop+JEydiw4YN+PTTTwOO33333Thw4ACWLl2Kpk2b4s0338Sf/vQnfPLJJ+jWrVuNfmvNmjV44oknsHbtWrhcrrDnvfzyyzUqP9qRVFeffPIJjhw5glWrVuGOO+5Ahw4dMGLEiBrZIo26rCcA+POf/2z9v1u3bujevTvat2+P5cuX47zzzgPANhUKSfXk5PYE1G1deb1eAMAf/vAHTJo0CQDQs2dPfPbZZ5gzZ44VvZk2bVot/qLoRFI9LVy4EIcPH8ZXX32FW2+9FdOnT8dtt91WI1skUtf9nz9z587FyJEjkZCQEHCc41QwkupJ/DhV3/ntJLqZOHGi0apVK+PHH38MOL5lyxYDgLFhw4aA4+edd1615zQhxHyRxx9/3HC5XIbH47FeAAy32220bds2bFlnnXWWcdNNNwUcmzt3rpGammoYhmEUFhYaHo8n6PdGjx5tXHrppdWyV3ek1FUo7r//fuOUU04xDCP666qu6ykcTZs2NebMmRP2c6e3KSn1FAontSfDqPu6KiwsNGJiYoz7778/4Phtt91mnHHGGWHLGjVqVNCc/Q8++MAAYC2Y17p166B1MaZMmWJ07969WvbqjJR6CsUrr7xiJCYmGiUlJYZhRHc9GUb99n8ff/yxAcBYv359lWVxnJJRT6GQOE4xvZwowTAM5Obm4r///S8++OADnHTSSQGfHzt2DIBvzoU/Ho/HerpcE0aNGoWvv/4a69evt14tWrTArbfeivfeey/s9/r3749ly5YFHFuyZAn69+8PAIiLi0Pv3r0DzvF6vVi2bJl1jlSk1VUovF4vCgsLAURvXdVXPYVix44d+PXXX0Omf5k4tU1Jq6dQOKE9AfVXV3Fxcejbt2/QFj3ff/892rZtG/Z7/fv3x8cffxyw9dSSJUtw6qmnolGjRtY5lbU7iUirp1B4vV4UFxdb9kRjPQF69H8vvPACevfujR49elR5LscpGfUUCpHjVP35+ySaue6664y0tDRj+fLlAVsDHDt2zDAMwygqKjI6dOhgnHXWWcbnn39ubNmyxZg+fbrhcrmMRYsWhS338OHDxrp164x169YZAIwZM2YY69atM37++eew3wm1IvaoUaOMO+64w3pvbhtx6623Gps2bTJmz54dctuI+Ph4Y968ecbGjRuNq6++2mjYsKGRn59fQ5X0QFpdzZo1y3j77beN77//3vj++++N559/3khJSTHuvPNO65xorKv6qqfDhw8bf/3rX42VK1ca27ZtM5YuXWqcdtppxsknn2ycOHHCKodtyoe0enJqezKM+u373njjDSM2NtZ49tlnjR9++MHaquiTTz6xzrnjjjuMUaNGWe8PHDhgZGRkGKNGjTI2bNhgLFiwIGjbnBUrVhgxMTHG9OnTjU2bNhlTp04VvxWVtHr65z//aSxcuNDYuHGjsXXrVmPhwoVGixYtjJEjR1rnRGM9GUb9308cPHjQSEpKMp555pmQ5XCc8iGtnqJlnKLTTZQAIOTrxRdftM75/vvvjaFDhxrp6elGUlKS0b1796DtCSry4Ycfhix3zJgxYb8TypEbOHBg0Hc+/PBDo2fPnkZcXJzRrl27AFtNnnrqKaNNmzZGXFyc0a9fP2PVqlVVKKE/0urqySefNLp06WIkJSUZqampRq9evYynn37aKC0tDfhetNVVfdXTsWPHjAsuuMBo1qyZERsba7Rt29aYMGFC0EDGNuVDWj05tT0ZRv33fS+88ILRoUMHIyEhwejRo4fx5ptvBnw+ZswYY+DAgQHHvvrqK+PMM8804uPjjZYtWxoPPfRQ0O//5z//MU455RQjLi7O6NKlS6U3yRKQVk8LFiwwTjvtNKNBgwZGcnKy0blzZ+PBBx80jh8/HvC9aKsnw6j/uvrHP/5hJCYmGgcOHAhZDscpH9LqKVrGKZdhGEbV8XBCCCGEEEIIIYRECud0E0IIIYQQQgghiqDTTQghhBBCCCGEKIJONyGEEEIIIYQQogg63YQQQgghhBBCiCLodBNCCCGEEEIIIYqg000IIYQQQgghhCiCTjchhBBCCCGEEKIIOt2EEEIIIYQQQogi6HQTQgghhBBCCCGKoNNNCCGEEEIIIYQogk43IYQQQgghhBCiCDrdhBBCCCGEEEKIIuh0E0IIIYQQQgghiqDTTQghhBBCCCGEKIJONyGEEEIIIYQQogg63YQQQgghhBBCiCLodBNCCCGEEEIIIYqg000IIYQQQgghhCiCTjchhBAijK1bt+Kaa65Bu3btkJCQgNTUVAwYMABPPPEEjh8/bp1XXFyMJ598En379kVKSgoaNGiAvn374sknn0RxcXE9/gWEEEKIc3AZhmHUtxGEEEIIqR6LFi3C5Zdfjvj4eIwePRpdu3ZFUVERPv30U7z++uu48sor8eyzz+Lo0aO4+OKL8dFHH+H3v/89Bg8eDLfbjcWLF+Ptt9/GwIEDsWjRIiQnJ9f3n0QIIYRENXS6CSGEECFs27YN3bt3R6tWrfDBBx+gefPmAZ9v2bIFixYtwk033YRrrrkGzz77LJ566ink5uYGnDd79mzk5ubi2muvxTPPPFOXfwIhhBDiOOh0E0IIIUK47rrrMGfOHKxYsQJnnHFG2PN27NiBrKwsDBw4EMuWLQt5zrnnnotPPvkE27ZtQ6tWrVSZTAghhDgezukmhBBChPC///0P7dq1q9ThBoD/+7//Q2lpKUaPHh32nNGjR6OkpASLFy+220xCCCGE+EGnmxBCCBHAoUOHsHPnTnTr1q3Kczdu3AgA6NGjR9hzzM82bdpkj4GEEEIICQmdbkIIIUQAhw4dAgCkpKRUee7hw4erPNf8zCyXEEIIIWqg000IIYQIIDU1FUC5Q10ZpkNd2bnVccwJIYQQUnvodBNCCCECSE1NRYsWLbBhw4Yqz+3UqRMA4Ouvvw57jvlZ586d7TGQEEIIISGh000IIYQI4fe//z22bt2KlStXVnrehRdeCI/Hg1deeSXsOS+//DJiYmIwePBgu80khBBCiB90ugkhhBAh3HbbbUhOTsb48eNRUFAQ9PnWrVvxxBNPoHXr1hg7diyWLl0ach/uOXPm4IMPPsC4ceO4XRghhBCiGO7TTQghhAji7bffxvDhw5GYmIjRo0eja9euKCoqwmeffYZXX30VV155Jf7xj3/gyJEjuPDCC/Hpp5/i0ksvtSLa7733Ht566y0MHDgQixYtQnJycj3/RYQQQkh0Q6ebEEIIEcYPP/yARx99FEuWLMGuXbsQHx+P7t27489//jMmTJiA+Ph4AEBRURGefvpp/POf/8R3330HwzDQsWNHjB49Gtdffz1iY2Pr+S8hhBBCoh863YQQQgghhBBCiCI4p5sQQgghhBBCCFEEnW5CCCGEEEIIIUQRdLoJIYQQQgghhBBF0OkmhBBCCCGEEEIUQaebEEIIIYQQQghRREx9GyCVkpISrFu3DhkZGXC7+eyCEEIIIYQQQqrC6/WioKAAvXr1QkyMM9xRZ/yVCli3bh369etX32YQQgghhBBCiDhWr16Nvn371rcZdQKd7hqSkZEBwHexNG/e3PbyZy79HgtW/4L+7RvjvI6+39p14DjmrvgJmWkJeHPiANt/kxBCCCGEEEJUsnv3bvTr18/yp5wAne4aYqaUN2/eHK1atbK9/NQmhxCTehx9urTHtYM7AgC++uUAXv7mCOLTEpX8JiGEEEIIIYTUBU6aouucv1QYXsMAALhd5cfcLt8bo+wzQgghhBBCCCF6Q6dbc1wo97rLfG546XMTQgghhBBCiAjodGtKZZFuLyPdhBBCCCGEECICzulWTGlpKYqLiyP+XrLHi5YpHiR6vDhx4kRZYUVomeJBwyR3+TFSp8TGxsLj8dS3GYQQQgghhBAh0OlWhGEYyM/Px4EDB2r0/bObGzitcTpSE4uxbds2AEBJqRf3/C4dHhesY6TuadiwITIzM+Fyuao+mRBCCCGEEOJo6HQrwnS409PTkZSUFLGDln/wOA4eL0aTBvFo2iAeAFBYXArj16PwuN04Kb2BCrNJJRiGgWPHjmHPnj0AoGSrOEIIIYQQQkh0QadbAaWlpZbD3aRJkxqVEXPcC1exC7Fx8UhISPAd9JTCFVMMl9tVfozUKYmJiQCAPXv2ID09nanmhBBCCCGEkErhQmoKMOdwJyUl1byQsrXS/OPjTGbWA7NeazJXnxBCCCGEEOIs6HQrRNmcXy5eXq9wLjchhBBCCCGkutDp1hTLrw4R6qbPTQghhBBCCCEyoNMtCMZXCSGEEEIIIUQWdLo1R7qj/dNPP8HlcmH9+vX1bQohhBBCCCGE1Dl0ujWlPIU8OL+c6eWEEEIIIYQQIgM63YKo66h3UVFRHf8iIYQQQgghhEQXdLrrAMMwcKyoJKLX8cJSnCguxYmi0vLjxSXBx6p4GUb14+LnnHMOcnNzcfPNN6Np06bIycnBhg0bcOGFF6JBgwbIyMjAqFGjsG/fPus7ixcvxplnnomGDRuiSZMm+P3vf4+tW7eqkJEQQgghhBDiMD7++GNccsklaNGiBVwuF958880qv7N8+XKcdtppiI+PR4cOHTBv3jzldlZGTL3+ukM4XlyKzlPeq5ff3nhfDpLiql/NL730Eq677jqsWLECBw4cwLnnnovx48fj8ccfx/Hjx3H77bfjT3/6Ez744AMAwNGjR5GXl4fu3bvjyJEjmDJlCi677DKsX78ebjef6RBCCCGEEEJqztGjR9GjRw9cddVVGDp0aJXnb9u2DRdffDGuvfZa/Otf/8KyZcswfvx4NG/eHDk5OXVgcQgMUiN++eUXA4Dxyy+/BBw/ceKE0aNHD8Pj8QS84HLX+OVJbWa0vH6e0frm/xgxjVtF9t0KdlT2gm+6uPXe7XYbLpcr5HlVlTN58mTDMAzjoYceqvL8+ny1bdvW2Llzp1FSUmIMGDCg3u2p6hUXF2c8+OCDhmEYxi233GLExMTUu03hXieffLLx22+/GQUFBcZJJ51U7/ZU9UpJSTFef/11wzAMY8KECbaUGRMTY+Tm5hqGYRjz5883kpOTbbV51KhRhmEYxv/+9z8jNTW13jWMphe1VfNS3Sac/IqNjTXuvPNOwzAM4x//+IeRmJhY7zZV9eratatx7NgxY9u2bUarVq3q3Z5wr7i4OOPhhx82DMMwHn30USM+Pr7ebarq1a9fP6O4uNjYsGGDkZmZGfKc888/3/B6vcbKlSuNJk2aGB6Px2jfvr3x66+/qryFVorX6zUuvfTSete/vl4XXnih4fV6jU8++cRo3LhxvdsT7uV2uw0AxsaNG42DBw9arxMnTlRZxwCM//73v5Wec9tttxldunQJODZ8+HAjJyenNpdXraDTXUPCOd3ffPON5bza+Zq/8D/Gkg8/UlK2ilePHj0MwzCMQYMG1bstVb3eeOMNqz4lvPr27WsYhmFkZWXVuy1VvT744ANj0aJF9W5HdV/jxo0zDMMwGjZsaFuZmZmZhmEYxsiRI223NyEhwTAMw5g4cWK9axdtL2qr7qWyTTj91bFjR8MwDOPiiy+ud1uq+1qzZo0xf/78erejqtfpp59uGIZhDBgwoN5tqe5r69atxtNPP13pOb/99pvx97//PeDY+++/r/AOWi0nTpyod93r+3X48GHjnnvuqXc7avKaOnVqlXUMVO10n3XWWcZNN90UcGzu3LlGampqzS+uWsL0cpvxer0AgEaNGuHLL79EQkJCjcrZsf8YDp0oxu3XjcHaL79AnMeNWLdvKbUWrVpj9aqVcLnsXVpt2LBh6NKlC+677z4AwMiRI5GYmIg777wz6NyMjAwkJSXh7LPPRsuWLXH99dcjMzMTS5cuxd///ndLB/PfSZMm4a9//aut9taWIUOG4IsvvoBhGJadcXFx2LZtW6XfO3HiBH755Re0bt26xvVbE5YuXYoxY8ZY8/RNmxctWoSePXvWmR3V4Xe/+x2+//77AG179OiBd999t54tC81TTz2Fhx56KEjbTz75BO3atatRmRs3bsT5558fVOY999yDCRMm1MrenTt3ol+/fkFl69jOpLFjxw5kZ2cHaZuXl4dbbrmlPk0Tz6ZNmzBo0CAlbcLprFq1CsOGDQvSdsaMGRg+fHh9mhaWPn36YPfu3QHjxJlnnomFCxfWs2WBvP/++xg7dmyQti+88AIGDx5cn6aF5ZRTTsHRo0cDtL344ovx7LPPAgCKi4uRlZUFAAHnmBgRrAekG/5/y+bNm9GgQYN6tKbuOHHiBNq3bw8gsE5Hjx6NadOm1adpIdm9ezf69OmDjRs3omXLltbx+Ph4W8rPz89HRkZGwLGMjAwcOnQIx48fR2Jioi2/Ewl0um3G7Kg8Hg9atGhRY6esKO4oEk4UIy4uzirXgK/smNg4NG/RAm6bne74+Hg0aNAALVq0AACcccYZeP3115GdnY2YmOBL5ddff8XWrVvx4osv4qyzzgIAfPjhh5a9/qSmplrl6oLZsA1fxgcAwO12V2nniRMnUFhYWKv6rQlNmzYFUK6t+W96erp22gZct2V2xsfHa2enSVpaGoBgbTMzM2tss7ngYMUyGzZsWGsdSktLQ5atYzuTRklJCQBqq4Jff/0VgJo24XSaNWsGIFjbxo0ba6uteV/hP04kJiZqZ2+TJk0ABGvbpEkT7Ww1MdfU8dc2KSnJstfs5yqe439MKv62t2jRwlFOt4l/naakpGh5nZoPBVJSUpCamlrP1tQNXOnKZsyL3K4otFmOYRiAzWVXxcSJE7F//36MGDECX3zxBbZu3Yr33nsPY8eORWlpKRo1aoQmTZrg2WefxZYtW/DBBx/glVdeCSjDbj3sxF9bne00CbgWQG3tRIW2KutL0rUgDWqrDmqrDonaShknolFb//9Hs9Otcx3ZTbg6dZIG/mRmZqKgoCDgWEFBAVJTU+slyg3Q6bYduy9yd0DHifKy66A/bNGiBVasWIHS0lJccMEF6NatG26++WY0bNgQbrcbbrcbCxYswJo1a9C1a1dMmjQJo0aNsuz1/1fHRi9lwDeRNPBTWzrdUqG26qC26pCorZRxIhq1pdMdfdDpDqR///5YtmxZwLElS5agf//+9WQR08ttx/ZId1mKkNfrVd6Ali9fHnTs5JNPxhtvvBH2O4MGDcLGjRut96+++iqA8nQsM4XbnDukE/6DkpnmovM2Z5IGfik3UyZ0uokJtVUHtVWHRG2ljBPRqG1VTnfFOd6S8Ldd53s6u/H/W+vCZ6hrjhw5gi1btljvt23bhvXr16Nx48Zo06YNJk+ejJ07d+Lll18GAFx77bWYNWsWbrvtNlx11VX44IMP8J///AeLFi2qrz+BkW67sesiN7u/cOnluj6DlDQ4SRnwTaitOuh0ExNqqw5qqw6J2koZJ6JRW0a6o49oj3R/+eWX6NWrF3r16gXAt7hpr169MGXKFAC+hdm2b99unX/SSSdh0aJFWLJkCXr06IHHHnsMzz//fP3t0Q1Gum3H7o4qoOM0XW2XCz63XL+GJGlwkjLgm0jUVsrTVn97ATVOt1m2nU63nfYSHyrrzelQW3VI1DZUtpmO9krWtjpjsP85JnS65RHtTvc555xT6XU5b968kN9Zt26dQqsig5Fum7E7TTmkY4g6mdJdI9x+6fCA3s6Auw5T9+0g3MCvY/qU/8qpOttp4m8vYI+24dqCHTpULEOCxlKoeC3YWW9OR2WbcDoStQ01Butor2RtKxuDQ51jQqdbHtHudEcD+vYYQrEtvbxCORXTy3X1uiVGY6V0TtRWHVLTy81yJWgsBWqrDkl9mDQkaitlnIhWbUOdY0KnWx4cu/SHTrfN2H2RMxqrjlCpbTraaSJp4JdyM2VCp5uYUFt1SOrDpCFRWynjRLRqW5nTzYXUZCJtap/TcNbVWAfYfZFLGZRMJA1O1FYd1JZOt1SorTok9WHSkKitlHEiWrVlpDv6kNKmnAqdbpuxu6Oq2Nmbx3TtDiUNTtI6J2qrDjrdxITaqkNSHyYNidpKGSeiVVs63dGHlDblVOh024xdF7nZZYTsFDVuQJIGJ2mdE7VVB51uYkJt1SGpD5OGRG2ljBPRqq0TnG6nIaVNORU63TZj99zgsHO6Ne1TKq7yqfNcaX9bdbbTxH+ujv+/OtosTVsVq9OqbAv+ZUjRWArUVh2SxgdpSNRWyjgRrdqGOsckGuZ061w/qpC2DpTTcN4VqRjbLnIrqB1isS+XW1efu9pPhLOysjBz5syA77355pu1/u1IypD2RDDcVkY62ixN23BP/Bnpdh7UVh0SI4ZSkKitlHEiWrWN9ki3zvWjipA+gwMfPuhKTH0bEG3URXq5zh1JTQen3bt3o1GjRtX6jXvuuQdvvvkm1q9fX+MyKtoazdrWB5K1pdPtbKitOiT1YdKQqK2UcSJataXTHX1IaVNOhU63zdjdUYWf013/HWJRURHi4uICjtV0cMrMzKy1PZGWIa1zkjTwS9aWTrezobbqkNSHSUOitlLGiWjVlk539CGlTTkV5hzYTMiL3DCAoqMRvVzFx+AqPga3UTaHqOg4jKLjvrJhwKhuWRF0nOeccw5yc3ORm5uLtLQ0NG3aFHfffbf1N2VlZeH+++/H6NGjkZqaiquvvhoA8Omnn+Kss85CYmIi/vKXvwAASktLfXaXpbc88MADSExMxEknnYR//etfQb9dMTV8x44dGDFiBBo3bozk5GT06dMHn3/+OebNm4d7770XX331FVwuF1wuF+bNmxeyjG+++QbnnnsuEhMT0aRJE1x99dU4cuSI9bkZKX/vvfdw/vnnAwAOHjyI4uLiamtWl0ga+KV1/HS6iQm1VYekPkwaErWVMk5Eq7aVOd3RMKdb5/pRhf/aPxKuU6fBSLfNhLzIi48BD7aIqJz2Zf/G/XjMV+7/3QEj1Vdm8r6vEf9I6+oV9LddQFxytX/3pZdewrhx47B69Wp8+eWXuPrqq9GmTRtMmDABADB9+nRMmTIFU6dOBQBs3boVgwcPxt///nfMnTsXixYtwqRJk7Bjxw6f3WV67Nu3Dx9++CFiY2Nx4403Ys+ePWFtOHLkCAYOHIiWLVvi7bffRmZmJtauXQuv14vhw4djw4YNWLx4MZYuXQoASEtLCyrj6NGjyMnJQf/+/fHFF19gz549GD9+PHJzcy0n3WTPnj2YM2cOLr/8chw7dgzz5s2z/l6dkDTwS7mZMqHTTUyorTok9WHSkKitlHEiWrWN9ki3E+cy+6/9I+E6dRp0um3G7ovcLMVAedBaZftp3bo1Hn/8cbhcLpx66qn45ptv8Pjjj1tO6LnnnotbbrnFOn/8+PEYOXIkbr75ZgBA165dAQC//fYbTpw4gePHfdH5G264AaeffjoA4IUXXkCnTp3C2jB//nzs3bsXX3zxBRo3bgwA6NChg/V5gwYNEBMTU2k6+fz583HixAm8/PLLSE72PXSYNWsWLrnkEjz88MPIyMiwzv3zn/+Mtm3bAgASExOxbNkyOt21RNrT1lD2+h+vTZlmeXY+ffcvQ4rGUqjodDs5amI3KtuE05Gorb/NOtsrWdvKxodw4x4QHU63zvWjCikPspwKnW6bCblaYGySL+IcAVv2HMHx4lKUrL8e2PQWjAsegNGiBbBwBI6n98SJWz9AQqyn6oJikyL63dNPPz2ggfbv3x+PPfaYlS7ep0+fgPO/+uorfP3111bKuHkeAGzbts1yuk8++WTreMeOHdGwYcOwNqxfvx69evWyHO6asGnTJvTo0cNyuAFgwIAB8Hq92Lx5MzIyMoJurgHA4/FUGoWvT8IN/Do+zfV/2qqznSah7PU/Xpsy7dyGLJRdUjSWQsUtw5wcNbEblW3C6UjUNtT2RjraK1nbysaHcOOeeUwqTnY26XTrDZ1umwnZUblcEaV4A4AR64WBUrg9vioyPHEwYuJ9xbk9vvKq43TbjL8TC/hSwa+55hrceOONAIBVq1Zh1KhR6NChA9q3b1+jRp+YmGifwZUQrnPSdS6TxEi3lI5fZXq5CdPLZUBt1aGyTTgdSeODiZRxIlq1jfb0cp3rRxVS2pRT0fcxnVDsusjN7s4V4kmwyvzyzz//POD9qlWrcPLJJ8PjCe3gn3baadi4cSM6dOiADh06oFWrVgCA2NhYxMXFIT7e96Dgxx9/tL6zefNmHDhwIKwN3bt3x/r167F///6Qn8fFxQVE1EPRqVMnfPXVVzh69Kh1bMWKFXC73Tj11FMDztU9tc1E0sAvreNX7XTbrQMdQ3VQW3VQW3VIGh9MpIwT0apttC+kpnMmgipCZY/ofJ06DeddkYqxfU53qI4TLmUbhm3fvh15eXnYvHkz/v3vf+Opp57CTTfdFPb822+/HZ999hlyc3Oxfv167Ny5EwCQn58PANaWYk8++SQ+//xzrFmzBuPHj680mj1ixAhkZmZiyJAhWLFiBX788Ue8/vrrWLlyJQDfKurbtm3D+vXrsW/fPhQWFgaVMXLkSCQkJGDMmDHYsGEDPvzwQ9xwww0YNWpUwHxuIPSAoyOSBn4pN1MmdLqJCbVVB7VVh6TxwUTKOBGt2jLSHX1IaVNOhU63zdh9kbvDNSBF/eHo0aNx/Phx9OvXDxMnTsRNN91kbQ0Wiu7du+Ojjz7C999/j7POOgvjx48HACsybtrcpEkTDBw4EEOHDsXVV1+N9PT0sGXGxcXh/fffR3p6Oi666CJ069YNDz30kFXmsGHDMHjwYPzud79Ds2bN8O9//zuojKSkJLz33nvYv38/+vbtiz/+8Y8477zzMGvWLOucygYcHZE08Evr+Ol0ExNqqw5qqw5J44OJlHEiWrWl0x19SGlTToVzum2mTiLdLhdUed2xsbGYOXMmnnnmmaDPfvrpp5Df6du3L95//30AwCeffIKzzz7bWgTNtPnvf/+7tRc2AIwaNSqgjIodfNu2bfHaa6+F/L34+PiQn1Uso1u3bvjggw9ClgEA2dnZ+OmnnwK0TU9Px/Lly8N+pz6RNPBL6/jpdBMTaqsOaqsOSeODiZRxIlq1pdMdfUhpU06FkW6bsWtusNnfSZufYdpm6qDz3Bp/W3W206SitjpfD/626mynSSh7/Y/XpkzA/nUD/MuQorEUVNab06G26qjoGErQ1t9mne2VrG1NtwzjnG6ZSPMZnEa9X5GzZ89GVlYWEhISkJ2djdWrV4c999tvv8WwYcOQlZUFl8uFmTNnBp1jflbxNXHiROucc845J+jza6+91pa/R2Wk2+oEXermdNcW/y0o/P/VsdH726qznSbhnrbrOLBwyzC1W0+Fc1501lgK4R5oUNvaw+3Y1CF5WyvdrwXJ2lY2PnDLsOhDyoMsp1Kv6eULFy5EXl4e5syZg+zsbMycORM5OTnYvHlzyDm/x44dQ7t27XD55Zdj0qRJIcv84osvAla23rBhA84//3xcfvnlAedNmDAB9913n/U+KSmy/azDURfp5ao6ejvSqiWlYUlLwwn1kADQ02Zp2kpLLzfLMcuVoLEUmAKtDmqrDkljr4mUcSJatWV6efRRlz4DiZx6dbpnzJiBCRMmYOzYsQCAOXPmYNGiRZg7dy7uuOOOoPP79u2Lvn37AkDIzwGgWbNmAe8feughtG/fHgMHDgw4npSUhMzMzGrbWlhYGLBK9uHDh0OeZ19jDyynYsepa38oaXCSMuCbqHAMVUFt6XRLhtqqgU63OiSNvSZSxolo1ZZOd/QhpU05lXp7/FFUVIQ1a9Zg0KBB5ca43Rg0aJC1NZQdv/HPf/4TV111VdBF969//QtNmzZF165dMXnyZBw7dqzSsqZNm4a0tDTr1blz55Dn2XWRm92dtAYkaXCSrC2dbnuR6nSrKtvpUFs10OlWh6Sx10RKO4tWbel0Rx9S2pRTqbdI9759+1BaWhq0Z3JGRga+++47W37jzTffxIEDB3DllVcGHL/iiivQtm1btGjRAl9//TVuv/12bN68GW+88UbYsiZPnoy8vDzr/c6dO0M63nZf5KHmPAH6NqBwC47omN4Sau6Ljnaa0OlWB51u4g+1VQOdbnVEq2OoA9GqbWVONxdSkwkXUtObqN4y7IUXXsCFF16IFi1aBBz333e6W7duaN68Oc477zxs3boV7du3D1lWfHw84uPjrfeHDh0KeZ5tF3lZ/ydlUDKRNDhJ1laK0y2l4w9lr//x2pQJqFnURJrGkgj1QI7a1h7VbcLJhHvgrbO2UtqZZG1runo5I90ykXZf6zTq7TFQ06ZN4fF4UFBQEHC8oKAgornW4fj555+xdOlSjB8/vspzs7OzAQBbtmyp9e/afmPtDjOn25bS7SfcKp86NnppTwRD3aAAej7NDbXom452mqhYpM7/71WhgzSNJUFt1aC6TTiZcDuH6KytlHYWrdqGG/fMY1KRcD+nCjrdelNvPUZcXBx69+6NZcuWWce8Xi+WLVuG/v3717r8F198Eenp6bj44ourPHf9+vUAgObNm9f6d22f041QDQiAph0iI93qkBjp1j2CYaLCXv/vq3iwI01jSTCLQA2q24ST8b9mAb3HXhMp7UyytpWND+Ee5JvHpCKhflQh7b7WadRrenleXh7GjBmDPn36oF+/fpg5cyaOHj1qrWY+evRotGzZEtOmTQPgWxht48aN1v937tyJ9evXo0GDBujQoYNVrtfrxYsvvogxY8YgJibwT9y6dSvmz5+Piy66CE2aNMHXX3+NSZMm4eyzz0b37t1r/TfVxZxunRuQbnO6XS4X/vvf/2LIkCEhPwM4p1sF0jp+FfZyTrdcqK0aOKdbHRU1lKCtlHYmKZhgwjndet/PqUKaz+A06tXpHj58OPbu3YspU6YgPz8fPXv2xOLFi63F1bZv3x7QaHbt2oVevXpZ76dPn47p06dj4MCBAXtML126FNu3b8dVV10V9JtxcXFYunSp5eC3bt0aw4YNw1133WXL31RXN9a6PoPUbXDavXs3GjVqFPIzKQO+CZ1uddDpJv5QWzXQ6VaHRG2ltDPd7muqQ22dbka6ZSKlTTmVel9ILTc3F7m5uSE/83ekASArK6taHcEFF1wQ9rzWrVvjo48+itjO6mJ7erkrxJwbjRtQXQ5ORUVFiIuLq/ScytYHkNY50elWB51u4g+1VYNEx1AKErWV0s7odMtCQv2oQkqbcirOy71QTKiOyjAMHD16NKLXsaNHcezYUZSUlgAACo8ewokDewD40kaqW06kHedrr72Gbt26ITExEU2aNMGgQYNw9OhRAMDzzz+PTp06ISEhAR07dsTTTz9tfe+nn36Cy+Wy5ujn5+ejR48eOHHiBABfo//5559xySWXoFGjRkhOTkaXLl3w7rvvWmV89NFH6NevH+Lj49G8eXPccccdKCkpsT4/55xzkJubi5tvvhlNmzZFTk5OlX+Py+XCm2++GWDjG2+8gd/97neYO3cuAF9GhYTOiU63Ouh0E3+orRokOoZSkKitlHZGp1sWEupHFVLalFOp90h3tBHqIj927BgaNGhQq3Ifmv649f8VHy5F86ahU6YrcuTIESQnJ1fr3N27d2PEiBF45JFHcNlll+Hw4cP45JNPYBgG/vWvf2HKlCmYNWsWevXqhXXr1mHChAlITk7GmDFjrDKeeuopAECTJk1wyimn4NtvvwXg02PixIkoKirCxx9/jOTkZGzcuNHSZefOnbjoootw5ZVX4uWXX8Z3332HCRMmICEhAffcc49V/ksvvYTrrrsOK1asqNbfFIo777wT06dPR0ZGBhYuXIj//Oc/uPDCCy07dcW0zc5trVQRylYd7TRRYW/Fm2BuGSYH/xsXLlJnH6rbhJORqK2UdlbR6dbZVpPqjA/h7inMY1KRUD+q4H2B3tDpthnJCzjs3r0bJSUlGDp0KNq2bQvAt485AEydOhWPPfYYhg4dCgA46aSTsHHjRvzjH/8IcLrHjh2LqVOnwuVy4d5778Vrr70GwKfH9u3bMWzYMKvMdu3aWd97+umn0bp1a8yaNQsulwsdO3bErl27cPvtt2PKlCmWnieffDIeeeSRWv2df/3rX3HxxRdj0aJFAICDBw9i165dlp26EuoGBdDTZv+tSCS0CRX2+n/ffwBUsWWYBI0lEWoxGmpbe/xv/qitvajub1QgpZ2F2wpVR1tNqjM+hDrHJBoi3TrXjypCbQNHp1sf6HTbTKiLPCkpCUeOHImonI27D6LUa+DFGX/Hs/+Yg79eNwatm6fjpimP4uxBOVjw7wVITYytspykpKRq/2aPHj1w3nnnoVu3bsjJycEFF1yAP/7xj4iLi8PWrVsxbtw4TJgwwTq/pKQEaWlpAWWceuqpAHw6+G/B5nK5cOONN+K6667D+++/j0GDBmHYsGHWivGbNm1C//79A3QbMGAAjhw5gh07dqBNmzYAgN69e1f77wmH+Zv+v3XgwIGgY7qhYi9pVUhLcWJ6OfGH2qpBYgq0FCRqK6WdMb1cFhLqRxWhgjNOfPigK3S6bSZUY3e5XNVO8TZJSipBqddAbNlCYbExMYiL9TnZMTGxSEpORnI1nO5I8Hg8WLJkCT777DO8//77eOqpp3DnnXfif//7HwDgueeeQ3Z2dtB3/Ikts9EwjCANxo8fj5ycHCxatAjvv/8+pk2bhsceeww33HBDtW2MVMdQmDZWjLpUPKYbnNOtDtX20umWBbVVh8vlsnSltvZBp1sddLplIaF+VCGlTTkVPv6wGbs7KrervNy6aEAulwsDBgzAvffei3Xr1iEuLg4rVqxAixYt8OOPP6JDhw4Br5NOOino+6a9oY63bt0a1157Ld544w3ccssteO655wAAnTp1wsqVKwO+t2LFCqSkpKBVq1bK/lYTCZ0TnW51qLJXpQ7SNJYEtVUHtVUDnW510OmWhYT6UYWUNuVUGOm2Gdsu8rL+LuxG9wo6xM8//xzLli3DBRdcgPT0dHz++efYu3cvOnXqhHvvvRc33ngj0tLSMHjwYBQWFuLLL7/Eb7/9hry8PKuMcE632+3GzTffjAsvvBCnnHIKfvvtN3z44Yfo1KkTAOD666/HzJkzccMNNyA3NxebN2/G1KlTkZeXpyw1JpTTrXMaDp1udah0ulVF9aRpLAlqqw5qqwY63epwotMdDQup6Xw/p4qwPgPRAjrdNmPXRW52f+WdohcGystW8QwyNTUVH3/8MWbOnIlDhw6hbdu2eOyxx6yVvZOSkvDoo4/i1ltvRXJyMrp164abb745oIzKIt2lpaWYOHEiduzYgdTUVAwePBiPP+5blb1ly5Z49913ceutt6JHjx5o3Lgxxo0bh7vuukvBXxpoq7+9OndOEp1uKR2/KntDza+y2+mWorEkVNab06G2aqg4nknQVsq1UPG+RmdbTaozPoQ7J5QTLgknj4dSHmQ5FTrdNqPqxrouGlCnTp2wePHisJ9fccUVuOKKK0J+lpWVBcMw8MMPPwDw6dCwYUM0bdoU+/btg8vlsrYTC8fAgQOxevXqsJ8vX7686j+iAv4Dh2mjianjHXfcgc6dOwcc0xGJTreUjp/p5cQfaqsOaqsGRrrV4aRIt9vtRmlpKZ1uoUhpU07FebkXirE7Tbm8AflnlOvbgPxX2Pb/V8dGL21rhVBRAf/jOiFtOytV9qrcEkeaxpKQspWRRKitGrhlmDqctGVYxXs4iUi4n1MFnW690bfHEIpdHZVZitvtWx08MP3Hlp9QQrg0LBWD07/+9S80aNAg5KtLly4R2SrBaZHUmUqyFWCkmwRCbdVBbdXASLc6nBTpNnek4ZxumXBOt94wvdxm6urGWtdnkHU5OF166aVBW5iZmNuCVYaUAd9Ekr2SbAXodJNAqK06qK0a6HSro6JNOttqUlunm5FumUhpU06FTrfN2HaRl/V3YRuQpv1hXTrdKSkpSElJqfH3pXVOFW+qKh7TCana0ukmALVVCbVVA51udUSrtuHmdJvHpCKhflQhpU05FeflXijG/yKvXXpOYGMxKpSta3coKQ2rpp1TfaVd+dum++qp0jp+Ot3EH2qrDmqrhmh1DHUg1APvisd1g0633vWjCiltyqkw0m0z/hf5rl270KxZM8TFxUV80XtLigAApaWlAICiEi+KSg3zR1BUeAInPPrNuSksLATg0+HEiROWHkVFRThx4kR9mhaEqW1xcTGKinx6m3aHwjAMFBUVYe/evXC73YiLi6szWwGZTreUeUWq7PUfALllmBxU1pvTobZqqOgYStBWyrUQauyteFw3qjM+hDqHTrdseF+gN3S6bcbskFNSUhAbG4tdu3bVqJw9vx0HABw+fAgAcOjIMfx64AgAoPDEcezY/jN+i9ev+vLz8wH4HNpt27ahpKQEALBz507tFrU4dMin7YEDB1BQUAAAOHHiBLZt21bp95KSktCmTZs6/3v8O07zgYFumpqEWhleV1sBdfaq1EGaxpKgtuqgtmrw11CKtlKuBX+bzLG34nHdqI62oc7hQmqykbYrj9PQz2sTjn/H1aZNG5SUlAR00tXB6zUw/o2PAACnp6QBAFJ2fYqGJ3yR1a8LCvFTUTJ+37GFjZbbg3/096STTrIae5s2bZCVlVVPVoWmUaNGAHwPSJo2bQoASE5OxkknnRT2Ox6PBzExMfXSiVXcEgbQtzOVEsEwUWWvyqfO0jSWBKMF6qC2aqgYjZWgrZRrQXKku7LxIdQ5XEhNNqHq1IkPH3SFTrfN+Dd2l8uF2NjYaq2k7U+p18DOwz5HPSbG91138RHEFPoazrESoNDwICEhwUbL7SExMRGAT4eEhARLj4SEBO3sNevFdKTN/+tmp4nE9HIpT1s5p5v4Q23VQW3VI0VbKdeCdKebc7qdg5Q25VT4+MNm7LjIvX6dndttNiDfy4cLXk37QycspFZf0OlWB51u4g+1VQe1VYc0baXYS6dbFjpfS6qR0qacCp1um7HjIvfv6zyeso3uDb9dwlwVTtKIik63zukt0tJw6HSrg0438YfaqoPaqkOatlLsDbWeSsXjulFbp5tzumXiX386tymn4rwrUjF2R7qtThGMdNuNlAHfRKLTLaXjV2WvynnX0jSWBOfLq4PaqkOatlLslRzpjnT1cs7plo20+1qnwTndNmPHwGEYQAxK0NO1BW4Y1jGzC3S5XAGOuU74/926N3ppnVOop+262itVW0a6CUBtVUJt1SFNWyn2Sna6mV7uLKS0KafCSLfNVOy4alQGDIzxvI/X4u9DzK41ZccCM8p1jXSH27ZEx0YvLQ0n1MCva/qUlK1gTLhlGPGH2qqD2qpDmrZS7HXSlmF0umVDp1tv9O0xhGJPejnQ3PUrAMBddMg6Vp5d7tK2Q2SkWx0S08t1Txs0UWWvyhRwaRpLgqn76qC26pCmrRR7pUe6a7JlWDTM6da5flQhpU05FTrdNmPPQmqGlVbuhq/zCJ7TLcPp1jkiy4XU1CH1gQbTywlAbVVCbdUhTVsp9kp3up2aXq7z/ZwqQmUv6HydOg3nXZGKsSvS7Spzul1GmdNdYfVyXfvDik53qOO6IGXAN+GcbnXQ6Sb+UFt1UFt1SNNWir1Ocrq5kJpspLQpp0Kn22ZsucgNwFMW4Q4V6XZB3znddLrVwUi3Ouh0E3+orTqorTqkaSvF3lAPvCse1w1GuvWuH1VIaVNOhU63zdgT6TYsZzsw0l32JNJliEgv131wktY5SXS6pcwrUmWvynnX0jSWBOfLq4PaqsNf24rHdETKtSA50h3plmF0umUj7b7WadDpthk75gZ7Q8zp9hrlke4YeLXtECWt8um/erm0Od2mtrraK2VVWhOuXk78obbqoLbqMHXUfew1kXItSLqvManp6uXRtJCazvWjCmm78jgN512RirFlITWUO9tuo7TsmGHN6fa4vCLSy3V/IiztiaDESLfuEQwTVfaqjEZL01gSzCJQB7VVh7+2FY/piJRrQdJ9jUl1xodQ53BOt2yk3dc6DTrdNmNfenlZOShPLzcd7RiXIWIhNd0HJ2mdk/9TW92dLGnack438YfaqoPaqkOq0637tSDpvsaEc7r1rh9VhHqQ4sSIv66wJmzGroXU3C5z9XIz0l2eXu6Bl3O6bUDKgG8iNdItSVs63QSgtiqhtuqg062eaNKWTnf0IbFNOQk63TZjT6TbP73cb0532ecel6FthxjuibCOT9qkPRGk060OOt3EH2qrDmqrDlNH3R94m0i6FqJR28qcbs7plgnndOuN865Ixdgzp9t/9fKySHeFhdQ4p7v2SBrwKyLF6ZbS8auyV+W8a2kaS4Lz5dVBbdUhOdKt+7UQSludqc74EOoczumWjeT7WidAp9tm7Ip0eyo63UCFhdT07BCZXq6WigO/rvZK05aRbuIPtVUHtVWHZKdb92tBythrwvRy/etIBZLalBOh020ztmwZ5jXgMhdSCxXpdnmha3cYarEvQM9GLzENp2KKm67pU1K2gjHhlmHEH2qrDmqrDm4Zpo6K2upqp0lNtwyj0y0bOt16o3evIRC7LnJ3Bac7YE43DEa6bUBi5yTlabuktEFAnb0qU8ClaSwJpu6rg9qqQ2qkW8K1IGXsNanO+BDqnGjap1v3OlKBpDZVE2bPno2srCwkJCQgOzsbq1evrvT8mTNn4tRTT0ViYiJat26NSZMm4cSJE3VkbTB0um3GnvRyw0ovNxdS81+9PMZVCk19bi6kphgpA7+0BxpMLyf+UFt1UFt1SHW6JVwLUsZeE6aX638/p4JQ2Qu6X6vVZeHChcjLy8PUqVOxdu1a9OjRAzk5OdizZ0/I8+fPn4877rgDU6dOxaZNm/DCCy9g4cKF+Nvf/lbHlpfjvCtSMbYspGYER7oNv0h3jMsLr6YrqXEhNbVUTC/X1V5p2tLpJv5QW3VQW3XQ6VaHU5xuLqQmG0ltKlJmzJiBCRMmYOzYsejcuTPmzJmDpKQkzJ07N+T5n332GQYMGIArrrgCWVlZuOCCCzBixIgqo+MqodNtM3ZFul3mQmoI3qebq5fbg8TOScrAL01bOt3EH2qrDmqrjooPZf2P6Yika0HKA28TRrr1ryMVSGpTJocPH8ahQ4esV2FhYdA5RUVFWLNmDQYNGmQdc7vdGDRoEFauXBmy3DPOOANr1qyxnOwff/wR7777Li666CI1f0g1oNNtM7ZEuuG3erm3pKzcCpFuTTtEzulWizSnW8q8IlX21oXTLUVjSXC+vDok9rtSkBzp1r2dSRl7TaozPoQ6h063bCT2r507d0ZaWpr1mjZtWtA5+/btQ2lpKTIyMgKOZ2RkID8/P2S5V1xxBe677z6ceeaZiI2NRfv27XHOOefUa3p5TL39cpRiR4dsGIaVXm7O6fZWWL1cArqvoCp5TrfuT9uldfx1EemueMzOsiVoLAlqqw6VbcLpSHa6dW9nUp3umka6o2EhNd3v51QgcVeejRs3omXLltb7+Ph4W8pdvnw5HnzwQTz99NPIzs7Gli1bcNNNN+H+++/H3XffbctvRAqdbpuxYwEHr4HgLcPgF+lGWaS7tBjYuxnI6AJo1Kjcbje8Xq/2A7/EzqniwK/roCJpKxhA/ZZhKh5ASdNYEtRWHdK2tZKENG0ltTOnbBnGOd2ykfQgyyQlJQWpqamVntO0aVN4PB4UFBQEHC8oKEBmZmbI79x9990YNWoUxo8fDwDo1q0bjh49iquvvhp33nlnvbRhvXsNgdi1kJqVXm6Up5d7Xb5nJFZ6+cfTgTkDgG9eraXV9iLlabvEzqniU2hd7ZWUNgios1dlW5CmsSSYuq8OKeODRKRpK6mdSY5012TLMDrdMpGYwVkd4uLi0Lt3byxbtsw65vV6sWzZMvTv3z/kd44dOxb0t9f39V3vNRHJnmvffvsthg0bhqysLLhcLsycOTPonHvuuQculyvg1bFjx4BzTpw4gYkTJ6JJkyZo0KABhg0bFvT0pKbYtZCatXp52b8GDBjuWAB+C6nt3eT7wm8/19xgBYQa+HVEotMtZeCXpq3q9HLVTrcEjSVBbdUhzTGUhDRtJbUzKWOvCRdS07+OVCCpTUVKXl4ennvuObz00kvYtGkTrrvuOhw9ehRjx44FAIwePRqTJ0+2zr/kkkvwzDPPYMGCBdi2bRuWLFmCu+++G5dcconlfNc19Zpebu65NmfOHGRnZ2PmzJnIycnB5s2bkZ6eHnT+sWPH0K5dO1x++eWYNGlS2HK7dOmCpUuXWu9jYgL/zEmTJmHRokV49dVXkZaWhtzcXAwdOhQrVqyo9d9kV6Tb7TI7dt8xrwEYHp/THesq9f3O8QO+D0uLavxbKuC8Y3VIGfilaava6VaxqKA0jSVBbdWhsk04HWnaSmpnUu5rTDinW/86UoGk7JFIGT58OPbu3YspU6YgPz8fPXv2xOLFi63F1bZv3x4Q2b7rrrvgcrlw1113YefOnWjWrBkuueQSPPDAA/X1J9Sv0+2/5xoAzJkzB4sWLcLcuXNxxx13BJ3ft29f9O3bFwBCfm4SExMTNsf/4MGDeOGFFzB//nyce+65AIAXX3wRnTp1wqpVq3D66aeH/F5hYWHAMvaHDx8OeZ59ke6ytBCXWS5g+KWXGwaA47/5PtTU6dY9tUViGo40p1tKx6/K3rqIdEvRWBJM3VeHtGisJKRpK6mdSRl7TaozPoQ6J5oi3brfz6kg1Dx93a/VSMjNzUVubm7Iz5YvXx7wPiYmBlOnTsXUqVPrwLLqUW9XZE32XKsuP/zwA1q0aIF27dph5MiR2L59u/XZmjVrUFxcHPC7HTt2RJs2bSr93WnTpgUsad+5c+eQ59l1kZenl5cNSkCF9HIDOHHAd7KmTrfuT4QlPWU3kTLwS9OW6eXEH2qrDmmOoSSkTO0ykdTOpIy9JjWNdNf3nFc70P1aUomkNuVE6s3prsmea9UhOzsb8+bNw+LFi/HMM89g27ZtOOuss6zIdH5+PuLi4tCwYcOIfnfy5Mk4ePCg9dq4cWPI8+yImHoNo3whNf9It196udeAtunlUhb7krx6ue4rqEpalRYItE3F6uX+N8FcvVx/qK06VLYJp1NRW911ldTOolHbys6h0y0TOt16E3Vbhl144YXW/7t3747s7Gy0bdsW//nPfzBu3LgalxsfHx+wd9yhQ4dCnmdPenn5AmpmKQYqpJd7S4DCMhs0c7oZ6VaHlKft0rT1t03F6uWc0y0LaqsOafOOJSFl7DWR1M6iUVtGuqMPSW3KidTbo7qa7LlWExo2bIhTTjkFW7ZsAQBkZmaiqKgIBw4cUPK79iykFjyn238htRiUIqHkSPkXSotr/Fsq4JxudUh0unW3FVDvdHPLMFlwvrw6mF6uDinjg4mkdiZV20i3DIumhdR0v59TgcQMTidRb1dkTfZcqwlHjhzB1q1b0bx5cwBA7969ERsbG/C7mzdvxvbt2235Xbsi3R4z0h2wkJqZXu5FYqlfpF3TSLfug5PEJ4JSnrZL01a60y1BY0lQW3XQ6VaHlLHXRFI7i0ZtGemOPiS1KSdSr+nleXl5GDNmDPr06YN+/fph5syZQXuutWzZEtOmTQPgW3zNnEtdVFSEnTt3Yv369WjQoAE6dOgAAPjrX/+KSy65BG3btsWuXbswdepUeDwejBgxAgCQlpaGcePGIS8vD40bN0ZqaipuuOEG9O/fP+zK5ZFgz0VuwGXO6baOAIbbV12xKEVCid/q6XS6a4TEzonaqsHfNjsfaDC9XCbUVh1ML1eHlIeyJpLaWTRqG+oczumWjaQ25UTq1emOdM+1Xbt2oVevXtb76dOnY/r06Rg4cKC1VPyOHTswYsQI/Prrr2jWrBnOPPNMrFq1Cs2aNbO+9/jjj8PtdmPYsGEoLCxETk4Onn76aVv+Jrsi3e5Qke6YBABAkquwQqRbz/Ry3QcniZ2TNKdbSoqT5Ei3FI0lwdR9dTDSrQ4p44OJpHYmVVsnbxmmex2pQOJ9rZOo94XUItlzLSsrq8qOYMGCBVX+ZkJCAmbPno3Zs2dX287qYkeHbBiwVi8PmNMdmwQASHQVI614X/kXNI10677CNud0q0Naxy/Z6ZaisST8ta14jNQOOt3qkDI+mEjqw6JR28oi3ZzTLRPO6dYb512RirFj2wuvYYRevdzjWz3d5QIyin4q/0KJXk43twxTh5QHGpK2ggECbbNTW7MM/1RaFVuG2V220+G2VupQ2SacTkVtdddV0jgRjdqGOodzumUj6UGWE9G71xCIPenloffp9pb1gW4XkFn0c/kXNI106+50S+ycpDzQkJQ2CMiOdDNiaD/UVh3UVh1Sxl4TSVNkpGob6erldLplIzGD00mwJmzGlsZuwNoyLCDSXfbOBaCFf6SbTneNkOh0U1s10Okm/lBbdVBbdUgZH0wkjRPRqC0XUos+JLUpJ0Kn22ZsW0jNFbiQmtcwyp1uF9CkZE/5F7iQWo2Q2DlJGfilaetvm7TVy7kKtP1QW3VQW3VIGXtNJI0T0ahttM/p1r2OVCApe8SJ0Om2GTsucgOGFek2F1IzDMB87hhUsuaRbl1TWySm4UhzuqV0/Ix0E3+orTqorTqkjA8mkqYhSdXWyauX634/p4JQ8/R1v1adhPOuSMXYvmWYWS4QEOkOQHOnW9cGL+kpuwm1VQOdbuIPtVUHtVWHlPHBRNI4EY3ahjqHc7plI6lNORE63TZjR8TUMIzQ+3T7zekOQLP08oqrfOra4CU+EayY4qbrk1xJq9ICgbbZmfWgcuE7rrCtDmqrDimLQUqkora6X7OSxolo1Layc+h0y4ROt97o3WsIxJb08rALqaGs7Apf0DTSrbvTLS0FGpDztF1ax19Xc7rt1IBzY9VBbdUhZXyQiDRtJY0T0agtI93Rh6Q25UTodNuMXVuGlc/pLnOyDMAwwkW69XS6dX8izDnd6pA0Vw+ou/RyFU4303Tth9qqQ0ofJhFp2kp68C1V20i3DIumhdR0v59TgX/96d6mnIjzrkjF2BfpDpFeXvZ/KZFu3QcniU8EpTxtl6YtnW7iD7VVh5TxQSLStJU0TkSjtox0Rx+S2pQTodNtM3ZHugPSy8u87lJ4Ar9Ap7tGSOycqK0amF5O/KG26pDy4FAi0rSVNE5Eo7ahzuGcbtlIalNOhE63zdizZViYSHeZC34UiWVnmh96AW8pdIGOoTqkaSslxYmRbuIPtVWHlD5MItK0lTQNSaq2Tt4yTPc6UoHE+1onQafbZuxJLzfgqRDp9vrt030ESb7/JDct/5JG0W4pT4Qldk5SBn5p2tLpJv5QW3VI6cMkIk1bSeNENGpbWaQ7GuZ0615HKpAW8HAadLptxp4tw8oj3W4zmA2X1YAsp7tBRvmXNHK6K24ZputiFv6Di5SFNyo+0NDVXklbwQChnW4VW4bZqQG3tVIHtVWHtK2XJCFNW0njRDRqG+qcaJrTrXsdqUDiVrhOwnlXpGLsmdMNuMxIt+V0I9jpDoh067NXt5QnwpKesptI2eNWorYqMjQ4p1sm1FYdUjKhJCJNW0njRDRqy4XUoo/KshdI/cOasBm7FlKrmF5uwGU5WodNpzuxMeCO9f1fo0g3nW51SNRWd1tNVGjL9HKZUFt1SOnDJCJNW0mpsFK1remWYXS6ZSLx3stJ0Om2GbsWUvO4Kiyk5lf2N2iHdbGnAb2vBDxxvhM0dLp1fyJMp1sdkrVlpJtQW3VIGR8kIk1bSeNENGpbWVSUTrdMJLUpJ0Kn22Zscbr95xGWFeM1yud0/2qkYWra/UC7gYCnLNJdop/Trfvcp8qe8uqKNKdbQgTDhJFuYkJt1SGlD5OING0lReWkalvT1cujYSE13e/nVOBff1LuvZyE865Ixdhykftt/1WeXu735NHlgvUQUuNIt+6Dk8QngtRWHXS6iQm1VYeUPkwi0rSVNE5Eo7ac0x19SGpTToROt83Y0SF7Df+URt+//quXAy54zf9r7HTrnoYlsXOSMvBTW3Vlhivb7vKdDLVVh5Q+TCLStJU0TkSjtkwvjz4ktSknQqfbZmxZLdD/Rs8q178j8a1wDqA8vVyj1csrbhmma4OXmIZT8YGGrulTkraCMVGxJUxdbxmm+/UrBSm7BEhE2tZLkpCmraRxIhq1rewcOt0yodOtN3r3GgKxZ053eaTbmtMNBES6rf/HxPv+1TDSrfvgJHlOt+4PNCR2/NIXUtNdX0lQW3VQW3VI01bSOBGN2laWXs453TKRGExyEs67IhVjy0Xu94SxvFP0c7Rd/unl3DKspkga8E0kaqu7rSbS08t111cS1FYd1FYd0rStzmJfuiBV2+puGWbCOd2ykXhf6yTodNuMPU53ifVfK70cFSPdZf+15nTrk14uZXCS2DlRW3Uw0k1MqK06qK06pGkraZyIRm0r/k0A08ulI6lNORE63TZjT3q5f6S77Jhf2XBBxEJqdAztR5q2EiIYJox0ExNqqw5qqw5p2krKiJKqbXW3DDOh0y0bife1ToJOt83YE+kOXkjNa4SLdOubXq77E2GJnZOUgZ/aqiuzLsp2OtRWHdRWHdK0lTRORKO2lTnd0TCnW/c6UoHEgIeToNNtM3Ys4BBqITX/1csD53TrF+muuHq5rotZ+D/RlbLwRsVBUld7Ja1KayJ99XLd9ZUEtVUHtVWHNG0ljRPRqG0oBzua5nTrXkcqCJWpQKdbH5x3RSrG9ki3lV7uCmhE5VuG6ed0S3kiLPGJILMI1MFINzGhtuqgtuqQpq2kcSIatQ0V6Y4mp1v3OlJBqDp1og66QqfbZmyZ022UR7pd1jH/zt4dYvVyLqQWKZIGfBMp+wdL1JYLqRETaqsOaqsOadpKGieiUVsupBZ9VDZlgNQ/rAmbsaWx+3WAIRdSg9+uYhpHunUfnCQN+CYSH2jobqsJnW5iQm3VQW3VIU1bSdlmUrWtzpZhdLqjh1B16kQddIVOt83Y09j9nlCVFeP1cwzh8ks198T7/tXQ6dZ97lOoQUlXW02kOd0SbqZMmF5OTKitOqitOqRpK+nhrFRta7p6eTQspKb7/ZwKQtWf7teqk3DeFakYe7YMC1693DD853S7/OZ0M728pjDSrQ5qq67Muijb6VBbdVBbdUjTVtI4EY3ack539ME53XpDp9tmbOmQjSrSy116r14uJQ1L0oBvImXgp7bqyqyLsp0OtVUHtVWHNG0ljRPRqG1lkW463TKh0603dLptxpatCrzlnZ0V6YYR0AkGRbpLCmv+ezYjZbEvfzuldNIVH2jomj4laSsYE24ZRkyorTqorTqkaStpnIhGbUOlItPplg3ndOuN3r2GQOxp7MH7dHu9gZFuX+wbfpFu/dLLdXcMQz0J1tVWE2YRqIMLqRETaqsOaqsOadpKGieiUdtQDpqZXs453TLhnG69cd4VqRg753R74Q6ZXu4KmNOtb3q57pFuSYu4mFBbdTC9nJhQW3VQW3VI07Y6i33pglRtq7N6Oed0Rw9ML9cbOt02Y0tjN3yNpcQVF7BPd3mkGyH26abTHSmSnrKbUFt1MNJNTKitOqitOqRpK2mciEZtQ0W6mV4uGzrdekOn22ZsaexeXwdY4o4t7zgD5nS74PXqn15Ox9B+pGkrIYJhwkg3MaG26qC26pCmraSMKKnaVuZ8cSG16INzuvWGTrfNqIp0V5zTbQhIL9f9iTCdbnVQW3Vl1kXZTofaqoPaqkOatpLGCcnaVjxW8T2d7uiBkW69odNtM7Ys4FDmdJe6YuEOF+k2/x+jn9Nt/u26L6TmP7hIWXijYoeqq72SVqU14erlxITaqoPaqkOatpLGCanahnKoKzuHC6nJhgup6Y3zrkjF2JNe7msshssFV9mc7Ypzuq1nkEwvrzGSU6CZRWA/jHQTE2qrDmqrDmnaShonpGrL9HJnwUi33tDpthk7twwz4PZzuv0i3S6330Jq+kW6pQxOkgZ8Eyl7oEvUlgupERNqqw5qqw5p2koaJ6RqW9nc3srOodMtk1B16sSIv66wJmzG3ki3G66YGN//4edoAX5bhum7ernug5OkAd9E4gMN3W01odNNTKitOqitOqRpKynbTKq2kTjdLpeLTrdwuJCa3tDpthk7F1Iz4Ia7zKn2egPndBuCIt26PmUL5RjqaquJNKdbws2UCdPLiQm1VQe1VYc0bSU9nJWqbSTp5S6XK+ScYGlIuZ9TAed06029X5GzZ89GVlYWEhISkJ2djdWrV4c999tvv8WwYcOQlZUFl8uFmTNnBp0zbdo09O3bFykpKUhPT8eQIUOwefPmgHPOOecc64me+br22mtt+Xtsdbpd7jBzul1+kW79nW5dGzwj3eqgturKrIuynQ61VQe1VYc0bSWNE1K1jdTpZqRbNtWpd1J/1NjpLioqwo4dO7B9+/aAVyQsXLgQeXl5mDp1KtauXYsePXogJycHe/bsCXn+sWPH0K5dOzz00EPIzMwMec5HH32EiRMnYtWqVViyZAmKi4txwQUX4OjRowHnTZgwAbt377ZejzzySES2h8Mep7tsTre/0x2werlfh2ill+u3kJruaViSBnwTKQM/tVVXZl2U7XSorTqorTqkaStpnJCqLZ1uZ0GnW29iIv3CDz/8gKuuugqfffZZwHHDMOByuQLmEVTFjBkzMGHCBIwdOxYAMGfOHCxatAhz587FHXfcEXR+37590bdvXwAI+TkALF68OOD9vHnzkJ6ejjVr1uDss8+2jiclJYV13ENRWFiIwsJC6/3hw4dDnmfnlmG+hdTK5nQHLKSmd6RbymJf/nZK6aSlpO5L2grGhFuGERNqqw5qqw5p2koaJ6RqG8mWYW63m063cDinW28i7jWuvPJKuN1uvPPOO1izZg3Wrl2LtWvXYt26dVi7dm21yykqKsKaNWswaNCgcmPcbgwaNAgrV66M1KywHDx4EADQuHHjgOP/+te/0LRpU3Tt2hWTJ0/GsWPHKi1n2rRpSEtLs16dO3cOeZ796eU+p7rinG7rtzR0uhnpVge1VQcj3cSE2qqD2qpDmraSxgmp2tY00h0Nc7p1ryMVREP9RTMRR7rXr1+PNWvWoGPHjrX64X379qG0tBQZGRkBxzMyMvDdd9/VqmwTr9eLm2++GQMGDEDXrl2t41dccQXatm2LFi1a4Ouvv8btt9+OzZs344033ghb1uTJk5GXl2e937lzZ0jH246Bw+XndHs85eX4R7oB3wrmHgH7dOv6RDjUIi662mpCp1sdKrRVWV9SrgWJUFt1UFt1SNNW0jghVdtIVy+Ppn26db+fU4GUTFOnErHT3blzZ+zbt0+FLbYzceJEbNiwAZ9++mnA8auvvtr6f7du3dC8eXOcd9552Lp1K9q3bx+yrPj4eMTHx1vvDx06FPI8O+d0Ay644hsElW3iNQw/p1u/SLfujV7SgG8iUVvdbTWh001MqK06qK06pGnrP57pPgZL1ZZbhjkLKfeITiXix0APP/wwbrvtNixfvhy//vorDh06FPCqLk2bNoXH40FBQUHA8YKCgojmWocjNzcX77zzDj788EO0atWq0nOzs7MBAFu2bKn179rR2I2yCdtelweuc8rnrlvpIlaniPKF1ErodEcKnW51SLqZMmF6OTGhtuqgtuqQpq2kMViqtlxIzVlIu06dRsSRbnMO9nnnnRdwPNKF1OLi4tC7d28sW7YMQ4YMAeC7SJYtW4bc3NxIzQqw44YbbsB///tfLF++HCeddFKV31m/fj0AoHnz5jX+Xf/fB+xJL4fLBVfLntZx62kkzPRyzumuDZIGfBMpHSq1VVdmXZTtdKitOqitOqRpG8rB09VmqdrS6XYWUu6/nUrETveHH35o24/n5eVhzJgx6NOnD/r164eZM2fi6NGj1mrmo0ePRsuWLTFt2jQAvsXXNm7caP1/586dWL9+PRo0aIAOHToA8KWUz58/H2+99RZSUlKQn58PAEhLS0NiYiK2bt2K+fPn46KLLkKTJk3w9ddfY9KkSTj77LPRvXv3Wv9NtnTI5pZhcAfMSSmPdJedZiDQ6TYMKwpen1Rs9LrOq5E8p1v3gZ9Ot7oy66Jsp0Nt1UFt1SFN2+o4hroQjdqG+ptCrXouDSn3cyrgnG69idjpHjhwoG0/Pnz4cOzduxdTpkxBfn4+evbsicWLF1uLq23fvj2g0ezatQu9evWy3k+fPh3Tp0/HwIEDsXz5cgDAM888AwA455xzAn7rxRdfxJVXXom4uDgsXbrUcvBbt26NYcOG4a677rLlb7JjAYfyhdQ8AQ2mvBP0j3THmr8MeEsBT8RVajtSGr3kLcN0f6AhaSsYE24ZRkyorTqorTqkaVudba10IRq15ZZh0Ye0h0NOo0Ye2oEDB/DCCy9g06ZNAIAuXbrgqquuQlpaWsRl5ebmhk0nNx1pk6ysrCo7gqo+b926NT766KOIbIwEO7cMg1+qD+C3IIbL3+mOK/9eaZEWTreURi8xGivlgYZEbRnpJibUVh3UVh3StGWkWx1ML9e/jlQgJTDjVCKujS+//BLt27fH448/jv3792P//v2YMWMG2rdvH9E+3dGKvft0B0a6g51uBDvdGiBlcKJjqA7J2nL1ckJt1UFt1SFN24r2+h/TjWjUNtTfRKdbNtKuU6cRcVh00qRJuPTSS/Hcc88hJsb39ZKSEowfPx4333wzPv74Y9uNlISdW4YZcAU8paqYXg7/1csBbfbqlvKkLZRjqKutJtKcbq/Xq72tJnS6iQm1VQe1VYc0bel0q6OmTjfndMtGSjakU4nY6f7yyy8DHG4AiImJwW233YY+ffrYapxE7HG6y54wutwB5YSc0+1y+aLdpUWMdEeIv9Otu60mErWV8tSZ6eXEhNqqg9qqQ5q2TC9XRyhtqzqHkW75SLtOnUbEj4FSU1Oxffv2oOO//PILUlJSbDFKMrZsGYbKF1IzD3nNTtFawbywxr9pJ1IaPR1DdVBbdWXWRdlOh9qqg9qqQ5q2dLrVUR176XRHH9KuU6cRsdM9fPhwjBs3DgsXLsQvv/yCX375BQsWLMD48eMxYsQIFTaKwo60FpfXnLsdGOkOOacbKE8x1yS93PzbdU/D8q8jKZ10xQ5V1/Qprl6ursy6KNvpUFt1UFt1SNOWq5erozr2VrZ6uWSk3M+pQNo0CKcRca8xffp0DB06FKNHj0ZWVhaysrJw5ZVX4o9//CMefvhhFTaKwp7Gbka6A1cvr/jkygiKdOuVXq57ow/1QENXW02kactINyPdEqG26qC26pCmLSPd6qhtpBuQG+2Wcs+hAmnXaaTMnj0bWVlZSEhIQHZ2NlavXl3p+QcOHMDEiRPRvHlzxMfH45RTTsG7775bR9YGE/Gc7ri4ODzxxBOYNm0atm7dCgBo3749kpKSbDdOInbO6TbgCbmQmtt0aMwPNHW6dX8iHOqBhq62moRbCEU3JDvdXEiNUFt1UFt1SNO2or3+x3RDuraVOd2hFlIDfPdFHo9Htam2I+V+TgXRvJDawoULkZeXhzlz5iA7OxszZ85ETk4ONm/ejPT09KDzi4qKcP755yM9PR2vvfYaWrZsiZ9//hkNGzase+PLqPGmzklJSejWrZudtkQFtszpNkKnl5c73WWNyop065VeLuVJW2VZBLoi0emWpi2dbkJt1UFt1SFNWzrd6qip081It2yk3H/XhBkzZmDChAkYO3YsAGDOnDlYtGgR5s6dizvuuCPo/Llz52L//v347LPPEBvr85OysrLq0uQgquV0Dx06FPPmzUNqaiqGDh1a6blvvPGGLYZJxR6nuyzVKmx6edl7a0633pFuXRs9nW51+F8DUgZAppcTE2qrDmqrDmnaMr1cHUwv17+OVCDtOgWAw4cP49ChQ9b7+Ph4xMfHB5xTVFSENWvWYPLkydYxt9uNQYMGYeXKlSHLffvtt9G/f39MnDgRb731Fpo1a4YrrrgCt99+e71lcFTL6U5LS7MqLjU1VUQl1hf2pJeHXr3cf99rA4DXWzHSTac7Euh0q8M/0l3xmK7Q6SYm1FYd1FYd0rSl060OOt3615EKpGVkAEDnzp0D3k+dOhX33HNPwLF9+/ahtLQUGRkZAcczMjLw3XffhSz3xx9/xAcffICRI0fi3XffxZYtW3D99dejuLgYU6dOtfVvqC7VcrpffPFF6//z5s1TZUtUYGek23C5Q87phuXQlH3gKXsipFl6uf9DAh0J90BDZ+h0q4NONzGhtuqgtuqQpi2dbnXU1OkOec8pDM7plnOdAsDGjRvRsmVL633FKHdN8Xq9SE9Px7PPPguPx4PevXtj586dePTRR+vN6Y74ijz33HNx4MCBoOOHDh3Cueeea4dNorGnsZc5KmHndJc5NNBz9XKJW4bpbqtJRft0HVQkbQVjwi3DiAm1VQe1VYc0bSWNE9K1rcmWYYx0y0PawyEASElJQWpqqvUK5XQ3bdoUHo8HBQUFAccLCgqQmZkZstzmzZvjlFNOCUgl79SpE/Lz81FUVD/+UsS9xvLly0Mae+LECXzyySe2GCUZuxdS88dqRO6yRlVxn+4SPZxuKY2e6eXqkBTBMGGkm5hQW3VQW3VI01bSOCFdW6aXOwNp12l1iYuLQ+/evbFs2TLrmNfrxbJly9C/f/+Q3xkwYAC2bNkS0L98//33aN68OeLi4pTbHIpqr17+9ddfW//fuHEj8vPzrfelpaVYvHhxQHqAU7E7vdwsyzCM8hToiquXxzXw/Vt4sMa/aSdSGr1Ep7vi02pd7a04xcD/mK6omAulcn6VxLlbUqC26qC26pCmraRxQrq2lTndXL08epAyvbMm5OXlYcyYMejTpw/69euHmTNn4ujRo9Zq5qNHj0bLli0xbdo0AMB1112HWbNm4aabbsINN9yAH374AQ8++CBuvPHGevsbqu109+zZ02qQodLIExMT8dRTT9lqnETsaezm3O1Ap9vE7a4wX7ZB2f50R/bW4jftQ8rgFGpOt662mkiLdFd1TCdUaKuyvqRcCxKhtuqgtuqQpq2kcUK6ttXRuqLTLX1Ot+51pAJp12kkDB8+HHv37sWUKVOQn5+Pnj17YvHixdbiatu3bw94yNC6dWu89957mDRpErp3746WLVvipptuwu23315ff0L1ne5t27bBMAy0a9cOq1evRrNmzazP4uLikJ6eXm9LsOuELZFub2Ck2+12B855MiPJph9uOd2Bcx3qi4qRbl2ftIWcL6+prSZSOlRJN1MmdLqJCbVVB7VVhzRtJY0T0rWtrtPtfw8kPdKt+/2cCqRkQ9aU3Nxc5Obmhvxs+fLlQcf69++PVatWKbaq+lTb6W7bti0AuU++6go7twxz+UW6/bGc2oqR7qN6Rbp1f9ooMb1cysAv6WbKhE43MaG26qC26pCmre72+SNd25pEuqU73brXkQqkXadOI+LHQNOmTcPcuXODjs+dOxcPP/ywLUZJxo6IqatsVXLD5cscCGpEZWVb/WGyGeneU+PftBOuXq6Oivbp+iS3spVSdaWifXauXm5nmXVRttOhtuqgtuqQpq0keyXZClTP3lDn0OmWDZ1uvYm41/jHP/6Bjh07Bh3v0qUL5syZY4tRklGxenmQo1X2NijSrVl6ue6OLOd0q4ORbnVl1kXZTofaqoPaqkOatpLslWQrwEg3oH8dqUDadeo0qp1ebpKfn4/mzZsHHW/WrBl2795ti1GSsXP1cpRFuoOeRroqRLo1Ty/X9Ykw53Srg063ujLromynQ23VQW3VIU1bSfZKshWwZ0631OmkUu7nVBDtc7rrA6/Xiy1btmDPnj1BbeLss8+OqKyIne7WrVtjxYoVOOmkkwKOr1ixAi1atIi0uKjDFqcb5kJqrpBlmY3KinSb6eXFx4DCI0B8gxr/th1wTrc6pAz8dLrVlVkXZTsdaqsOaqsOadpKsleSrQAj3YD+daQCadep7qxatQpXXHEFfv7556D24HK5ArY7rA4RO90TJkzAzTffjOLiYmvrsGXLluG2227DLbfcEmlxUYedke5wC6mZWKuXxzcAYpN8TveRAm2cbt07vlCDi662mkjpUOl0qyuzLsp2OtRWHdRWHdK0lWSvJFsBOt2A/nWkAmnXqe5ce+216NOnDxYtWoTmzZvXWs+Ine5bb70Vv/76K66//noUFRUBABISEnD77bdj8uTJtTImGrCnsVe+kJrbE+jUAvClmP/2ky/FvEn7Wvx27ZHS6OkYqoPaqiuzLsp2OtRWHdRWHdK0lWSvJFsBOt2A/nWkAmnXqe788MMPeO2119ChQwdbyot4woPL5cLDDz+MvXv3YtWqVfjqq6+wf/9+TJkyxRaDpGNHY3ebC6mVrZgWbk63178/1GgF83Dp8LoRqo50tdVESodKp1tdmXVRttOhtuqgtuqQpq0keyXZCtTc6fb/l3O65cE53faSnZ2NLVu22FZexJFukwYNGqBv3762GRIt2NLYraeLYSLdFdK3AWi1grmkRu9yuQJ01NlWQM4DDW4Zpq7Muijb6VBbdVBbdUjTVpK9kmwFar5lGFB+X8RItzykPRzSnRtuuAG33HIL8vPz0a1bN8TGxgZ83r1794jKq5bTPXToUMybNw+pqakYOnRopee+8cYbERkQbdi5kBr8OsCAz90uwFsh0q3RCuaSGr10p1tXexnpVldmXZTtdKitOqitOqRpK8leSbYC9kS66XTLQ9p1qjvDhg0DAFx11VXWMdNvcLkULaSWlpZmVVxaWlpEP+A07N0yLMw+3W53mdPt1yEm6xPpltToJdkKyMkioNOtrsy6KNvpUFt1UFt1SNNWkr2SbAXodAP615EKpGRDSmHbtm22llctp/vFF18M+X8STF043S6UzbcJmV7OSHckSLIVkGMvnW51ZdZF2U6H2qqD2qpDmraS7JVkK0CnG9C/jlQg7TrVnbZt29paXo3ndJPQ2Jte7pvTHXZuTsj0ci6kFgmSbAXkdKh0utWVWRdlOx1qqw5qqw5p2kqyV5KtQO2cbvM+iAupyUNKNqTOvP3227jwwgsRGxuLt99+u9JzL7300ojKrpbT3atXr2pX3Nq1ayMyINqwZ/Vys6MLE+l2mZFuv4NML68RkmwF5Nirq12VQaebmFBbdVBbdUjTVpK9kmwFGOkG9K8jFUi7TnVkyJAhyM/PR3p6OoYMGRL2PJdL0Zxu/x89ceIEnn76aXTu3Bn9+/cHAKxatQrffvstrr/++oh+PBoxn7DV7kIv6+jcYVYvd1eRXm4YQD02NEmNXpKtgBx7pdjpD51uYkJt1UFt1SFNW0n2SrIVoNMN6F9HKpB2neqIf4aH3dke1XK6p06dav1//PjxuPHGG3H//fcHnfPLL7/YapxEzMZem7QWa5/uMHO6PaGc7sRGvn9LjgOlRUBMfI1/v7ZISm+RZCsgJx1e2vYqALcMI+VQW3VQW3VI01aSvZJsBWq/ZRhAp1sidLrrh27duuHdd99F69atKz0v4l7j1VdfxejRo4OO/+Uvf8Hrr78eaXFRhz2NvayjK3O6gx1Dd9lv+R30xJX/v7S4Fr9de6Q4hoAsWwE5HaoUO/1hpJuYUFt1UFt1SNNWkr2SbAU4pxvQ/35OBdICSdHCTz/9hOLiqn2viK/IxMRErFixIuj4ihUrkJCQEGlxUYctc7rLFlJzhdmnuyzQDcN/JbUAp7uoxr9tB5IGJ0m2AnLslWKnP3S6iQm1VQe1VYc0bSXZK8lWgOnlgP51pAJp16nTiHj18ptvvhnXXXcd1q5di379+gEAPv/8c8ydOxd333237QZKo062DLOeQvoddHsAuAAY2kW6dW70kmwF5NgrxU5/6HQTE2qrDmqrDmnaSrJXkq0AnW5A/zpSgbTr1GlE7HTfcccdaNeuHZ544gn885//BAB06tQJL774Iv70pz/ZbqBUauV0o3Kn2+0KMafb5fJFu0sLGemOAEm2AnLslWKnP3S6iQm1VQe1VYc0bSXZK8lWgE43oH8dqUDadeo0arRP95/+9Cc62CHw76BqF+k253SHW728LNJdsT/0xPqcbi8j3dVFkq2AHHul2OkPnW5iQm3VQW3VIU1bSfZKshWwx+mWPqdb9zpSgbTr1GnUaJWBAwcO4Pnnn8ff/vY37N+/H4Bvf+6dO3faapw0/DuoWq1eDt/q5WYaedAKk2VtKOgppCfW9289p5dLWuVTkq1A+AcwuiHFTn+4ejkxobbqoLbqkKatJHsl2QrUbvVy81/pkW7d60gFXEhNbyK+Ir/++muccsopePjhh/Hoo4/iwIEDAIA33ngDkydPtts+Udgf6a58TndQd2gupsb08mojyVZAlr3+tulspwkj3cSE2qqD2qpDmraS7JVkK8D0ckD/OlKBtOtUZ4qLi3Heeefhhx9+qPLcf/zjH8jIyKjyvIid7ry8PFx55ZX44YcfAlYrv+iii/Dxxx9HWlxUYZvTbc7pdleVXl6hQ3TrEemW1Ogl2QrIeopJp5tOt1SorTqorTqkaSvJXkm2AnS6Af3rSAUSswx1JTY2Fl9//XW1zr3iiiuQnJxc5XkR18YXX3yBa665Juh4y5YtkZ+fH2lxUYVdTre7ioXUPOZ8m1BzugE63REgyVZAlr10uul0S4XaqoPaqkOatpLslWQrQKcb0L+OVCDtOtWdv/zlL3jhhRdsKy/ihdTi4+Nx6NChoOPff/89mjVrZotRUlG1kFq4eTfBc7r1TC/X+UmbJFsBWR0qnW463VKhtuqgtuqQpq0keyXZCtTO6bayKYUvpKb7/ZwKJGVDSqCkpARz587F0qVL0bt376Bo9owZMyIqL2Kn+9JLL8V9992H//znPwB8Fbp9+3bcfvvtGDZsWKTFRRX2RbrLFlILN6fbinTLcLp1bvSSbAVk2Uunm063VKitOqitOqRpK8leSbYCjHQD+teRCqRdp7qzYcMGnHbaaQB8wWV/aqJtxE73Y489hj/+8Y9IT0/H8ePHMXDgQOTn56N///544IEHIjYgmrA90u0Ot5BamdNd8SGkp6w6vSU1/m07kNToJdkKyLKXTjedbqlQW3VQW3VI01aSvZJsBeh0A/rXkQqkXae68+GHH9paXsS5F2lpaViyZAneeecdPPnkk8jNzcW7776Ljz76qFqTyCsye/ZsZGVlISEhAdnZ2Vi9enXYc7/99lsMGzYMWVlZcLlcmDlzZo3KPHHiBCZOnIgmTZqgQYMGGDZsGAoKCiK2vSJ2bRlmLqTmCrOQmjmnW9fVyyWlt0iyFZCVDu9vm852mnDLMGJCbdVBbdUhTVtJ9kqyFajdlmF0uuVCp1sdO3bswI4dO2pVRkS9RnFxMWJiYrBhwwYMGDAA119/PW677TYMGjSoRj++cOFC5OXlYerUqVi7di169OiBnJwc7NmzJ+T5x44dQ7t27fDQQw8hMzOzxmVOmjQJ//vf//Dqq6/io48+wq5duzB06NAa/Q3+qFpILVzHKCW9XOfBSZKtgKwOlZFuRrqlQm3VQW3VIU1bSfZKshXgnG5A//s5FUgLJOmO1+vFfffdh7S0NLRt2xZt27ZFw4YNcf/999eofUR0RcbGxqJNmzYoLS2N+IdCMWPGDEyYMAFjx45F586dMWfOHCQlJWHu3Lkhz+/bty8effRR/PnPf0Z8fHyNyjx48CBeeOEFzJgxA+eeey569+6NF198EZ999hlWrVpVq7/Hvi3Dyp7ShUkvd7vDPIV0l6WXlzK9vLpIshWQZS+dbjrdUqG26qC26pCmrSR7JdkKML0c0L+OVCDtOtWdO++8E7NmzcJDDz2EdevWYd26dXjwwQfx1FNP4e677464vIgfA915553429/+hv3790f8Y/4UFRVhzZo1AVFyt9uNQYMGYeXKlcrKXLNmDYqLiwPO6dixI9q0aVPp7xYWFuLQoUPW6/Dhw0Hn2Denuyy93BVmn26XGemu8EVNI906N3pJtgKy7KXTTadbKtRWHdRWHdK0lWSvJFsBOt2A/nWkAmnXqe689NJLeP7553Hdddehe/fu6N69O66//no899xzmDdvXsTlRbyQ2qxZs7Blyxa0aNECbdu2DZrHvXbt2mqVs2/fPpSWliIjIyPgeEZGBr777rtIzap2mfn5+YiLi0PDhg2Dzqlsn/Fp06bh3nvvrfT362qfbvNtcHq5uU83ne7qIslWQJa9dLrpdEuF2qqD2qpDmraS7JVkK0CnG9C/jlQg7TrVnf3796Njx45Bxzt27Fij4HPETveQIUMi/pFoYPLkycjLy7Pe79y5E507dw44xz6n20wvjwlZlhnpDuoPzUg3Vy+vNpJsBWTZS6ebTrdUqK06qK06pGkryV5JtgJ0ugH960gF0q5T3enRowdmzZqFJ598MuD4rFmz0KNHj4jLi9jpnjp1asQ/EoqmTZvC4/EErRpeUFAQdpE0O8rMzMxEUVERDhw4EBDtrup34+PjA+aRHzp0KOgc/0n1tUovNyPdZXO3Ky6M4PGE6RA1jXTrvJiFJFsBWR0qnW463VKhtuqgtuqQpq0keyXZCtTO6eZCanLhQmr28sgjj+Diiy/G0qVL0b9/fwDAypUr8csvv+Ddd9+NuLwaX5FffvklXnnlFbzyyitYs2ZNxN+Pi4tD7969sWzZMuuY1+vFsmXLrD9MRZm9e/dGbGxswDmbN2/G9u3ba/y7Jv5OcG0au9ua0x1un+5wc7r1cLolNXpJtgKyHhJwyzBuGSYVaqsOaqsOidr6j2k62ytNW24Zpv/9nAqkPRzSnYEDB+L777/HZZddhgMHDuDAgQMYOnQoNm/ejLPOOivi8iKOdO/YsQMjRozAihUrrEjxgQMHcMYZZ2DBggVo1apVtcvKy8vDmDFj0KdPH/Tr1w8zZ87E0aNHMXbsWADA6NGj0bJlS0ybNg2Ab6G0jRs3Wv/fuXMn1q9fjwYNGqBDhw7VKjMtLQ3jxo1DXl4eGjdujNTUVNxwww3o378/Tj/99EjlCMD+1cvDLaTmex9+y7DiGv+2HUhq9JJsBWTZy0g3I91SobbqoLbqkKity+US4SRJ05bp5frXkQqkXacSaNGiBR544AFbyorY6R4/fjyKi4uxadMmnHrqqQB8keKxY8di/PjxWLx4cbXLGj58OPbu3YspU6YgPz8fPXv2xOLFi62F0LZv3x7wJG7Xrl3o1auX9X769OmYPn06Bg4ciOXLl1erTAB4/PHH4Xa7MWzYMBQWFiInJwdPP/10pFIEYZfT7bHSy8NtGRYm0u02I910uquLJFsBWZF5Ot10uqVCbdVBbdUhUVsp44Q0bel0619HKpCUDSmBxYsXo0GDBjjzzDMBALNnz8Zzzz2Hzp07Y/bs2WjUqFFE5UXsdH/00Uf47LPPLIcbAE499VQ89dRTNQq15+bmIjc3N+RnpiNtkpWVVa1OoLIyASAhIQGzZ8/G7NmzI7K1KuyLdAduGRY0pztch6hJermkRh+NA6kuSLmZMqHTTUyorTqorTokaitlnJCmrVPndNs1zVMqkgIzErj11lvx8MMPAwC++eYb5OXl4ZZbbsGHH36IvLw8vPjiixGVF7HT3bp1axQXB0dSS0tL0aJFi0iLiyrseipopZeHmdNtNqqwq5cz0l1tJNkKyLJXys2UCZ1uYkJt1UFt1SFRWynjhDRtnRrptiv4JRVp16nubNu2zdqp6vXXX8cll1yCBx98EGvXrsVFF10UcXkRPwZ69NFHccMNN+DLL7+0jn355Ze46aabMH369IgNiCbsSmkpTy8PM6fbSi8PM6fbS6e7ukgZ8E2orTrodBMTaqsOaqsOidpKGSekaUunW/86UoG061R34uLicOzYMQDA0qVLccEFFwAAGjduHHIXq6qIONJ95ZVX4tixY8jOzkZMjO/rJSUliImJwVVXXYWrrrrKOrcmG4dLxq5tCqz08nBzusveBq9eXladXL282vjbqrOdJpJS97l6OVcvlwq1VQe1VYdEbaWME9K0derq5XS66XTbyZlnnom8vDwMGDAAq1evxsKFCwEA33//fUQLh5tE7HTPnDkz4h9xCnZFut2VrF7ucrm4ermNSHnKbkJt1cFINzGhtuqgtuqQqK2UcUKatnZEuiXO6fa3Wfc6UoG061R3Zs2aheuvvx6vvfYannnmGbRs2RIA8H//938YPHhwxOVF7HSPGTMm4h9xCrY53YYXcJXP6a4YjTWLD15IzXS6uZBadfG3VWc7TSR1qFJupkzodBMTaqsOaqsOidpKGSekaVsbp7t83SDZkW4J93R2IynTVAJt2rTBO++8E3T88ccfr1F5ETvdJDx2Od3l+3THBJXnc7rN1J8KX3Sb6eWMdFcXKQO+CbVVB51uYkJt1UFt1SFRWynjhDRtOadb/zpSgbTrVAJerxdbtmzBnj17grI/zj777IjKotNtI/all5dVaoUO0Px/+DndTC+PFCkDvgm1VQedbmJCbdVBbdUhUVsp44Q0bel0619HKpB2nerOqlWrcMUVV+Dnn38Oag8ulwulpaURlUen20bsdrpDLaTmc7rL5tsISS/XudFLGfBNqK066HQTE2qrDmqrDonaShknpGlLp1v/OlKBtOtUd6699lr06dMHixYtQvPmzWutJ51uGzHTDmrvdAeml1ec021GuoPndMeWGaJXpFvneTX+tupsp4mkDlXKzZQJnW5iQm3VQW3VIVFbKeOENG1r43Rb29IKX0hNwj2d3XBOt7388MMPeO2119ChQwdbyqv2FdmmTRvk5ubi/fffR0lJiS0/Hm2YTnBtG3pVkW7zfXB6eZnTXc/p5ZIaPbcMU4eUrWBMuGUYMaG26qC26pCorZRxQpq21blX4JZh0Ye0h0O6k52djS1btthWXrUj3a+88grefvttTJw4EXv37kVOTg4uvfRSXHzxxWjYsKFtBknG/i3DwjndZb8HppfXFilP2U2orToY6SYm1FYd1FYdErWVMk5I1bay+1Kml0cfEq9Tnbnhhhtwyy23ID8/H926dUNsbGzA5927d4+ovGo73QMHDsTAgQPx2GOP4dtvv8Xbb7+Np556CuPGjcMZZ5yBSy+9FJdeeinatWsXkQHRhP1zuoP36Xa73X5zuit80Yp00+muLlIGfBNJWQTStKXTTUyorTqorTokaitlnJCqLZ1uZyEpG1ICw4YNAwBcddVV1jGzXblcdbSQWpcuXdClSxdMnjwZ+fn5+N///oe3334bf/vb39CuXTs8/PDDuPjii2tStGhsd7pdPqc73JzuoIXU3KbTXb/p/5IaPed0q0PKzZQJnW5iQm3VQW3VIVFbKeNENGrLOd3Rh6TAjAS2bdtma3m1XkgtMzMTEyZMwIQJE3Ds2DG89957iI+Pt8M2cdjhdBuGUWV6uRnpDnoIyfTyiJEy4JtQW3XQ6SYm1FYd1FYdErWVMk5Eo7aMdEcfEq9TnWnbtq2t5dn6GCgpKQmXXXYZBg0aZGexYrDH6faf0x2cXu5yuQAz0l0xv5z7dEeMlAHfhNqqg043MaG26qC26pCorZRxIhq1pdMd/VCD2vPKK69gwIABaNGiBX7++WcAwMyZM/HWW29FXJbzci8UYqa11CalxTAMuF2Bq6CHi3SHndNdz1uGSUpv4erl6pCyKq0JVy8nJtRWHdRWHRK1lTJORKO2XL08OpHyIEsCzzzzDPLy8nDRRRfhwIED1hzuhg0bYubMmRGXp3+vIQg7It1eb/mkfJcrJqg8/zndwauXcyG1SJHWOVFbdTDSTUyorTqorTokaitlnIhGbauKdEuf0y2hjlQgpU1J4KmnnsJzzz2HO++8Ex6Pxzrep08ffPPNNxGXR6fbRmxJL/fv5Dy+6gleSK2qOd16pZfr/ETY31ad7TSRNPBL6/jpdBMTaqsOaqsOidpKGSeiUdtwf5N5L8RIt0ykZXDqzLZt29CrV6+g4/Hx8Th69GjE5dXYy9iyZQvee+89HD9+HIDMxmk39kS6y1ceD5debj2FrKi5FenWy+nWudFLGfBNqK066HQTE2qrDmqrDonaShknolHbaJ7TLaF+VCGlTUngpJNOwvr164OOL168GJ06dYq4vIhXL//1118xfPhwfPDBB3C5XPjhhx/Qrl07jBs3Do0aNcJjjz0WsRHRgt2RbnPLsHDp5eG3DGN6eXWR1jlRW3XQ6SYm1FYd1FYdErWVMk5Eo7Z0uqMTKW1KAnl5eZg4cSJOnDgBwzCwevVq/Pvf/8a0adPw/PPPR1xexJHuSZMmISYmBtu3b0dSUpJ1fPjw4Vi8eHHEBkQT9jjd/pHu0E63C2aku8KXzfRyoxTwRrZhu51IGpykdU7UVh10uokJtVUHtVWHRG2ljBPRqC2d7uhESpuSwPjx4/Hwww/jrrvuwrFjx3DFFVdgzpw5eOKJJ/DnP/854vIijnS///77eO+999CqVauA4yeffLK1lLpTsSe93G8+ijvcnG7z9yp82UwvB3wp5m4P6oOKf7/Oc6X9bdXZThNJA7+0jp9ONzGhtuqgtuqQqK2UcSIatQ33N5n3QpIXUpNwP6cKzum2j+PHj+Oyyy7DyJEjcezYMWzYsAErVqwI8oGrS8RX5dGjRwMi3Cb79+9HfHx8jYyIFuzZMsxv9XJPmPRyd5inkGakG6jXbcO4ZZg6JD3QkLIVjAm3DCMm1FYd1FYdErWVMk5Eo7bRvGWYhPs5VUh5kCWBP/zhD3j55ZcBAEVFRbj00ksxY8YMDBkyBM8880zE5UXca5x11lmWAYCvQr1eLx555BH87ne/i9iAaMKW9PJS/y3DwqSXh5vTXTHSXU9IeiIsrXOitupgpJuYUFt1UFt1SNRWyjgRjdoyvTw6kdKmJLB27VqcddZZAIDXXnsNGRkZ+Pnnn/Hyyy/jySefjLi8iNPLH3nkEZx33nn48ssvUVRUhNtuuw3ffvst9u/fjxUrVkRsQDRhS2P3S+cJtXq52+22tgwLmtPt9gAuN2B463UxNUmDk7TOSVIWgTRt6XQTE2qrDmqrDonaShknolFbOt3RibRpkzpz7NgxpKSkAPBNrx46dCjcbjdOP/30Gk2pjrg2unbtiu+//x5nnnkm/vCHP+Do0aMYOnQo1q1bh/bt20dsQDRhz5xuX6S71Cjfjzso0m2eG6pD1GCvbkmDk5QB34TaqoNONzGhtuqgtuqQqK2UccIJ2lZ0uiXP6ZZQP6qQ0qYk0KFDB7z55pv45Zdf8N577+GCCy4AAOzZswepqakRlxdxpBsA0tLScOedd9bkq1GNPauX+5xuL9yIKSsmeCE18ylkiALcsQBOaBXp1vlJm7QngpIGfmkdP51uYkJt1UFt1SFRWynjRLRq63K5gu5dzXshyZFuCfdzqpC2VpHOTJkyBVdccQUmTZqE8847D/379wfgi3r36tUr4vIidro//vjjSj8/++yzIzYiWrDF6TZ8T+m8cAc9dTT/by6kVhqUX47yed2MdFcLKQO+CbVVB51uYkJt1UFt1SFRWynjRLRqG8rpZnq5bKS0KQn88Y9/xJlnnondu3ejR48e1vHzzjsPl112WcTlRex0n3POOUHH/Cu1tLT+9oeub+xIa/EaZqQ7dKNxuVyIqdTpNtPL9Yl069zopXVO1FYddLqJCbVVB7VVh0RtpYwT0aptqHPodMtGSpuSQmZmJjIzMwOO9evXr0ZlRZx/8dtvvwW89uzZg8WLF6Nv3754//33a2REtGBLWktp1U53tSLd3DKsWkhLw5GUui9lKxgTbhlGTKitOqitOiRqK2WciFZtQ51Dp1s20ex0z549G1lZWUhISEB2djZWr15dre8tWLAALpcLQ4YMUWtgFUQc6U5LSws6dv755yMuLg55eXlYs2aNLYaJ4cRBAL5N0u1JL/eV4UXozjIg0h1yITX90st1Hpw4p1sd0jp+RrqJCbVVB7VVh0RtpYwT0aptqHPMeyHJC6lJuJ9ThbRgUnVZuHAh8vLyMGfOHGRnZ2PmzJnIycnB5s2bkZ6eHvZ7P/30E/76179aW3/VJ7ZdlRkZGdi8ebNdxcmhYJP1X3sWUisBEOh0V+wUPS6ml9uFlAHfhNqqg043MaG26qC26pCorZRxIlq1ZXp59CGlTUXKjBkzMGHCBIwdOxadO3fGnDlzkJSUhLlz54b9TmlpKUaOHIl7770X7dq1q0NrQxNxpPvrr78OeG8YBnbv3o2HHnoIPXv2tMsuOez5FsAfAdi9kFr49HIPF1KzDWmdE7VVB51uYkJt1UFt1SFRWynjRLRqS6c7+pDSpkwOHz6MQ4cOWe/j4+MRHx8fcE5RURHWrFmDyZMnW8fcbjcGDRqElStXhi37vvvuQ3p6OsaNG4dPPvnEfuMjJGKnu2fPnnC5XEGN8fTTT6/0aUPUUrDB+q8t+3SXlq9ebhKR0+2m0x0J0jonaqsOOt3EhNqqg9qqQ6K2UsaJaNWWTnf0IaVNmXTu3Dng/dSpU3HPPfcEHNu3bx9KS0uRkZERcDwjIwPfffddyHI//fRTvPDCC1i/fr2d5taKiJ3ubdu2Bbx3u91o1qwZEhISbDNKFAXfWv+1o7G7ylYvN/wi3RXnZ1Qe6dYvvVznuTX+tupsp4mkgV9ax0+nm5hQW3VQW3VI1FbKOBGt2oY6h3O6ZSNtTvfGjRvRsmVL633FKHdNOHz4MEaNGoXnnnsOTZs2rXV5dhGx0922bVsVdsjl4C/A8QNAYkNbGrvX63O6S2sa6ebq5REhrXOS9EBDyqq0Jly9nJhQW3VQW3VI1FbKOBGt2nL18uhDyoMsk5SUFKSmplZ6TtOmTeHxeFBQUBBwvKCgIGg7LwDYunUrfvrpJ1xyySXWMdNHi4mJwebNm9G+fXsbrI+MajndTz75ZLULvPHGG2tsjFh2fwW0G2hPYy+7KIzqON1CVi/XudFL65yorToY6SYm1FYd1FYdErWVMk5Eq7ZML48+pLSpSIiLi0Pv3r2xbNkya9svr9eLZcuWITc3N+j8jh074ptvvgk4dtddd+Hw4cN44okn0Lp167owO4hqOd2PP/54tQpzuVwOdbrX2+Z0m6uXl7qqXr28REh6uc6NXlrnRG3VQaebmFBbdVBbdUjUVso4Ea3a0umOPqS0qUjJy8vDmDFj0KdPH/Tr1w8zZ87E0aNHMXbsWADA6NGj0bJlS0ybNg0JCQno2rVrwPcbNmwIAEHH65JqOd0V53GTCuT7FlOzpbGXRaiL/aqm4rxjM9LtrXT1cjrd1UFa5yQpdV+atnS6iQm1VQe1VYdEbaWME9GqbWVOt+Q53RLqRxUVfYZoYfjw4di7dy+mTJmC/Px89OzZE4sXL7YWV9u+fbv2f2/Ec7pJCPZ9D8Aup9vnLPs73eEWUqs80l1ScxtqiaR5x9I6J0kDv5SbKRM63cSE2qqD2qpDorZSxolo1TbUOea9kORIt4T7OVVIW6soEnJzc0OmkwPA8uXLK/3uvHnz7DcoQmrkdO/YsQNvv/02tm/fjqKiwIjqjBkzbDFMFPt+ALxee9LLy5zuEsRax4LSyyuLdLsZ6Y4EKQO+CbVVB51uYkJt1UFt1SFRWynjRLRqy/Ty6ENKm3IiETvdy5Ytw6WXXop27drhu+++Q9euXfHTTz/BMAycdtppKmzUG1cMUHwUOLzLnrSWKtLLAyPdIVJ/mF4eEdI6J2qrDjrdxITaqoPaqkOitlLGiWjVlk539CGlTTmRiPMvJk+ejL/+9a/45ptvkJCQgNdffx2//PILBg4ciMsvv1yFjXrTsI3v333f25PWEiK9PGykO1R/aKaXe+svvVzSvGNpaTiSUvelbAVjwi3DiAm1VQe1VYdEbaWME9GqLbcMiz7odOtLxL3Gpk2bMHr0aAC+vc6OHz+OBg0a4L777sPDDz9su4Ha07id79+939s6p7vEVfWc7kr36dYo0q3z4MQ53eqQ1vEz0k1MqK06qK06JGorZZyIVm0rm9MteSE1CfdzqpAWTHISEV+VycnJ1jzu5s2bY+vWrdZn+/btq5ERs2fPRlZWFhISEpCdnY3Vq1dXev6rr76Kjh07IiEhAd26dcO7774b8LnL5Qr5evTRR61zsrKygj5/6KGHIje+8Um+f/fZ5XT70stLKot0u2Q53To3eikDvgm1VQedbmJCbdVBbdUhUVsp40S0asv08uhDSptyIhE73aeffjo+/fRTAMBFF12EW265BQ888ACuuuoqnH766REbsHDhQuTl5WHq1KlYu3YtevTogZycHOzZsyfk+Z999hlGjBiBcePGYd26dRgyZAiGDBmCDRs2WOfs3r074DV37ly4XC4MGzYsoKz77rsv4LwbbrghYvutSLdNTrcrRKQ7XHp5aKfbXL28uMY21BZJg5O0zonaqoNONzGhtuqgtuqQqK2UcSJataXTHX1IaVNOJGKne8aMGcjOzgYA3HvvvTjvvPOwcOFCZGVl4YUXXojYgBkzZmDChAkYO3YsOnfujDlz5iApKQlz584Nef4TTzyBwYMH49Zbb0WnTp1w//3347TTTsOsWbOsczIzMwNeb731Fn73u9+hXbt2AWWlpKQEnJecnByx/Whir9MdyerllW8ZRqe7OkjrnKitOuh0ExNqqw5qqw6J2koZJ6JVWzrd0YeUNuVEIna627Vrh+7duwPwpZrPmTMHX3/9NV5//XW0bds2orKKioqwZs0aDBo0qNwgtxuDBg3CypUrQ35n5cqVAecDQE5OTtjzCwoKsGjRIowbNy7os4ceeghNmjRBr1698Oijj6KkJPziY4WFhTh06JD1Onz4sO+DRmXp5UcKYJzwHbMlvbySSHeMtZBaqC3Dyr7H9PJqIa1zorbqkOZ0V/VbpOZIamfSoLbqkKitlHEiWrWtzOmWPKdbQv2oQkqbciIRO93jx4+vcgPy6rJv3z6UlpYiIyMj4HhGRgby8/NDfic/Pz+i81966SWkpKRg6NChAcdvvPFGLFiwAB9++CGuueYaPPjgg7jtttvC2jpt2jSkpaVZr86dO/s+iE8BUpoDALwHdwKo5QIOJVUvpOY2I92lobYMq/9It6RVPqWsnGpSsQPV2WZp2kpbvbxieRI0loR/W6O29iFpfJCGRG2l9GHRqm2oc8x/JUe6JdSPKriQmr5EfFXu3bsXgwcPRuvWrXHrrbfiq6++UmGXbcydOxcjR45EQkJCwPG8vDycc8456N69O6699lo89thjeOqpp1BYWBiynMmTJ+PgwYPWa+PGjeUfNusIADB2fAmgtpFun9Nd6gqfXh5TrS3DmF5eHaQ9EaS26pAY6ZamsSSorRok9WHSkKitlHYWrdoyvTz6kNKmnEjETvdbb72F3bt34+6778YXX3yB0047DV26dMGDDz6In376KaKymjZtCo/Hg4KCgoDjBQUFyMzMDPmdzMzMap//ySefYPPmzRg/fnyVtmRnZ6OkpCTs3xAfH4/U1FTrlZKSUv5hX1/5xne+VdTtmNNdWkl6udtlzukOFelmenkkSOucqK066HQTf6itGiT1YdKQqK2Udhat2tLpjj6ktCknUqP8i0aNGuHqq6/G8uXL8fPPP+PKK6/EK6+8gg4dOkRUTlxcHHr37o1ly5ZZx7xeL5YtW4b+/fuH/E7//v0DzgeAJUuWhDz/hRdeQO/evdGjR48qbVm/fj3cbjfS09Mj+hsAAB0vBtqfC8PrmxNem4vcVZYWHs7pdrvd1kJqIafbaJBeLmlwktY5VUyZ0tlmadrS6Sb+UFs1SBofpCFRWyntLFq1pdMdfXBqlL7EVH1KeIqLi/Hll1/i888/x08//RQ017o65OXlYcyYMejTpw/69euHmTNn4ujRoxg7diwAYPTo0WjZsiWmTZsGALjpppswcOBAPPbYY7j44ouxYMECfPnll3j22WcDyj106BBeffVVPPbYY0G/uXLlSnz++ef43e9+h5SUFKxcuRKTJk3CX/7yFzRq1ChyIVwuYPBDMN553/e2NqndXtPpLk8vrzg/I8ZavbyyOd36RLp1bvTSOidJA7+UmykTOt3EH2qrBkl9mDQkaiulnUWrtqHOMe+FJC+kJuF+ThWc060vNXK6P/zwQ8yfPx+vv/46vF4vhg4dinfeeQfnnntuxGUNHz4ce/fuxZQpU5Cfn4+ePXti8eLFlgO/ffv2gAvojDPOwPz583HXXXfhb3/7G04++WS8+eab6Nq1a0C5CxYsgGEYGDFiRNBvxsfHY8GCBbjnnntQWFiIk046CZMmTUJeXl7E9ls0OxVGbAMAx+AySmtcjKsac7rdfnO6DcMIbFSesu8x0l0tpAz4JtRWHSpspNMtF2qrBkl9mDQkaiulnUWrtox0Rx9S2pQTidjpbtmyJfbv34/Bgwfj2WefxSWXXIL4+PhaGZGbm4vc3NyQn4VaKf3yyy/H5ZdfXmmZV199Na6++uqQn5122mlYtWpVxHZWhVHm8LpQi44qgoXUAKDUayDG49eo3HS6I0Fa50Rt1aHCXjrdcqG2apDUh0lDorZS2lm0akunO/qQ0qacSMRO9z333IPLL78cDRs2VGCObLxl87BrldRSll7udVeykJq/020YgZWoQXq5pHnH0tJwJKXuS9kKxkSFvdwyTC7UVg0St16SgkRtpbSzaNU21Dl0umVDp1tfIna6J0yYoMKOqMBw1z7SXb6QWtVzugFfpDsAM71coy3DdB6cOKdbHdI6fka6iT/UVg2S+jBpSNRWSjuLVm05pzv6kBZMchLOvSoVYIvTHWIhtaBIt6saTjfTy6uFlAHfhNqqg0438YfaqkFSHyYNidpKaWfRqi3Ty6MPKW3KidDpthGjLCXchZo/HTQj3V539eZ0Bz2I1CC9XNLgJK1zorbqoNNN/KG2apDUh0lDorZS2lm0akunO/qQ0qacCJ1uG7En0u1zliub0+3xc7qDtg2zIt0lNbahtkganKR1TtRWHXS6iT/UVg2S+jBpSNRWSjuLVm3pdEcfUtqUE6HTbSNG2UJqrlp0VGZ6udcVV36sQgPypZj73pdW/C0r0l1YYxtqi6TBSVrnRG3VQaeb+ENt1SCpD5OGRG2ltLNo1bYyp1vynG4J9aMKKW3KidDpthGvDenlbm9wenmoRRHMaHfwnO6y7dtK9HG6dV7Qwt9Wne00kTTwS+v46XQTf6ht3UBt7UPS+GAipZ1Fq7ahzjHvhSRHuiXcz6mCC6npi3OvSgWYc7rdLhsi3ZXM6QYqcbpj6t/p5pZh6pD0QEPKVjAmquxV+WBHmsaSoLbqoLZqiNZtrXQgWrXllmHRh5QHWU5E/15DEHakl7ursU83AHhcYZzu2ETfvyXHgXrqMCU9EZbWOVFbdaiyV6UO0jSWBLVVB7VVg6TxwUTKtRCt2nJOd/QhpU05ETrdNlK+kFotVi8vc7oNT+hIt/kk0l1VpBuotxXMJQ1O0jonSVkE0rSl0038obbqoLZqkDT2mki5FqJVWzrd0Ye0aZNOgrVhI+VbhtkQ6fZbSC1UCnRMWKc7sfz/xcdrbEdtkJQCLa1zkjTwS7mZMqHTTfyhtuqgtmqQND6YSLkWolXbyuZ0S15ITcL9nCqkTZt0Es69KhVguD0AAJdR+4XUwkW6g+Z0B61eHgug7Px6mtctaXCSMuCbUFt10Okm/lBbdVBbNUgaH0ykXAvRqi0j3dGHlDblROh024jhqn16ueV0u6rndJeUVugUXa7Aed31gKTBSVrnJMFGE8na0ukm1FYd1FYNksZeEynXQrRqS6c7+pDSppwInW4b8bp8kW53LdLLPUZZerkn/D7dQPlCat5QnWI9r2Auad6xtDQcSenwUlalNVFlr0odpGksCWqrDmqrhmhdYVsHolVbrl4efdDp1hf9ew1BWHO6jdIal2FGulHVPt2eskh3xTndQPm8bs7prhJJTiwgqzOVZCvASDcJhNqqg9qqIVqjsToQrdpyTnf0IS2Y5CSce1UqwNoyrDYLqUUa6Q7pdJuR7hM1tqM2SBqcpAz4JpLslWQrQKebBEJt1UFt1SBp7DWRci1Eq7ZML48+pLQpJ0Kn20ZqHen2euExv+upYp/ucKuXA35zuul0V4W0zkmSvZJsBeh0k0CorTqorRokjb0mUq6FaNWWTnf0IaVNORE63TZiuMzVy2vYUZmp5QDgriLSXZnTbUa6i+l0V4W0zkmSvZJsBeh0k0CorTqorRokjb0mUq6FaNWWTnf0IaVNORE63TZipZfXNNJdWmT911vllmG+qgvaMgwon9PNSHeVSOucJNkryVaATjcJhNqqg9qqQdLYayLlWohWbel0Rx9S2pQTodNtI15X2cqPNd0yrLQ80u2qaiG1skOhF1LTa063zgta+Nuqs50mkjpTSbYCdLpJINRWHdRWDdHqGOpAtGob6hwupCYbLqSmL869KhVgRrrdNXa6fZHuEsMNd1Vzusv+DbmQWj3P6eaWYeqQ9JBAylYwJtwyjPhDbdVBbdUhaYwA5FwL3DJMBox0y3mQ5UT07zUEUb6QWu2c7mLEwO0O3WgqzukOHelO8P3LOd1VIq1zkmSvJFsBRrpJINRWHdRWHdK0lWKvpPsaE6aXOxMpbcqJ0Om2kfKF1Go6p9uXXl4MD2Kq6XSH3jKszOnWJL1c50YvrXOSZK8kWwE63SQQaqsOaqsOadpKsVfSfY0JnW5nIqVNORE63TZiuE2n24ZId5gUMfP/lUa6Y/VyunVOw5Kciqd7Zyqt46fTTfyhtuqgtuqQpq0Ue53kdHNOt2ykTNlwIqwNG6l9pLvc6fZUN9IdcvVyM738eM3sqCWSBicpA76JJHsl2QrQ6SaBUFt1UFt1SNNWir2S7mtMGOl2JlLalBOh020j1pZh8AI16azM9HKjOk63r+pKSitLLy+M3AYbkDQ4SeucJNkryVaATjcJhNqqg9qqQ5q2UuyVdF9jQqfbmUhpU06ETreNeMvkdLsQsOd2tSn7TlGF9PKQTnfZodD7dJtOd/1Eurl6uTokpcNLS3Hi6uXEH2qrDmqrDmnaSrGXq5fLgE43nW6d0b/XEIQ1pxuo2XzqgPTy8sOVRbpLK53TzUh3VUjrnCTZK8lWgJFuEgi1VQe1VYc0baXYK+m+xqS2kW7Jc7ol1I8qpLQpJ0Kn20YMmE8JUTOH12/18nALqZU73WVfqXTLMD3mdOv8RFhS5BiQ1ZlKshWg000CobbqoLbqkKatFHud5HSb90KSI90S7udUIS2D00k496pUgNk92RPprjy9PMZcXbLS9HJGuqtCyoBvIsleSbYCdLpJINRWHdRWHdK0lWKvpPsaE87pdiZS2pQTodNtI1ZjB4CSmszpNiPdVTvdbnPLsEoXUtMj0q1zo5fWOUmyV5KtAJ1uEgi1VQe1VYc0baXYK+m+xoROtzOR0qacCJ1uGylv7KhhpNvndBcZVS+kFlPZlmGc011tpHVOkuyVZCtAp5sEQm3VQW3VIU1bKfZKuq8xodPtTKS0KSdCp9tGAiPdNZnTHTq9PNT8DNMpLxEwp1vnRu9vm4Q5QJI6U0m2AnS6SSDUVh3UVh3StJVir6T7GpOaOt3mvZDkhdQk3M+pgnO69cW5V6UCrMZe40i3n9NdRaS7Wgup1VOkW1KDl2QrIOshgZStYEy4ZRjxh9qqg9qqQ5q2UuyteH+gs60m3DLMmUh5kOVE9O81BGFXenkxYqw5277yQjndlWwZptGcbt0bvCRbAVn2SrIVYKSbBEJt1UFt1SFNW0n2SrIVYHq5U5F2nToJOt02Yld6eVG19uku+0pl+3QX18DxtwFJDV6SrYAseyXZCtDpJoFQW3VQW3VI01aSvbrbVxE63c5EUptyGnS6bSQg0l1aizndFRZSC5X+E1OtSHf9O926p2BJshWQlQ4vreOn0038obbqoLbqkKatJHsrOqW6wzndzkTKlA0nwtqwkdpHus30ck/VW4aV/Vta6T7d9e906z44SbIVkGWvJFsBOt0kEGqrDmqrDmnaSrLXKU43I92ykdSmnAadbhup/Zzu6i+kFuMpc7pDppcn+v4tOQHUQ6cpqcFLshWQZa8kWwE63SQQaqsOaqsOadpKspdOt/7Q6ZbVppwGnW4bMdNaah7pLp/TXdVCalakO2R6eXz5/+thBXNJDV6SrYAseyXZCtDpJoFQW3VQW3VI01aSvXS69YdOt6w25TTodNuI2djdLletVy+vMtLtrszpTiz/fz2kmEuadyzJViDQRt3n6kibV8Qtw4g/1FYd1FYd0rSVZK9pn+52mnDLMGdCp1tfZPQcQghML6/dQmr+c7pDOYbuypxuTyzK4u314nRLcgwl2QrI6kwl2Qow0k0CobbqoLbqkKatJHudEunmQmqykRZMchLOvSoVELiQWu3mdFeVXm5GuktCOd0uV+C87jpG4iBa8f+6IsleSbYCdLpJINRWHdRWHdK0lWSvU5xuRrplI6lNOQ063TZS+0i3L728qBrp5WYk3BvK6QbK53XX017dJro3eGmdkyR7JdkK0OkmgVBbdVBbdUjTVpK9dLr1h063rDblNLRwumfPno2srCwkJCQgOzsbq1evrvT8V199FR07dkRCQgK6deuGd999N+DzK6+8Ei6XK+A1ePDggHP279+PkSNHIjU1FQ0bNsS4ceNw5MiRWv0dtd8yzD/SXX64Mqc75JZhQPm87nreNkz3Bi+tc5JkryRbATrdJBBqqw5qqw5p2kqyV8p9jQmdbmciqU05jXp3uhcuXIi8vDxMnToVa9euRY8ePZCTk4M9e/aEPP+zzz7DiBEjMG7cOKxbtw5DhgzBkCFDsGHDhoDzBg8ejN27d1uvf//73wGfjxw5Et9++y2WLFmCd955Bx9//DGuvvrqWv0tqrYMCzU/w/w85JxuoDzSXc9Ot+7zalxhdNYVSZ2pJFsBOt0kEGqrDmqrDmnaSrLXKU4353TLhnO69aXer8oZM2ZgwoQJGDt2LDp37ow5c+YgKSkJc+fODXn+E088gcGDB+PWW29Fp06dcP/99+O0007DrFmzAs6Lj49HZmam9WrUqJH12aZNm7B48WI8//zzyM7OxplnnomnnnoKCxYswK5du2r8t1iNvZbp5cXwBCykFrpTrMLpNud0Fx+P3A4bqLgKpq5I65wkPSSQtCotwNXLSSDUVh3UVh3StJVkL1cv1x9GumU9yHIa9dpzFBUVYc2aNRg0aJB1zO12Y9CgQVi5cmXI76xcuTLgfADIyckJOn/58uVIT0/Hqaeeiuuuuw6//vprQBkNGzZEnz59rGODBg2C2+3G559/HvJ3CwsLcejQIet1+PDhoHPsWkityKj+QmpVR7rrfp9uQM4TYWmdkyR7JdkKMNJNAqG26qC26pCmrSR7pdzXmDC93JlIalNOo16d7n379qG0tBQZGRkBxzMyMpCfnx/yO/n5+VWeP3jwYLz88stYtmwZHn74YXz00Ue48MILUVpaapWRnp4eUEZMTAwaN24c9nenTZuGtLQ069W5c+egc+xaSK06+3RXGem25nTXT6RbyuAkrXOSZK8kWwE63SQQaqsOaqsOadpKslfKfY0JnW5nIqlNOY2Y+jZABX/+85+t/3fr1g3du3dH+/btsXz5cpx33nk1KnPy5MnIy8uz3u/cuTPI8Q6IdJfWfCG1kkrSy830n0q3DAPUR7oNo+zpQmikDE7SOidJ6fDStKXTTfyhtuqgtuqQpqeka0HKfY1JbZ1uyXO6pdSRCiRNQ3Qa/8/eecc3Ve5//HOSNk13aaEtBQpl7z0ERVBRQJw4kIuAC8cVF4r7gv6894Ibr3vhuiI4kKuiDNmyp0yZZdMyu3dyfn98c3JO0qRN25zmPMn3/XrlleTk5OTJ94znfJ7veAK6Nxo2bAiz2Yzs7GyX5dnZ2UhNTfX4ndTU1BqtDwAtW7ZEw4YNceDAAec23Au1VVRU4Pz58163ExERgbi4OOcjNja20jp193Q7wssRBpOXk0Y5mRRPuN3bSKSeOd17fwP+1RhY94HXVdwLchgV0S5OIt6guL82Kiy6GS1sW/1g2+pHMApDoxCMtvUcSUn3QiJ7ukW4n9MLkZwzoUZAj0qLxYJevXphyZIlzmV2ux1LlixB//79PX6nf//+LusDwOLFi72uDwDHjx/HuXPn0LhxY+c2cnJysHnzZuc6S5cuhd1uR79+/Wr9f2qU011WCLiPImrDy6vJ6TYHytNdfAH46WEKW18xnf6HB0TpnETq8N0xentFsy2LbkYL21Y/2Lb6IUrfqyDSsRCMtuXw8uBDpHMq1Aj4UNCkSZPw8ccf44svvsCePXvwwAMPoLCwEHfeeScAYNy4cXjmmWec6z/yyCNYsGABXn/9dfz111944YUXsGnTJkycOBEAUFBQgMmTJ2PdunU4fPgwlixZguuvvx6tW7fG0KFDAQAdOnTAsGHDMGHCBGzYsAGrV6/GxIkTcdtttyEtLa3W/0UNa0HVYvf0HuDlDODbsRSmraBMGSaHwezjPN32+srptlUA5w8Bvz0FFDqiBIovAFv/63F1UTonES9ObFt9YNHNaGHb6gfbVj9E6R8URDoWgtG2LLqDD5HOqVAj4KJ71KhReO211zBlyhR0794d27Ztw4IFC5zF0o4ePYpTp0451x8wYABmzZqFjz6es106AAEAAElEQVT6CN26dcP333+PefPmoXPnzgAAs9mM7du347rrrkPbtm1x9913o1evXli1ahUiIiKc2/n666/Rvn17XHHFFbj66qtxySWX4KOPPqrTf3GGtUio2tO99b+U8/3XL8CmT9Xlmnm6TV5Omsqebi85N3XxdNvKyYOtPP6aD/ynO/CfHsD2ObROt9H0vPZdEuRu8JRh+iFK6L5IU8EAPGUY4wrbVj/YtvoRjNNaGYVgtC1PGRZ8BLPofvfdd9GiRQtYrVb069cPGzZs8Lruxx9/jIEDB6JBgwZo0KABhgwZUuX69YEhCqlNnDjR6al2Z/ny5ZWW3XLLLbjllls8rh8ZGYmFCxdW+5uJiYmYNWtWjdpZHS7h5UXngT0/A3kngQuHAdkOxKUBFz0I7JqnfmnBs8Dy6ZSDnXMUAOV0a8PLPeZ0Oz3dXhpT05xuWzmw5EVg3yLg3AFAtlVeJ8zq+A9/B7qPAfYvAnKOAL9PBQY/A0TEVGqn0Tsn0XK6AXFG20W78LOnm9HCttUPtq1+iNI/KIh0LASjbavK6Ra5kJoo93N6IKIzyRfmzJmDSZMm4YMPPkC/fv0wY8YMDB06FHv37q00IxVA+nH06NEYMGAArFYrXn75ZVx11VXYtWsXmjRpEoB/YBDRHSy4FFIryQHm3F55pZPbgLzjgCUGaNwNOLIaKDzjskrNcrq9ebqt9OzLfOG2CuCHe4Dd8zx/bo4A+v8duPRJwBKlLr/4EWDxFGDtOzTAcN8KILKBSzuNfsKL1OErsG31gUU3o4Vtqx9sW/0QpX9QEOlYCEbbcnh58CHSOVUT3njjDUyYMMGZfvzBBx9g/vz5mDlzJp5++ulK63/99dcu7z/55BP88MMPWLJkCcaNG1cvbXaHRbcfcZ7s7a4GOiUAFzKBmFSgYRug6Byw7Wtg11xaue0w4Jo3gYNLAckEfHeH07vsk+h2PNu8XRN9Fd3lJcCP9wK7/weYwoFr3wJaDgYiE5RfBMzh9HBnwMNAQjrw29Pk8d7xPdB3gks7jX7Ci3hxYtvqA4tuRgvbVj/YtvohSv+gINKxEIy2ZdEdfIh0TgFAfn4+8vLynO8jIiJc0oEBoKysDJs3b3ap8WUymTBkyBCsXbvWp98pKipCeXk5EhMT/dPwWsCi2484T/YmPYBbprh/COQeAzJX0vtONwDWOHoGgL73AuvfB+AQ3dXldJuV8HJvU4Y5RHe5Q3QXXwBObKZ2nPoTOLqOvNLnD9JyUzhw6xdA+xG+/2FJAjrdSCH0C58Ftn+rEd1weTYqol2cAHE6ftFsy6Kb0cK21Q+2rX6I0j8oiHQsBKNtWXQHHyKdUwDQsWNHl/dTp07FCy+84LLs7NmzsNlsznpfCikpKfjrr798+p2nnnoKaWlpGDJkSJ3aWxdYdPuRKk92SQKumQF8eClgiQZau+30y56B/Oc3yCmuwFk5DqbqcrolJbzcW/Vyh+guPA2cPQB8eiVQfN7zuhHxwG3/BTIurf5PeqLzTcCi54HjG4BzB4GkVpBsVMDNVJpXzZcDC+d064doF34W3YwWtq1+sG31Q5T+QUGkYyEYbcs53cGHaDndu3fvdsmxdvdy+4Pp06dj9uzZWL58OaxWq9+37yssuv1ItSd7UivgwQ2AKUwtdKZgjYf94W0Y/H+LUIKI6j3d1U0ZltyBng8uBT4aBJQVUKh7TDIQ3wxoOYgqk5fkAj1upxD42hKbSiHpB5cCO74DBj8NExwDEGUFVMzN/f8aBNEuTkDljtGoiFSVFuDq5YwrbFv9YNvqRzBW2DYKwWhbrl4efIg0kAUAsbGxiIuLq3Kdhg0bwmw2Izs722V5dnY2UlNTq/zua6+9hunTp+P3339H165d69zeuiDGlUMQfDrZ45sAsSkeP7JZ4pELqgBuqmshtYxLgWEvU754WQHlXt+/ih6jZwH97gMGTgKufLFugluh6230vOM7aqciumEH9vxS9+3rhGgXJ0Cc0XbRbMuebkYL21Y/2Lb6IUr/oCDSsRCMtuXw8uBDpHPKVywWC3r16oUlS5Y4l9ntdixZsgT9+/f3+r1XXnkFL730EhYsWIDevXvXR1OrhD3dfqSuJ7tdc4HztXq5N0c3AOCi+4GGranA2cAnyMutFy0H0/O5g4DdphHdALZ8AXT1PMVboBHx4iRKxy+abVl0M1rYtvrBttUPUfoHBZGOhWC0LYvu4EOkc6omTJo0CePHj0fv3r3Rt29fzJgxA4WFhc5q5uPGjUOTJk0wbdo0AMDLL7+MKVOmYNasWWjRogWysrIAADExMYiJifH6O3rCotuP1PVkt2kUtK/h5V493Qqth1TOH9eDKKUaoAwUX4AkU7skSQIOr3LmehsNES9OonT8otmWRTejhW2rH2xb/RClf1AQ6VgIRtuy6A4+RDqnasKoUaNw5swZTJkyBVlZWejevTsWLFjgLK529OhRl3SJ999/H2VlZbj55ptdtuOpUFt9waLbj9RZdGsucNr0G085N4ooN0ydC3M4YE2g+ckLzzo93Saz4xDLXGl40S1anpbRL6aiXfjrQ3T7G9FsLBJsW/1ge+pHMApDoxCMtvW0DhdSExuR6iTUlIkTJ2LixIkeP1u+fLnL+8OHD+vfoBoSXHsjwNQ5vFwPT3d9EpVEz/mn1PDyMEcVQoNWMRepw1cQpeMXzbZ6i249bCCajUWCbasfolzDREQ024p0ngWjbdnTHXyIdE6FGiy6/Yhfw8t9zOm2GUhzI7ohPZ8/CKXFkuLpLi0ISJOqQ8SLkygdv2i2ZdHNaGHb6oco1zAREc22Ip1nwWhbFt3Bh0jnVKjBotuP1DWsxea8WHg/aZTXYU7RbSDVrXi6zx6AMmYgmcz0osyYopunDNMP0UKc9J4yTA8biGZjkWDb6odoUy+JhGi2Fek8C0bb8pRhwQeLbuMixpVDEOoeXk7PZrfvexKGJqfoNtBFURHd5/ZD+Qsms0N0l+YHpk3VIGJOtyij7aJd+NnTzWhh2+qHKNcwERHNtiKdZ8FoW87pDj5EdCaFCqF7VOqAvwqpaefodt9eZU+3gUS3El5+dr8aXm4SR3SLcnESpeMXzbYsuhktbFv9EOUaJiKi2Vak8ywYbcvh5cGHSOdUqMGi24/4q5Cau6fb40ik49lmpIui4unOOer0dEtSDcLLywrpUY+IeHESpeMXzbYsuhktbFv9EOUaJiKi2Vak8ywYbcuiO/gQ6ZwKNXjKMD/ir0JqZh883WYjerqjHJ5uyB483dWIbrsNeO8iev3wNkD5ns6IeHESpeMXzbYsuhktbFv9EOUaJiKi2Vak8ywYbcuiO/gQ6ZwKNVh0+xG/hZe7fV2Y8HLF0w2onm6zj57u4gtAzlF6XZILRCXq0MDKiHhxEqXjF9G2Ciy6GbatfohyDRMR0Wwr0nkWjLatSnSLnNMtyj7SA5HOqVCDw8v9SF1PdrsXT3dVhdTssoFGI6M1otvxbHJOGVZNTndJrvq6HvO/tftKlMIbonT8Il749bAti24xYdvqhyjXMBERzbYinWfBaFuP6YuOeyHD3FvWANlZG0mM+zk94EJqxiV0j0odqOvJrni6fQkvD9OsYxhvt8bTXWnKsOqEdGme+roepxcT8eLk3jEaFZGmglHQY0oYnjJMTNi2+iHa1EsiIZptRTrPgtG2PGVY8CHSQFaoIcaVQxD8ldNtkqoX3doK54YppubM6daMCPs6T7dWlNdjMTURL06ijLazbfXbpvu29dp+KMO21Q9RrmEiIpptRTrPgtG2nNMdfIh0ToUaLLr9iN/m6RbV022JBswRAKAppOYIL7eVARVl3r9bovF0Byi8XJSLkygdP9tWv226b1uv7YcybFv9EOUaJiKi2Vak8ywYbcuiO/gQ6ZwKNVh0+xH/FVLzntPtDG/SrGMY0S1Jzrm6leY5c7qBqr3dAQovFzGn2z0EzKiIeOFn0c0osG31QzTxIhKi2Vak8ywYbVtVTrfIhdREuZ/TA5FSNkIN3ht+pD6nDDOkpxtwVh1XPd0mICyS3miFtTsunu7AiG7ROlKjt5dtq9823bet1/ZDGbanfohyDRMR0Wwr0jUsGG3Lnu7gQ6RzKtRg0e1H6hxeXoNCambDim5XT7ckSUBEDL2pSkwbwNMtysVJlI6fbavfNt23zfgfUc4zEWHb6odothWpnwhG27LoDj5EOqdCDRbdfqSuYS1qITXX5d4uisp6xhLdVMHcWb1ckoCIWHrja3h5PeZ0c/Vy/RAxxEnU6uWi2Fck2Lb6wbbVD9FsK1I/EYy25erlwQeLbuMixpVDEOpeSM33ebq16xmmejmg5nQ7qpabTCbA4oOnWxteHqDq5aJ0pKKMtot44RfV0y2KfUWCbasfbFv9EM22IvUTwWhbzukOPkR0JoUKoXtU6oBehdS8XTgV0V1hM5Dodni6FdHt6ul2eLDLCoHFU4GTW9XvcXi5z4jS8bNt9dtmfWw71GHb6gfbVj9Es61I/UQw2pbDy4MPkc6pUINFtx+pz0JqAGB2vLYb6cLoFN1UtVySJI2n2yG6d80DVs8Afn9R/R4XUvMZUTp+tq1+26yPbYc6bFv9YNvqh2i2FamfCEbbsugOPkQ6p0INFt1+pD4LqQGASfF0Gymnu1lfwBwBKSIagOLpdgsvP3eAns8fVL+nzeMu43m6q0KUjp9tq98262PboQ7bVj/Ytvohmm1F6ieC0bYsuoMPkc6pUINFtx+p68muhIn7Gl6uTBtmN5LoTu0CPH0EUmwqAPfwcofovpBJz7nHAVs5vS5lT7eviNLxs23122Z9bDvUYdvqB9tWP0SzrUj9RDDalkV38CHSORVqsOj2I3p5uoUqpAYA4ZHOdroWUnN4sC8cpmfZDuQeo9clgc/pFqXwhigdv4gXfhbdjALbVj/Ytvohmm1F6ieC0bae1uFCamLDhdSMS+gelTpQ9ynD6NnsdpJ4u3AaspCaA5epJxRPtyK6z2eqKyoCPECebhEvTu4do1ERaSoYBZ4yjFFg2+oH21Y/RLOtSP1EMNqWpwwLPkQayAo1xLhyCILfqpe77RWvotvx2lCF1By4jAgrnu6yAqD4AlCSo6544TBgt7l6twM0ZZgoFydRRtvZtvptsz62HeqwbfWDbasfotlWpH4iGG3L4eXBh0jnVKjBotuP+Gue7jA31e1VdJsdnm4j5XQ7cLl4Oz3dBapnW+HCYdciagAXUqsGUTp+tq1+26yPbYc6bFv9YNvqh2i2FamfCEbbsugOPkQ6p0INFt1+xF9ThpmqyOnWvnZ6ug0suk0mk1q9vMyb6M5zXRagQmqihYwZ/WIq4oWfRTejwLbVD7atfohmW5H6iWC0Led0Bx8ipWyEGrw3/Ii/wsvNbl+vNqfbwKKbwssVT3eems9tTaDnC4fVImrmCHq2lwMVpfXaTvfXRkaUjp9tq98262PboQ7bVj/Ytvohmm1F6ieC0bbs6Q4+RDqnQg0W3X7EX+Hlvs7TbTbilGEOXMPLNfN0K9OFtRxEz9rw8rjG6gbqydst4sVJlI6fbavfNutj26EO21Y/2Lb6IZptReongtG2LLqDD5HOqVCDRbcfqXP1cqWQmttJ4l100+8Y0dPtEgJt8RBenuEQ3SW5QM5Rem1NAMKjHOvWT143Vy/XDxFDnLh6OaPAttUPtq1+iGZbkfqJYLQtVy8PPlh0GxcxrhyCUP+ebno23DzdcPd0awqpnT9Mr1M6ATEp9DprOz1b4wBLNL2upwrmIl6cRBltZ9vqt8362Haow7bVD7atfohmW5H6iWC0bVWebpFzukXZR3og0jkVarDo9iP1UUjNk6fbZsB5ul28sYroLi8E8o7T6wYt6AEAp/6k54g41SsegPByUUavRen4Rbzws+hmFNi2+sG21Q/RbCtSPxGMtvW0jnIvJLKnW5T7OT0QMYIzVAjdo1IH6l5IjZ7Nbt/36umWlO8Z78Lo6umOA8Ii6QPZTu9jUlTRrXi6I+I0lc7rJ7xcpA5fQZSOn22r3zbrY9uhDttWP9i2+iGabUXqJ4LRtpzTHXyIdE6FGmGBbkAwEahCajYD5nS7XLzDLMDfZgOZK6loWqsrAEkCkjvQyiW59GyN01Q6D/FCajnHgKNrgc43A17mbTdUez1gWNtWAYtuRoFtqx9sW/0QzbYi9RPBaFsW3cGHSOdUqMGi24/4a8ow3wupCSK6AaDlYHpoadLL9b2Lp9shugvPAvsWAvYKEunN+urSTvfXAWfB08Bfv1BhuQ7XqMvLioTp+A1r2ypg0c0osG31g22rH6LZVqR+Ihhty6I7+BDpnAo1WHT7EX/ldJvdgv6953QbX3RXmVfTuDsACYCj/VYPOd3zJwG7/0evTWHApL+AmEZ+b2e1ba1vco7Q84nNqug+dxB4fwAkx7TmRr+YinjhZ9HNKLBt9YNtqx+i2VakfiIYbetpHeVeSORCaoa6n6tnOKfbuITuUakDdT3Zax5e7iikZkDR7T71hEescUCjdur7CG31ckdO97EN6uf2CnWebwCoKAXO7POtQWVFgK3cazurbWtNObIGmP84UJJX/bontwJZO12XFefQc/YuddmhZUBFCSSHbYzeqYg0FYwCTxnGKLBt9YNtqx+i2VakfiIYbevpXo093WIj0kBWqGGIK8e7776LFi1awGq1ol+/ftiwYUOV63/33Xdo3749rFYrunTpgl9//dX5WXl5OZ566il06dIF0dHRSEtLw7hx43Dy5EmXbbRo0QKSJLk8pk+fXqf/Ue/h5ZLr94yEzyPCaT3V1xGxaqXzskKg8ByQf4reJ3ek5/wsepZl4JvbgHf7AEfXkbh9uzcwoyuw5CUgP1vd7pG1wJudgBld6LWCrQJS4ZlKba4z5w4CX98KbPwE2PVj1eseWgF8fDnw+dWugwLFF+j59G512em/qJ32Cv+2VydEvPCzp5tRYNvqB9tWP0SzrUj9RDDa1tN/YtEtNiKdU6FGwEX3nDlzMGnSJEydOhVbtmxBt27dMHToUJw+fdrj+mvWrMHo0aNx9913Y+vWrbjhhhtwww03YOdO8hQWFRVhy5Yt+Mc//oEtW7Zg7ty52Lt3L6677rpK2/q///s/nDp1yvl46KGH6vRf6r+QmnE93T53Tk00ots9vDx7B71ukAEktqTXBQ4xvWsucHApvT68Cji2Hji3n8KyV70G/HgffbZvEfDVDUDxeRLwn49QhfD//g7p86srtblOlBcD341XPfU5R+k5ayewfzF5rpWOLOcY8P2dVNG9JBcoOkfLK8rUnPbcY6rX+4xDdMs2/7VXR0S88LPoZhTYtvrBttUP0WwrUj8RjLZl0R18iHROhRoBF91vvPEGJkyYgDvvvBMdO3bEBx98gKioKMycOdPj+m+99RaGDRuGyZMno0OHDnjppZfQs2dPvPPOOwCA+Ph4LF68GLfeeivatWuHiy66CO+88w42b96Mo0ePumwrNjYWqampzkd0dLTXdpaWliIvL8/5yM+vPKWVXp5u7zndju8ZWHRXG4alLaYWEe9aSC3LIbpTOwOxqfS6IJsE+cLn1e+d3gOc3U+vY1Lo+awj7HzB00BFCdB2ONDxBkC2Aavfos+ObYDW1H4JGdvyldpuAMg9DuSdAj4aDHx9M/D+AGCRo+2/PKYKbYCKxgGql1v7/zTPSpuNfjEV8cLPoptRYNvqB9tWP2pl242fAJs833PpjUj9hGjHbW1Ft8jzdHNON+d0G5mAHpVlZWXYvHkzhgwZ4lxmMpkwZMgQrF271uN31q5d67I+AAwdOtTr+gCQm5sLSZKQkJDgsnz69OlISkpCjx498Oqrr6KiosLrNqZNm4b4+Hjno2PHjpXWqXshNXquytOtPZnCgsHTndIZMEfQ68gGqqe7+IKa55zSRRXTBdnAli+A/JOAKZyWnd5DXm4AaH0lPeefojzu84fo/bVvAZdOptc5RwG7Hcg7AW3r/HJxOu3IwW6QQc+5x8m7bdeEjh9cRr9/ZDW9D7PSc5EX0Z29kwS543OTIB2/SDdTCiy6GQW2rX6wbfWjxrYtPEf1R355jFKj6hmR+gnRjtu6eroB8YQ3e7q9awYm8AR0b5w9exY2mw0pKSkuy1NSUpCVleXxO1lZWTVav6SkBE899RRGjx6NuLg45/KHH34Ys2fPxrJly3Dffffh3//+N5588kmvbX3mmWeQm5vrfOzevbvSOnUOL5drFl5uEqB6ebW2CLMAI14HLn4ESGoFpHah5Yf/AE5sotepnVXRnZ8NnNpOr3vcTs9n9wHZjv3RfACJcdkOHN8AQKZpt2KSgYRmtE7ROSrIZivzf4d/4Qg9t7iEnvOOq8XflP92dh9w/iBQXkQDDo27q+0CPHi6d6vebsA5UGD0TkWkmykFFt2MAttWP9i2+lFj2yozZQDA7nn+b1A1iNRPiHbcsugOTUQ6p0KNoJ4yrLy8HLfeeitkWcb777/v8tmkSZOcr7t27QqLxYL77rsP06ZNQ0RERKVtRUREuCzPy6tclVoJa6m9p7tmhdTCgkF0A0DPserrpn2AhHTyRp87QMtSu6iiuiBLFactBwE7vqf86eOO4nuN2gFxaXQjcfgPWpbQnGKyrfEUwl6aCxylyAi/e7ovHKbnFpcAW78Cck+o3oMWlwLnD1N7lWnQGrWlAQGAPA4A5Z9ryd6lFpIDh5frCYtuRoFtqx9sW/2osW1zj6mvd/4IDHxch1Z5R6R+QrTjlkV3aCLSORVqBFR0N2zYEGazGdnZ2S7Ls7OzkZqa6vE7qampPq2vCO4jR45g6dKlLl5uT/Tr1w8VFRU4fPgw2rVrV+W63lBO9tqGc3ibp9urp9vx+vXFe/He8gO1+k1/0SY5Fl/d0xcRYWYAPk4Z5glJAjrfDPzxBr23xgPxzVShXXCacrQBIKkNkNyBBLfsiM1Pak3r5xwBDjvCtxu0ULef0AzIznVWMdcGFdT54mS3UTg5ADTrB0gmCis/ts7RtpZqe3f+QMuSOwGWKHrtHl4e15Q85dm7gRQlskJyDhQYPWxIpKlgFHjKMEaBbasfbFv9qLFtczSiO3sH1Udp2EaHlnlGpH5CtOO2JlOGecsDZtEtHiy6jUtArxwWiwW9evXCkiVLnMvsdjuWLFmC/v37e/xO//79XdYHgMWLF7usrwju/fv34/fff0dSUlK1bdm2bRtMJhOSk5Nr+W/8GF7u9n1vF8NOaTSQUFJux4Wi8oA+Nhw+j/3ZBZXaWavOqcvN6uuUziTEneHlWaooTXSIWIXoZCAyAYhvSu+V8HSt6I53hJgfXeNop/pRnTvSvJMksk3h5K2PTaPlp/50tCNDba8yFVhyByCqIb12L6TWrA+Fn5flqxXX03pU9nTnZwFFbt5xAyDihZ893YwC21Y/2Lb6USdPN6D2NSV5aiqXjojUT4h23NbW0629F1IiOEWBC6lxITUjE/Dw8kmTJmH8+PHo3bs3+vbtixkzZqCwsBB33nknAGDcuHFo0qQJpk2bBgB45JFHMGjQILz++usYMWIEZs+ejU2bNuGjjz4CQIL75ptvxpYtW/DLL7/AZrM5870TExNhsViwdu1arF+/HpdddhliY2Oxdu1aPPbYY7j99tvRoEGDWv+XuhdSUzzlvoWX33VJBq7smILSClutfs9f3PHZRhy/UIzicrUddeqcUjpROPXp3SS6ASC6ESgY3DHqGptGHmJN2LVzdF4R3bYyem7QXF1Hyet2FFjza3i5Elqe0AwwmYH4JuSpVrzwiRn037SkdFKLvSmebkVAx6QAfScAa99RhXiryyFhldreYxuBL64FYhoBD/8JGKijEelmSoFFN6PAttUPtq1+1Dyn2yG603oAJ7cCe38FBj0J/DoZ2D4bGPcTpXLphEj9hGjHrVDh5WWFwI7vgM43ARGxtd4Me7rFOqdCjYCL7lGjRuHMmTOYMmUKsrKy0L17dyxYsMBZLO3o0aMuozYDBgzArFmz8Pzzz+PZZ59FmzZtMG/ePHTuTOLsxIkT+OmnnwAA3bt3d/mtZcuWYfDgwYiIiMDs2bPxwgsvoLS0FBkZGXjsscdc8rxrg16e7qpOoGaJUbX6LX8SZw0HUIyiMj+JbgAY/AxNrdXtNnpvDgeiklRhmtSKnrWebqfobuK6LZfw8nSXj7TNc7b1fCZ5xM01PD2UgjQJDpEf35TmDwcAyUzb1LZXaX9JLr0udCukFtmACsztW+DIb5eAjEtVT3dpPjB7NFBRTDnweSfUQQUDIOKFn0U3o8C21Q+2rX7U3NPtmEq1620kurN3A7Zy4NByWn7gdxbdDkQ7boUS3Yv+AWz6lO6/rnyx1pth0S3WORVqBFx0A8DEiRMxceJEj58tX7680rJbbrkFt9xyi8f1W7RoUe1FomfPnli3bl2N21kd9e3pNgqRFsrjLvan6O54HT20xKZ6EN0aT3eSIrrdhKen8HKlndrXkgTs+RmYczsw6Gngsmfog6LztCypNXDdf4CKMiBrO5DW09WzrHi6ld9TPO4AiWFzuGt7I+KBuCZAtCO83D2nO7IBEB4JXP8e8MU1QONuQGJLtXr5gcVASq66vTN7WXTXERbdjALbVj/YtvpRc9HtqEPS4hLAEkvpTEfWUNFSADixRYdWqojUT4h23AojusuKyMsN0LEHUASGrUy91/MRFt1inVOhhnFiUYMAvebp1m7TiCdQlEN0l3gIL/drXk2MJt8+0XEhjmmk5kS7h5crJHgIL3e20+x8bTKZgH0L6c3eX+nZbgN+uJvm1N7yBQnwVa8Dn1wB/OY2xZwyXZgSzq4V+Ikt6Tm6IeWeA+TlliTvOd2RjlSH9H7AQ1uA238AYlJUT3dBNs3xne6oZ3DmLxgJES/8LLoZBbatfrBt9aNGti0tUPubhHQ1/enPb9R1Tm6lflAnROonRDtuDZvTvXMu8PsLQHkxvd/zM1DqmBHo1J9AaT7dY318GR2jCrLs+t4DnNPNOd1GJnSPSh2o68nuLbwcMPbF3hpOwlUbXl7r6uVVEaOpUK8d/bx0MtDmKqDFQHofpwkvj0lRq4MDQLxreLkpWa3SKkkSebAByicvLwaWTwMOLlW/cGILsH8Rvd74MXUWCu7h5dp2NMhQX6d0dH1WPN3F5wG7XZ0yTBHdAA0WWOOBMAskczi1XQIVncu4lNYxmOgWqSqtAlcvDxJObacqzHWAbasfbFv9qJFtlSJq1njAGgekOmqoKFNaAkB5oa59i0j9hGjHrSGrl+/8Afj+TuCPN4H5T5CQ3vqV+rmtFFj/IVCQTal3eSdo+flDVL9mejMgc6XXzbOnW6yBrFDDEOHlwYJe4eXabRrxBIp0iG6/FVLzhtbTndRafX3R/fRQsMap83FrvdwACdywSMqDjoiHlNgSAFU5l2wV6nzg9grg2AZgzTv0Xpk7PHO5Wo0cAL6/m4R98wHqfNyewssTNaK7w3WUL9f+Gnof5aiuL9uBkhygOIfeRyZ6NIMUFgGghDzefSYA5x2/e3afx/UDhYgX/nrzdNsqgA0fAaYwoOc4INzqv22HOsU5wKdXApYY4In9NS8uWHQe+PF+SPuoxoKQtq0oBY5vBA6toBvUwtPA6DlAo7aBbhkAPm71pEa2VYqoKYPRqV3oubzIdb0TmysXAfUTIvUToh23hgsvP74J+PEB9f22/9I9z+FVACQ6xrJ3AmveVtcpOgec3gN8fLl6XO5fpDob3Kh0H56fRYNK4ZH++x8GR6RzKtQQY7hOEOosup2e7sqfGfliX1V4uV/bG+vwdEsm1zxtTyiC1309SVI/i28KSSPkpZxMmvJLYfVbJM7jmgL9H6JlW74CZBtVT2/Sm0Zlc49SlVclJ9uT6NZ6unvfBTx/Bmh9Bb03h1OnAFCIuTO8PMHjXyPRDUhxaUBad6ChY175M3/RqLFBEPHCr6voBujmNXMV8OX1wMJngN8mA+/0AQ7/UbdtC2LfeuHsfqCihM7HwtM1+25pAfD1LcD+hZCOKNMKCmbbC0eA//QAPh8BrHwFOLaOvER7/lf9d+sJPm71o0a2VYqoKWlXKV1cP1dmDzm+yU+tq4xI/YRox60v7fSL6D60nKZMrY4VL9M9U9vhwOXP07K/fqHntsPIIQGQEFcoPAvs/okEt5nufZC1g2rr/PdmEvGaNrrchx/bCMzoCnw7vvq2BREinVOhBotuP1Ln6uUOT7doOd1qeHmFc5munu74poBDeHrFm+gG1BuM+KaQYho5F0vn3MJRDzrmg287FGjam14rnUHzAcAdvwATlgIjPwGU3PCIODUsPLIBvQdcPfOSBIRZXH9LyevOPwWUFajf94AURR5wKb2fum3JRKFYBdkevxMIRLzw6yq6847RaP0X1wBH/iBPbGxjuvH94R6gvISmTanB3LhGvi4EjAuZ6mvFk+crP00ETjgiXyRBwxQXPkshmVFJQJdbgDZDafn5wwFtlhY+bvWjdp5uR5+Y3IH6EoVed9CzUkyttICqTCsRYX5ApH5CtOPWl/ZWJ7qrzek+tJwGkb++1XXQP2sHcHi1+r68hAacARLclzwOXPY80O9+4NavgFs+A5r2qrz9onNAvkPQtxysbvvYeuDAYuDPWTTDi1t7JQBY/A8S+QeXUvRPiCDSORVqsOj2I3UV3RVVhJcbOZdIrV6uXpx1KaTW/BIS0d3HVL9u55GUU91ueOXP4rWiW/V0m845wrMbuU3r1XYYjfiHaUKA0y+icKUmvYCutwDDX6bljbtBrXQmUbXzK6YCjdpV3V4lr9sp/CXV++2GqQGFAkpKyHq4VfWkGyivW1LC5AFIVeRgGYkqb1IOLgU+GVLjar7ObSr2iGsKtLwMuHc58NBmep9/isLNP70K+HAgcHS9b9tWnu0VVa4XUpzXiO7cGohuux34y1FAMTZNta2nY0GWgY2fAMc316xtmStp4OU/PYCfH6WwSYBmPtg2i7w3ChWlwOYvKnuQ9i8GjnqZfePA7+Q5kszAHfOBmz4h4Q24DkYEGNHEi0hUaVtZpqgaZZpK5fxQBqItUeoAcXQjoP0Ien16Fw0IbvgQWPMf4JdH/d5er202EKIdt7UV3dr7tmo93bt+pOfsHWrl8bIiirT54lq1wOzRNRQ5GJNKYeQmEzBoMt07dbxOvZ9yp+gshYgDQKvL6dpWdE6tdg4AS/9J129Ne00nNwNH19Ln9nIS6iGCSHUSQg3eG37Eb55uwQqpRdVXTndsCvDIn8Dgp6tft9ttwKTdQJOeHj4bDTTuDnQd5RperuREK6P7AOV/Zwwkz3TjbupypWK4Qt8JwD1LgJtnui7vdCMwcJIqxL2h5HUrxZ+s8YDJ7HFVj7ZVRP0Z4+R1S5qbfGnJC0BJXuAa4yNVHrdr3qY82f89SDnZNd1mhSMf7d5lwLh5VG3fEg1c8igtX/wPymcDHDluPmw7m24kpHwfQvsCiVKltj44f0h9XRPRnXeCvCKmcKDDtZqxMw/HwuE/gPmPA59frd5oVsfyl+km9MRmauPmz4CfHGkrv04G5j0AzJ2gVor+czbw88PA4inqNtZ/CHx9M3mV3D1QtgrgN8e1sd995LUE1JkTzrPoDgWqtO2uH0kM/fQwvXd6ujWpUEpIeWoXIC6N8r1lOx3zytzdx9bTd5f9G/jiuqrP71+fpCk3tQNKHtrrtc0GQrTjVio8Q8/wLpzrFF5utwN7VS8zNn1Kz/t+o4Ed2aaZ790ROdh6iPf7ocgG6tSvitOh6Lw68JiYod7rbJ+jfi97J7DkRSBzFWTF073xY8efccicEzUcIBUYkc6pUINFtx/xV063cIXUnJ5uncPL/UXz/sB9K4D0fm6i+wC9aHW5WoCt5WC1AEcTR4h5RLx6Q6ulaW/XYm81wV10R3kuogZUJ7oN5OnOOay+LsgClr4UuMb4iNfjtqJM9S6e3k2CqabbBEgAuR8jPca6VuYH1Cr61W3bITCl8iLyRBmRP2cD/0oFtn/ruvzEZt/yAGuK1qOrzEHsC4pYb9AcSO1Stadb2T8VJcCsUUD2rqq3XV4MrJ5Br3vfBVznKNB42lGHQfn+7nnA/En0+tQ2ej7peN7xvTpNYWmua94jQFWBz+2nAozagUklIib/ZP0OflSBofsHwanStop3cO+vJJpPbqX3yZoiaW0d6QhKWkKbK+l590+uEThLX6Ic3cwV3iMvyovJO77nZwoDrqK9XttsIEQ7biUlVQay14HiWoluWwV5s09upfncTTSjCnb/BBScBnb8oK6rDCA7RfcVVTf6yheBLrdS2DlAOd2Kpzs2VS32V1FCz51vpufVM4AvroHscJ5IBVnUryrb8VV0l+QB+cZJ06sNIp1ToQaLbj9S1ynDbIJ6uq0ePN26TBmmA6ZYVexI9hIKSU9qBbS6jBZ2vkldWeksWl/h1Qtda9zDy73kcwNeQvcbtafng0vpRifnGLDnl4AWVjMpRXrgmN5sy1dqWKNB8ZrGcXKra0Xfpf9U51X3dZsSgGb9Kq8QbgWGTKUR+Q7X0jIf87pNjmnqTJLs/cZXT7J2AG90BDbN9L7OuvfoWeuZ2PUjhVnPHOp/Iaj16NYkp1uZBSCxFZDSEaaqUmSUsHCzheaX/e2pqrd9aDkdP/HpwIg3HCHfElCWT954ZVocSMDmz2kmBOU3zh+iUHOtxxugEEsFuw1Y9Rq97v+ga2pKZAMaKAQojN0AGDldSnS82ra0QBU+tjLg50co7Da5o2tV+66jgEe2A33vpfeKCP/zG4oEUdCez95qiShiCQBWve7R2y1SKKxox63JMahikqDmRbuvU5spw74bD7zcHFjgGNxrP4JCw+3llPN/YLG6buYqGvw8s4f6OOXeyhvtRwA3fazW48k/BTg89ohNA1K7qutGNwKufwcY9BQVZ4tNU51frS4n50orx32br6J75lDgzU7Aug/o/in3eLVzgxsNFt3GhacM8yN1Di93fD/MQ/lyI4tYpXp5cbnOOd06IEWrHmVTxmBg7OskqK/6J9Dtb0CzvurKra+gEHJtUTR/oRRSy3EIVR9Et8ux0HYoTV12IZOmMTuymjxht30DtL/a/+31AUlzgy8ltQFKD5I3rvddAWmPL3gd3FJG69tfQ8IleyeFBN9Svcdb3SZcjyct3f9GQqwkj7xCFzLptTXO+4aLc2g0Hw4veubK6r0I/mbjpyQYl08HeowDzG5dyrmD6hR7xzaQOMw5qglvPUqifODj/mlPaYFrxfKahJcrnu7ElkCj9mp4uWyrvK4SUXLFFOD3F+j4OL5JLbjojlKht91wOhDCrVRbIveoKoSs8UDDtpTCcHyTKrplG+3bvBN00xqT4rgRPUspCgDNq3x2H21DEUsKkgQktqD9cD7Tc5ROPWPkQWTR8WrbA4tdRbNSKLTzSPcNULSHQouBVM9E8Sy2vpK+K2vSG7TiWotWjOccJeHey7WStEgCQajjtqJMTT8CyP4J6ZVWq3EhteOb1evZ8Q303H4EpUrN/hvN5AJQmHjOUfKEK4OSTXpXeW/jgnJPdHo3AJm86VFJqqcboIK24ZHAZc/Se1mG/f1WwNlMmAY/SZ5xs6No7bkD5In/6xcqcBvXmJwsGZdSGgVAA8CnHUUCFzxFg+tl+XTdTXKkg0kStcUcTts2W4CIGKDXnUCLiynd6Hwm0PVWWicAGP2+O5ThPeNH/DZPt2Cebuc83aKEl2uQtKO7176uhmlHxALp/SrnHjXt7XUqrzqheLoVaiq6IxsA1zs8invnq6Gnu//nx0bWDBfR3dVRzGnLl96/UHCGRFpBDad58gcXjgCbZjoFVmXR7ZjSK2MQcN3bgGQGds31yb6Scl0AgGYXeV/RHA5EJ9GNAKDmd3vj+CaXmn2orlhdWSGQVc02T2wBFj5HRb62OcJB7XbyortXf5VlYN9Cel2QTUW83Nk1V31dmkdh1HPvpdcxKbR81Rv+2+fuxcJqIrrPOUR3UivAEg3JkfIhKWJDQZYpLByg/MSuo+j1H2963q7dpuY9agfAlLBvxW6JLYE0Rw2Kfb+5ho9v+5qeG3VQ82+LNJEWSrrDRX/3PFCjFFo0SDE1UfoHEfFq2z0OodT8EtflndxEtzuWKBLeCp1vousgAOesHb54ugFg/Qde2+uxzQZDqOP22HpIFRRFJEnwGvVT4/DytY7UGCUFzxxBKQjtR1DfqCTmdLtNHWRWRPqAh3xvv5Jyp0T0xKZS8TUX0e12LEsSZFOY63+ISlTrWnw7liI0Nn4MLPk/4Mf7gLe60aCANndcMpOwLsun17IdOLsXOLmFPObH1tFA68EldK3e8R3V9/hwEPDZcOB/f6fXxzb6/n/9iFDHaYjBnm4/UvdCavQs3JRhTk+3zoXUdMAwHX5iK9f3ivDygFfbthlCN93r3qOOKWsHTaVhK6//EdeKUkj5J5xvpU43AptfpzDtk1uBtB6u6x9cCnx1o/r+b98Bba/Sv53lJVTAavc8QLZDOuPBthVlVDgIAFpcAqR0BC55jMJ5f50MtBsB2CsohLjFJTTqDVAo/SdXQlpLQkcyh6lpAFWR2pW8mqe200i+N46tV/OOAcoBLs7xPii04Gka9Pjbt2rIqDs/3k83FwAJubSeJJxXvAxc9hww6El13VN/uoYsbvsv0G6Y6/Z2Oirbmi0U0rryVfKOhEcD9/xO86ee3EIFma6dQYUA/5xFx4jZQiGLPceRJ2Lz5yRQr39PFZa2crrhKckjL4RSf6FRe/JGl+RWHzGg4AwvJ4EqxTUGcKKy6M49TjdjpjA6by9+hETxX79Q+7WhugB5rovOkhe6+cXq8qRWlA+rFBpKbKUWfvxrvus2lPdNegCFjptQbXi5Uo+i9ZWe/5uzmNoh4NAKskdaD4oMOLgEaHOVaruacGYvDc70uYcGA9Z/SMd/j9ur/Joo/YMu2O2kgnT67x5tW1GqDpBdMQX47g46d9N60HFYHW2HqiHDLQfR4PPW/1K/svLV6j3d6f2pkvTpPZQLbImq1N5KbTYghj5ui3MozafnOBrEP7hUHZQF1Cg6N7z9J0mSIMuyq+jOOaoONN/2NUXbhFlVJ0HPcYA1ga5Xfe52VMt3RIld/AhVKfeV6CTX97GN6TkqkeaTP/MX1d9xw+N9eJNeaiRTm6FAamcg7xSFvJ/cSoNBhWfVIrqJGcD4n+m4TulEgvz0Lspll+0URm9THmV0jd/6FfXBkpmugad3AZ8OofuDIS9Qv3B4NYn2nuP0cd44MPRxGuKw6PYjfiukJpin21m9vIxFd61p2pvE0Nl95Bmr4qa1StsO/TfdACc0B15vRzf7R9bQjVJ9cv4QJGjSDWIaUmjtnp+Aj6+gkfEb3lcLxm1Xpv+QAMjA/oX1I7oPLFa9sWYLpPIL1AqtbU9uoXzcqCQ1NHfQkyQwCrJpn+1fSGHGjToAY76jKXgOLgPO7oVUSh5iyRpPI/XV0bgrjZ5XV0zt2Dr1pircSjcDR9Z4TieQZdXbumse5bktfIZC5vo5wpFL8lTB3bgbierl09Tw58OrXEW3Mjdqow5087L3N7pxUaI2jm2kGw9TGIU8r32H9j9AoXcJ6XS8fjYM2PIFCb8f7yMvuML+RXRzf+X/kfcdMk231udu+nztO2R3hehG9JzSmfZN8QXydls1haI8YberueCOATAS3YBkcxPdSmh5Umua1aBRO/J4H/idClS5i+69jmnI2lzlOvilCOGyAvW94um2ueW+Ku/TetLxCKg1BSpKKdwccA0L1qJ41fctpKnOwqzA3YuABc/SnPFth1EqSk3CEnNPUOXqgiy6EW99BW0bEnm9ahqtEwqc/gv49Eqq3XD9u7oIb4+2Pb6RBoqik4GmfYDuoynHuuc43zbafgQVTkvpoobiXvkiFfcDqvd0p3ah8N7CMyS8NfMxG6YP9gFDH7cbPgKW/YuE361fAgd+1xSDRN1Fd8EZGpSVbVRgVutx1tLxOlVcd7yeBqdbXUFTp9aEKHfRrSk2OuZbGnRsWDnVz+N9eJuraHC2/TXALZ+r12FZpoGK+ZNoAFHxdMelqQ/AEYre2Htbe42nqWz/+oUKo0Y3pBocf35DkYf7F9E0s8oAxNp3KSpAp3scQx+nIQ6Hl/sRf4WXe/J0GzmnW61eXll0Gz23RGvPgLZVksibMOAhmkLKPdzcZdUqLqiSRJ4Lc5g6R7nWa1acUz/F1c7ug7Z1kiSRcErrQZ32vgWqAJNl8nQD6k2gUlVXbxQPds9xwI0fqjcp2qrXSkfZ4hL1JjksgsQxQDc5mY51zuyhubzzTjq3LSWRuJKU/NvqUArFeBPdv04GXmkFHF6ttjcill4oQhigfb3oeRJaFzLVXOeDS2m9DR+R8FaqnivzmMY1BYa/Qq93z6ObdYBC02WZHoXnVDHZ/0Gags9eAfxwD3kFjm2gqa0AumFXjkUFRTQ37083ZrIdmD2aBHdqF+Dat6gNDTJINH9/J6BMe7N/ET2XlwBrHSkVyhR+SsGdxAzKmQYqVzA/n0lpDFq004U5vic5wrilsgLXc0bJtdZGLSgebE/H7TFH3mPLy1yXu0e3JLUiIR+h8cq7pyM06anmOiqe7pxjAGQgPKryjaqCEl6uhNtXlAAzh5HgBuh4+OMNqtrr6XF6D7DpM9XLVVZE+8tRUwAXMh2CG9SWaubEDfqbQm/X2XXv0jG+7WuNvfyLR9seccxX3OJiGlgZ/Cxw/x+Uh+oLcWlUXO32712XK0KoOk93TIo6FZlb2gyLbj9xzhHt8tevwP7FQNZ2SI6CrxJA9SM84O0/KfdDdrudvMIfXkq1YsKjgcv/4VubGrUFnjpMXvGaFp+1xFDouoIigJXXXkS/x4LGXW4BHt4GjPqv68CnJFG/DtCghFLQMk4zhZ6vNO8PDP0XkNye7t9ueA/4+zoa0LSXO+4jJPLYF2QB346jvlIHjKwXQh1jKyLBUE72uhZSM3vYK0a+2EfW1zzdOiBSh6/gs23bX0PPf82nG8A9vwAvtwDW/EffBgIkujXNkySJhNC9y4GLHqSFyo159i7qhMIigYsecHy2k0K39EYRROn9gc4jITk8pZIiRgA1n1ub1wiQ0ARIaDmmZkF0I/ovW76kcEoAkiOUXlLCzqtDuZk4/VflPOqyQipeVnQWkG2QHCFqkhIxsPt/FA5/9gCJ/zVvU/j84dXqNgqyKJwbIKF83JF3phQ8a9yNqqw3Ub1RAIDi8+RR/exq4NWW6vpth9KASngUcGgZ8Gor8uaV5ABN+9KIfpNe6rQyzfq53jANeUH9zJpAHtded9Bc07f/oHpMox1TrR1aTqJv+2waSIhrSqGAyvEOkNdYEd1Z26kS7cpXgW9GA//pDrzTm2xw+A/67IjDPg2aO4vBSY76DlJFKXlBFBTRndxRXaakS7iLbrtNtZO7Pd3DehNbkSBq3E1d1kmTcmG20NROirBWPN2OCvZIaO7dc6p4ugGydUyqWo2/g8MrtfQl4PW2nh/vXQT88ijdKJ7aTqLx1J/UlnH/U/PzlbZVU31flP7BK6e2AzOHUzSLO3t+Bl7JoNoIABUNfKcvXdO0UyktfNbnWQpqgmfR7Ti+lcEhcxidgzWxf2RC5RQEZarD6jzdsakUpguw6NYLJWfbXg58T8VKJUel8Np6ugGHM2nHt5SO0KAFMGGp94KRngiPrF1EhyS5DiJqPd1V4NH5pdx/eGqH0k+U5qpF1LQCvy40ageMng2M+poG9u9dTuI/JhWoKFaniPUzhj5OQxwW3X5EOdnrOmWYaOHlkR5yukUZafM2TYaR8TmKoOVgwBIL5B2nvM1VrwOQKQTZGxcOAwueqfv8yWcPQBuw4dLWtO70rBT1UqrotriEvIcR8eR1VMRNXagoA/b/TqFe7lOGVJSqcyA7pvIyJdAItylrB4maijJ1blplRNz9f+yaR7nDYZGqB2DHd84balMihfz6fF1ISCfxbi9XBbHCya0UKRCTCoyeA9Mlj9K2I+NoWUkOidHPhqnTzxWdIy+mltOaeaWV40EruiVJLXqT2Eqt2L9rHnBUc/x0vY3mHW85CLhrIU2JJdspr63d1cDYuZTLHB6p5qe7V9dObAkMfopE+40fUmi+QlIrEt7dxwB3/EICu6KExP1qx+BR/wfJezH037QPAKoCrmxn6T/VSrSKd162U5765yPos3kPqP/VgSmC8k5NEoC/fqaF2uq2yRpPt3Is5Bxx9V6c2Uvi1hKjVhpXSGgOaONBlHBzJa8bADIG0rzbAImkMIsaBVPkJrq9hZYDNNWO4jXqfReFWMY1AS5+lEJRL3qQ0gAgeX5oveh7flI93pc8RteZB9YCE5YBfe+j5dV4ukWbeqkSq2fQefD9na7XytICikSR7cD696m43pYvKG1j5jCgvJCOzbbDKWVg48c1/20l2sQLlWxr0wysKREh/iLWMdhSVuB5WiWnp1szv7LbnPY1mjKsNB9Y/nLl6JV6wuSIFFGmajQUWps4UnRM3W6lZ8nxub3yTAzezkUX0a30DT3HuV739Eab1x3rmxCuccSpJUpNS1Km3fSX6KaGAB2uocHntO40c4XSF+hU2NLIeiHU4ZxuP6JneLmRTyLF011SbofdLsNkkgzdXi0ijbIr+GzbcCt1kuveBX59Ui0U5c0rAVB+7K4fKfx2zLdVb7/oPN3wJzRXc7MB6thPbqkcXq7gvPnaSbm0Smh56yuog0rrRpW4T21TQ7hrQ8EZ8rgqHduO74GHNqvemlPbSdxHJTkFj2SlMG0JNvLmNetHI9JRDSsXQVM83UrYdloPytX85TE11C82DZK9QWUbVIUkUSjyjm/JNlqxr4TDp/cD2g2D9McJx1dMNPXPuvco91m2Ua51Skeapk0pItP8YtXrpeBJdANAxxuA22aRh2rJ/9F/UubczrgUGPeTq+egcVfg4S3kUYlvRgJRy40f0EBLGw/Fvi6dDAx8wrMnokkv1Uvcdiiw6VNg3t9pgMGaoKYkNGgO/G0OieImvZyRBgDopqrdcHX9U3/SQIzdMeOCck4owhfa8ww01czpv4CdmvDaRpqptyIb0HfPH6Kc69ZDaLmSf924e+XwSu20YdZ49RxS8rpNYZRzn9yRwsCV5e7h5Rc0nm5vmEyUx3t0PXDpEzRQ8tgu1d7D/k2PqvhzNuXcb/9WvcFXoguik+ih2LGaegSG7R/O7qcBkqryNyvKKHwXoJoBP94PjJ1HNl75qppfL9td6w0oaRq97qDBnX2/qWHfvpK1k+YR7n0nTWvpAcmRYiEpXuas7SSKrfGu0Rn+ICKWwo3LC2nfSxINfCkCzunpTlEFupKm4uEYkCSJUld2zaW6D9r55gHKhV0+jdIkrn/Hv//FBySH2JaU2hdGwVahhkZLZrr+t74SkiOaRpIkutblZwHxrkVaffJ0K4PTSp9XX/jL010d8c0oNUlJv6mikK1fSGxJ4eZKv+xnDHt9ZdjT7U/8NU+3WVBPNwCUVLhOuWTE9moJatENABfdT52wIrgBytH0REWpejO5f2HVHvGKMuDjy4GPBlMo5fTmVCBty1d0Y3R2n5pn7N7WpDbkdSsroBxo5caz1RX07C1UtybIMoXDXsgkT2FkIt2UaKesUQRs076VbgAlgHJYlcrS2nxu5/9oTTfoCk17k3DK0IShp/dzTktXo+NLqcqqFDFzttnhsXJ45l2OhS6OHGrZRoJt5EdAv/tdvz/4afV119vo+fhGykHVFlGjjVI+doMW6kCJclPS6grPAtkcTt5pd8ENkPeg7VXeQw19sY9Sdb0kB4BE3gNt2H7LQZSiIEl0nAGUI337D7TuVS+Rl6HLzcCk3cDje4G7Fqih6xovjsuxkLXdVXB7qvqsiGLtcXvCIboVT7g7Sth3Ykv1/2dcSl6d9iPIjp1uoJDwzjfR54r3R6li7ounG6A8+QfXkeCmP1j1+u60HUrHVc4ROsZSOruGrQNqPYIzeykqwAuG7B/yTgIfXEJT/njwCDo58gd5Eq0JFAGQuQLY+iWldKx9l9YZNl1NmYhJBW76lF6HRQLdRqtTKZ3br6YJeCP3uCp6NnxE182171auS+BAOkXrSkqki3IdT+9fs0J5vqKI6UPLKX3pu/F0/a0oU6MxYlKBhu3IJqW5LlP5uVZZLwNm3QL8+gSwxoOoPu5I41GKGdYzksODLBVkU1uNQkGWet3v/3c6zgY/U/k88xBiXp3othfnqvcP9S66NbVtfPQ+1yrN033+cn96uj2hXDf1Ft1KHRTGMLDo9iN+m6e7ikJqRgzHs4apolsppsaF1PSjRqH7CelUqEpL4Wl1fjotmSvVSsoA8PuL3sMYd/5AglaZp7Ukh/Kaf5oILKYQa+myZ5yru7TVHKaKmxWvkLc5IV0NuVI69uObqNjQ0n9RXnppfvX/F6A2b/mCKomawijndKjDi7fqTRL5Redp6ipAvQHWtFOyRNL/W/kafeAeWg445gzVeOKb9qFnJUcWAJpdVDuB4cjFw6k/1ZtyWda02YPoTuuphoFf+iR5npv2UW8oEltRXnpyRxosuPw5uqmpKKEK4bKdxKcnj0KKW9Ga1lf4/l/8ScallDIBUKG1qqagaXMlcPVrFPauzZPWIkkkeO9eSEJJmXMbGts6oh8gmSnf/NlTFErt7rlWBotOaES34unWhoxrUYS7tqhaVCLw+B4K+waAvhOAKWepUA+gmb/2LB0TTk+3242jv4ls4HoeaHPoFeLSaIBLtlWZHlKjc0KWPV+v/M2RNXQuXMisnNah5S9HikLH64DLn6fXy/5NFZDt5TRtW7/7KXpDMgNXv0qDPON+Au6YT/s3KlGNlDhahbe73FHw7qPBJGp3z6Plsp2unR6Q8igKQSo+Rx5QZftVTT9YF5S87o2fUMj8np9oej8lAsgURsesUukfcAkxdzkWlr6kpgG520WWKfoJUCOJ6hnJEa0g2cvVOh5GIEfjob3qn8CzJ4CmvVTbKteqGohu5T5DVs7j+GaVp/HSmzp4umt0P1dJdNeDpxtQZ8zwMybHIInJXk7XAMYwiKEyBKHunm56Fi283GSSYA2nQ0nJ6zZye7UEvacboPkxTeGqt9JeQWGR7vz1Cz23G0FTCh1bRyHW7sgysPoten3FP4BnTwIPrKF8ZsW70+pySF1vrdRmJ0pblJvIbqNVz5sz53s7MP9xYOUrwOy/AW92Apa8RN7D45uAufeRt33mcODL64GvbwXm3E5VVn9+hLYx6GkSn11vJeFYmkv5zq9kqHmpDgGrbafUbTQgmegmGqhcRE1B68FURHf7awAluD69X+3OhdhUR7VfWfW2nz9EIcXmCKfYd9m2JJFQu/YtYODjyh9ShaTirb9jPvDgBrrRUG7E171Pz0o+tzupndXX0clqJeL6JjyS8sRv+0ad6swbJjMJ1hQfQmoTW5KHXFMoymnbuCZ0wznyI5qOzRLl2UaK6D62nqrGb/hYrVuQ5kV0d7iW7Ok+MFYVivenooQK62kLqemNVmh38CC6JUlNCakixLxG58SCZ4BpTdS5yPVCW/Nhz8/q64pS14Gvvb/R63YjgD4TqDJ8QTZ5vM0WYPjLZIdBT5L4UQaGWg5ymSrLOYii5JF6YutXDq+wTHPal+RSzQuA0k/O7KOb6tljgFm3AQVnIDm8y5K9gqIulEKQ6TqJbsXTrdQ6AKiInPK/opNVD7tSTE05Lw6tgPTnN9ReSXIt8nlyK/23b8cBs0aRHZTZCYov+F75+dBydXYJu43euxeo9IWKUprJQGlr5sqab+PEFuqr3PLaAdCxdWJLzURS3kmqLK5EDihFwZSq5c7zzGF/DxXMqw0vz3aIbm8Dl3qi1K+wxFIqgw/U6j5cK7rNEa7pcnrgFN06ebod1zIJsut5yQQcFt1+xG+ebsHCywFNBfMyFt16U2PbpnUHJm4Exv+iFmVyz+u221UPTp+7yEsDkOh1L4q0fzGFhVtiacoZSzTdTF36BDBhCTD4GWDkJ86wao9tdfecdv+b+rpBBoVuAnSD2fU2CnEuyaU5Pz++DPjkCioYdmIzFTQ6tJxC4vf8TDf7YVbyNjkKjcFkBm7+lHJttQVZohqqYknTTimtG82jC1CHrHho3FG88nFN1DzQ2BTybg18AmjcvfbnghJirkyRpYTDp/Vwhm9X2nZKJ8oZNWvKdVw6mUKrh7zg+M+Jal6fMpjg8I55DYOOSVHFXqvLax6a7E+a9fU8F7mfcdo2Ogl4bKcavu+Nxt0ASOSBXvM2hcjay+mca9DC83daXQ5M3l+1x94dSzQd3wB5rpTc7urCy/1Bx+vJ492kl/eBFyX6Q6nMXZqvhgU78PmcKMmleXTLi9RBQb3Qim5lxoeKUuDza4A3OtLnxzfSuRIeRSI6zKKeVwAV9VOiFySpcrVvLUpRM29pPBVlwB8z1PclOfTc+w4S/LIdWPC0GtWz7zdg+b8hQbEtgGX/pO9FJ3uPtqgrMW4eyKTWlOOtVG/XeiiVwdZj68i2346FtIqiiSQ4ohn63keROGUFNLCw+380pZ0Suq/gi2C5cAT4aiRNX1iSR+H5X17vKCrqRnkxsGyaa0Xp/Cxg1RtU0yHnmDpFI0B9zqbPaHANoIGEt3sBf87x3p6179L3lNkjNnwMrP+QXm+bRX3b4inV/y+AZnD48FLgo0Fqm7VFKKE5zxRP9/nDlTZTrehWPN31HVoOqOK3qhoLbtROdGuunXFp+vdvyhSOxecptcvPSI4IK0lC1VE7TL3DhdT8iMf5AWuAqIXUACDKEoYLReVOTzdXL9ePWoXuKzlEMSl0oS/IUj2AFw6TSCg8TfmvLS4FWl5OnooDiynPEaBq1Dd9QiGAABXzcUxZ5aRxN+eIuKnknHNxpbZqPacZl7qKEkkCrnyRvBNDppLotdvpxnLrV9SJlBaQCGp3NYWyVpSR589WRt6m9tdUDoVr1I5yewH6/vlDdENoiarUTpPJRAMByR1JZHg7NjpcA+y/iaoRa+k7wfM2a0KbK8nzs30OiQ/l5kcTDu/TtsMi1GJj7nT/m6Pi9jkSc30meF5PkoD0i2gftB9Rs/8hKDXebxExNNBzaBmJ0r2/0bnWtI9/b+IkiQZA8o6r4evWhMpFp/QgJpmmvDFbvP8nxSO29zegzz1U4fvMXzRtjsM77rStXAFs/47EvKc6AH/Np/QTgM57WSbhktxeLVbnD2zlaiFBSBRifno31ahQUjp+ewrOCJaON6iCuuP1QJdbqYDawCd8/810xxzsp/6kiAVLNL0vLaDK58ccAj8mBeh2mxpd1OVWGnQ58DvN/KB4sgFg00znrBEmCWqUTOebaj5Psq8onm6AjsNr/wN8frU6h7tWdLcZSlEgh5ZTilJJrtpegOZTHjaNbH94FYlghY2fuv7uuYNUR0OWKaQ9Ng1o1sd1na3/dfQPNhqMVeyRuQq4zO1/bPwUWDGdzqkx35EgXvAMDZyZwoCRH7va9uhaNQS+9RUUFXbuAPDbZLp2RyVS/ZQf7yXP5jVvqvUe9i+itvzqOF7aXa0WFN36FaUtWKLIM7/0JSC+KZ1LWk79qXr+d3xHz/Gu80s7z7OwcAAyRZYNmarWdYD365zq6d5Nh30gPN1K2k3Dtj5/pc6ebr1DywHqK6KT6Z7rQiYQ2aP67/iK3QbJcS2TABrw7HO3/7YfYN599128+uqryMrKQrdu3fD222+jb9++Xtf/7rvv8I9//AOHDx9GmzZt8PLLL+Pqq/UftPcGi24/omchNaOLWGd4Oed0606dBmBiU8hLXeDItzu5jQoHKXP29hqv3vyO/Aj44jog2+Hp3vsr8MFAKqpiTVCnlKqmnR7bqvWS9Rhb+cu97qCHgslE3sCO1zlyPG2u3tyaEhHjsTJ6Jdt68/wqWKKBm2dWuUqt91eLgRQm/seb5OkByMPW8Ya6b1shIgYY+i/f1r3mTYpsCFQ+dz1TK9sOn66+LjpPAyZK8Td/EpVIgkwp1FYfXm4F94E2d9pcRZ6cC5nABxeTRxYANnzoFN1O2x5bD8z9g9I+et4BfDOKwkhv/oxE/Q5N8brjG6mw4KLnKPXj5s+oyJyvnNhMg3fN+tAg3Z6fSBxZ4ynct6KEXqf3p/Nt9hh15gNzhOoxCo+mtBoFSQJuqsXUXwnpNAVe3nESge2G0fIVL7uGWV/8KA2aHdtA4cPKgOWgJ0mQ2UrJ2+wQucrh6nLUdr2l5u3zlViNF7L5AHoo+x9Q53AHgEZt6dqfvZMELQCp7TAA35M39ob3aXCgSS8S3dpwaCXVJ8xK++rcAeoHfn2CoiFMYTQnsjI7gt3mmh51cpsqerO207Gg7fMVAX18I4V4//4C/aZkopSsnT+oto2IAaAJAz+yRo1EKsmla3bfe8mrfv4gCeyL/q4WJLOVAd/dqX7/+Ea1baV5FLHVbRQNrPzxJi1v3MM1PUEZcANUWyvh5Q6c51lYBJDWkb6z9J/Adf+pvI6XnG77+UwgyVR9X6gHLQcDY76vkeCvlfNLGyGgdxE1hcSWJLrPH3KJtgNA9zjr3qdZN2p67mZth6ncUXtAgjpo6A8OLqUCnl1uDki025w5czBp0iR88MEH6NevH2bMmIGhQ4di7969SE5OrrT+mjVrMHr0aEybNg3XXHMNZs2ahRtuuAFbtmxB586BSZFj0V1H3nrrLcTFxQEASkpKAPghvNzDtcLonm6lgnkR53TrTp1sq9wAFWSTKJgzlgR3k17AoKeoCJBCVCJw30ry1h3fSHnVyk3DVf90GS2vqp0e2xqZQDeSOUcpr7UmSFLdBHeVm/b/cVvrbUoScMUUKsz25zc02t/5JhfRU6/nWUwy0MaP3kWDU2fbRiVSnrgeKLmOSki03kXUaoI1Dhj7I01tVZBNQrY0n3Jgzx0Eklqpti3NBRBFXt0mvUhkAFSHIr6Z6pmUTLQtZco62Q78cDd5m30Z1Di9B/h0KH3v3mUUFrz5M5qab+yPalGsJr2AzjeT6FaEzICH6T8oET6DJvvvxrzDteTVXvUa/Y+SHBKQAIm09P4UWWIyU5V9LRc/Qp7LrB1Ux2HVa8DxjWoItHIjkdjSe00Bf6AV1c0H0HWr+9+AZY7BPPcCWJ1uJNHtCJeXeowB8D0kcxjNfgCoUwQCgGSmY0qpQ9LmKhowOX+Qands/YqW2yuoP2s3nELFE1uq02gBNGispFWVFdD3K0pJJDcfoA6qFF+g7ZcVUORX6yE0hdn+Raptm/UB0kFRUHt/JTHijJQAHadr3yUvu8Kat13tUKzJST+wxHWGkW3/JdGt/DcA+O1JShU6tIwiKrTpEApunm6Xa9iwaXRObv0K6HefM7++0nVu51zg0DLn+3c3lKFhYjzw5oeVf6/eqKLugRsFBZq8e1+xRFP0UNHZSlOq6UZiS0qz8JQmsfMHYOEzACQaqKpJlMHhP1wH3s4doHs9T3nqtgrf76UKz1JtBVsZnb9XTKWBvuhG6nmrM2+88QYmTJiAO++kAasPPvgA8+fPx8yZM/H0009XWv+tt97CsGHDMHkypUu+9NJLWLx4Md555x188MEHldavD1h015HXXnut0rKoqCgPa1aP09PtIbw8Ojra5dloRIXToVTi8HQbvb0KkiQhMjISpaWlsFqtgW6OT9TJtopQLjgN/PQQeRIaZFDYdWSDyuubTHSD3244cOX/UWhgxqVAj9ur/amIiAiYzWbnoxLXvV15WYCJiaHpp/x53NZ5m2ndvXoZ9GgvQxjatkp+vVJvQTO/uCFIzADG/0xT9PW6g2Yg2L+Qqlpf9iyiD84HAESHO/q6rB2uomXnDyS6ZZuaS3pqG4VTA5QbnLWDRNaY7yi/2ht2O/DTw6qndM5YtfjcoWUUjaBEDDTpRV4caxwNRsamUTpHRQnVsjCHkxj2F5c8SjY5vpFCjpU5tZM70owLVQkHczjV6cg9RvYoPA0c34jo2DgAZxDdIAVADoWj6zkopxXVSmHGrqNU0a0V5QDQeaQ6gBHfDFGtKLfd5Txr2tt1mw3bAps+JQHe8XoSxYf/cIhoiaKyts+hQZtdc11/r1EHx9SUq12XH10LLPoHeZb/9p1rnZO1junKmvWjNIBdcwFbGaIdQWDRLfsCd/0fsG8Rie7d/yPRH5NCbT3sKNzWpBelTu38gea5V5YpglnJXVemI4xsQDm+mStplg2laF+YlQaGvnEUxawo8Sy63QbfXO4V0i+iAdw9P1Ho/LUzALhd52QZWPgskH8K0eEm5AD4z4YyAGeABT7mmhuEGl+3E9JJdNdHeDmgmTbssOvykjy1HgJkigi5Y77rOWy30UCkp/P68B/O62q01SGGT2ymCJDTe+h1eCTVKdjzM9BzLHDNDNr3x9YBu+bRoOLFj7huf9ssEtwAsHoGzVZQVkDHZqP2lMYWm0oOGXsF1SYoOE1RkVf8gyJcfp9K11prPC3Pp+3n5+cjLy/P+VMRERGIiIhw+VtlZWXYvHkznnlGnRXHZDJhyJAhWLvW8wwQa9euxaRJk1yWDR06FPPmzfO4fn3AoruOjBkzxnnRAoDu3bujSZPanbTOnG4PJ9J7772HP//8Ex07+lCJNwBYHZ5uJaf7oYceQmxsrHNEysj897//RX5+PuLj6yEn0g889dRTSE9Px6hRo6pf2R3lBujCYTVk+ZbPPAtudwY8BGQMokJBPtzERUVF4YsvvkB4eDjCwsS41EydOhXdunXDNdd4qMxcS0aOHIljx47h1ltvrX7lGjJ06FD861//wogRoZFnXZ8MGzYM//rXv/x6LPgNxdOteNI6V1PkLRA0akcpCQAJ7/0LKWRyyxcYGZGDY4MjcOttfwNO/UBzxB/ThEHu+EH9b93H0OfKdFHWeOCuReTp3vsr8PUtdAOb0hkY8Rp5L5e8RNe4imISasc3kMCRzKrgVkK75z+hhsA36U3XNnfveXgkTSvnb2JTqf7Dmv/QIKgyt/klj/kmlCMT1MiXrrcB5w9h7PBOyOu1B+Ou7AHk/EHF3fQkIZ2KalqigVSHR65Bc6pzse+3ygOGiS1pIOXUNqDTDUhOScHHH3+MlBSNOI9LowGP/JPk6W/ahyIAmvVTi7EpIrnNlTQ7RftrqLhmWRHlyG+aSaLg2hnk4XXnjzfV4nS/TXb9TBG06Reps1IAuK+XBaauo3DP/ffTgmZ9AUgkMgBq3zUzaAClWV/qK/ctItGt1CZoP4LyeQ8spoKb8x4gEQ1Q/1qaT4NLX1xD203rSWlVv79AQku2U9qFMr1ngxZ0rAOVPN1t2rTBf/7zH3To4Jiers/dJLp3/UhTLoZZ0Lt3b7z6yivoP2AAzQmffwoA8MnNyZi3VabBpzZXVdq2kencuTMyMjJq9qWO19O1IeNSfRrljjJQuutHOl5ikulcys8iD3JCOlBwhgaLvr+T5l6vKKHowKwdlBqWMYhSzgrPUGRO7gmgvBDN4k1479/PoEXRNgCraRvN+tL0g8oxr7D5c7ruZO1wrXRuK6PIitI8ivjY/Dktb3UFHZ/K8VdRol6bAeD4ZjrWlXoDAG23+xi1LoVCHl133XXN1KlT8cILL7gsO3v2LGw2m+t1AkBKSgr++usvjybOysryuH5WVpbH9esDSZa9TcTLVMXx48fRrFkzHDt2DE2b+udi1PWFhcgrqcCSxwehVaOY6r9gIO77ahMW7srGP2/ojNsvqsf8QqZmbP8OmHsPXURL8yg06In9ga1GzTBMzVj5muotzBgEjP8psO2pDlsF8F4/dX7lmFTg+ncodPeVDArptcQCjnmQnTRoQdPb7foR+PE+WtZ1FHk2y0so5UXxfgNA0750U3l2X+U2DH+VvMO/PErXvQfWAF/eAJx2TN8U1RB4aHP1Oev+pvAc8J8elNcOUOTRxE26pdDowoUjVFxPW2W6rJBEQCMPRbCOrqfw/itfAmIaed7mlq+okN6N79Og8Kk/KX/cGg/8K1UdKLntG8+zGdgqSDhYooA3u6j54Wk91Pxpd2IbO0UnAOCOX0msTGtGAzgAHY/a2Szev5jCbQHgqn8BAya6brM0H5jeXB1Eun0upQ0UnSMh+0pLNdR8yIuUQvTVjcA5R0XyEa8Dve6iQatG7amavjLbRFJrikJb8zYdv08eRJXYbVSJvyALuG0WDQCcPwR8ciUNNLUeQgJPiyWWthsW4XmbwYQs19+90Jm9wLveC4BhzPdUBG3FdO/reCOtB3DPEko/+f4uOmd6jqfBvaiGJPgbtaPByiX/p37PEksDTQcW03ul2FtiK0p/sMQAj++l17KdInJyjtGgqK2coluUa29qV5rR5vcXXEPoBzxE17iSXBw/dhTNxryF3bt3uzgrPXm6T548iSZNmmDNmjXo37+/c/mTTz6JFStWYP369ZXMYLFY8MUXX2D06NHOZe+99x5efPFFZGdnV1q/PhDoqh78OOfpFlAARVkc4eXltmrWZAKKEl5e6gjladKLBTfDiEaUpjL/gIcD1w5fMYcB965Q81YbtlWrf6d2oXBaRXA3u4jCHAFKaQmLcPE2OucKD7fSjWnWdvLQ/e/vatGguCYkYCxRlM8YHkliBiBR3agDXQtv/568TEmtyfsaEYDB7ugkYMJS1VuU3l8swQ14LuRnifYsuAEgvR89qqLnWHooaPNa45uRVzKuCXlhPWEOU+2Y1k0V3b3udBXdSmE2gKIOFBFitlD/aA6n6daU8HT3+gnp/VXRrVSk1xIRS9tRjs20HnRcKrNmNO1Dglr5LKEZcPciEkv5p6iiu8lE4hogr/4fb9DrJr3ofFjztm+FzkxmSp9Y+w6F47cfAax4lcKqt81SC6pCAuC4IW19RWgIbqB+74UataMUnOILdDwXnqEaBLJM51PrITSgGhFDMxqEW8nbHe2Y5rQgm45JyaROS5mQTgNUyowrHW8Akl8jT7NSnHHov6legEJYJH3WdRSlu0Q2IKH8x5skuAH1ut3lFkcRWs252LA1PQCg+cV0HbbGU5RTRCxNx/bplTQA1uFaGmhT7Hz8OIC3EBsb66yN5Y2GDRvCbDZXEsvZ2dlITU31+J3U1NQarV8fCHZlD26qmjLM6Fjd5ulmDIp7fp2eBXYYhtEHZS7o5E7iVJN3v1lTSO1KohsAIuKpeODnI4CMgZSDCqjFwEpyXf+vyaTWPIhJpjmZrXF0M6vYyJ1ON6qv49JcZ0kIFNobV6Z6GrUn0d1znG8DFGk9KH/VFE6DL78+QSKgYVuqkL3hI1qvy63AmnfI85zWg4QOQML4yGryhLvPvZ5+EbDxY6pwr8xR707GpSS6E9IrF7TSim7l/IhKBMbN87ytbrepojutJ/3+PUt8L6bY9VYS3XsX0LSg25V5xWWKKAGo8voGR+G0EJkiMiBUF8oeZvE+S0xSK7WGgjdMZuDyfwCzHZ7eBi3UwUeF/n+nh5YrplKqhCWGBiT3/ESDm4OfQZXENKIaG1rSulM0yqFlFK5ey4ENi8WCXr16YcmSJbjhhhsAUKX6JUuWYOLEiR6/079/fyxZsgSPPvqoc9nixYtdPOX1DYvuOnLlGytgifcSHlVD1DmuxRPdkeGu1csZgxLrJrq1VWIZhhGDFgOBW7/0/xzggUDJ0QVoGr8WF1OYd1yaZv4ribzBst37fNPN+gKTdpH4UcQSE5xc+X809Vt/zzfblWgxkJ7TL6LBn+SOFFnQ+SbyIG/8hAZ24ptSEbf9i1y91q0up+JRSlE/LW2uIoHSfIDnueYBErobPqa8e3cU4ZTc0bfUhkbtyAN6ZA3Q6jJapi08Vx2pXWmw7vQuNdddSTdT6HcfRY+cO6DPlIdM/dFuOEVjHF1LU5D6MkglSWpkBUDHQ11oM8QvM59MmjQJ48ePR+/evdG3b1/MmDEDhYWFztpR48aNQ5MmTTBt2jQAwCOPPIJBgwbh9ddfx4gRIzB79mxs2rQJH330UZ3bUltYdNeR/JIKhFnK/ba9RrERSIr2cuE2MJEW13m6GYNiTaCwOaUKpfv8kAzDGB9JosI/wYCL6HZ4+jx5qSWJCqFVhVWMYphMHUluTw9fadYXuPM3yiUFqJryn3PIoxuVCNy9mFI2JAkY9DSFxvfTTPfXchBVkE5qU3nb1jgKB6+KRu2AZ456/qzFxcDNMynlwVdum0Vhydr5pX1Fkqh46vd3qWHxN30KzLmdCmBFJdEAxOhZNd82YzwkiY6XU9toikSBGTVqFM6cOYMpU6YgKysL3bt3x4IFC5zF0o4ePeoyP/uAAQMwa9YsPP/883j22WfRpk0bzJs3L2BzdANcSK3WKIXUVm79C43T/DfFQFqC1ZkfLRLvLjuAVxfuxW19mmH6TV5CrBhj8GZnmmYmoTnw6PZAt4ZhmFDGVg78uwnd8N/0KeWcMgyjLxVl5OEPjwR63wl8dweFl7cdBvxtTrVfZ5i6okdBaqMjnrozGBkNo9E0WaxK43qg5HQXsafb+MSkkOjm0HKGYQKNOZzyrA8to7BZhmH0J8zimss76GmaV9nXkH2GYWoMi27GLyg53cWc02184psAJzbVLA+MYRhGL0Z+GOgWMExok9weuPPXQLeCYYIaFt2MX4iykOjmKcMEYNBTVMWyx+2BbgnDMEy9cjqvBKfzSwPdDFjDzWjVKBqS6IXwGIZhGJ9g0c34BQ4vF4iUTlT9lWEYJkSQZRkfrTyElxf8BbtBKtlMvbYj7rw4I9DNYBiGYeoBFt2MX4i08DzdDMMwjCu/bD+JL9Ychi3ASrew1Ia92fkAaJYQcwA9zGU2O84XluHHrSdYdDMMw4QILLoZv8Dh5QzDMIyWEznFePL77YaJgAozSZh6bUfcflHzgIZ1n8kvRd9//47tx3NxKrcYjeMjA9YWhmEYpn5g0c34hUgOL2cYhmE0vPDTLhSV2dAzPQH3D/Iw93Y90zYlFi0aRge6GWgUG4Ge6Q2w+cgF/L47G2P7twh0kxiGYRidYdHN+AUlpzsrrwS9Xloc4NYw3jCbJFzSuiFG9myK+MjwgLZFhox1h85h7pYTOFtQVWEjzx4pb46qqvxX3r/jz9+omQfN2+rtU2Px0g2dDeEFq7DZ8eqivfhxy4mAhwkbjYgwE+66JAN3X5JhiKJYu07m4tm5O3D8QnFA2yEDOF9YhjCThOk3dUXblNiAtsdoXNUxBZuPXMAiFt0M4xfySsrxj3k78cf+s4FuCgDgopZJeHt0D5hMge8XGGPAopvxC2kJViRGW3C+sAznCssC3RymCuZuPYG5W08EuhlMNRy/UIxtx/7A7Rc1R1iAO+1V+89ifeb5gLbByPxz/h5sPHweXZrEB7QdRWU2zFydiZJye0DboeXvg1ux4PbAlR1TMO23v7D24Dm8vWS/18G3+qJTk3hc1i45sI0wKDa7jDkbj+F8YeCr3jOekWXgx20ncOhMYaCb4mT+jlO4uHVD/K1feqCbwhgESZZldlvUguPHj6NZs2Y4duwYmjZtGujmGIL8knKczCkJdDOYKrhQVIYfNh/HmoPnYDfAqd843opbejdDj/QEj95mGZ7b6K3pVf2lmm7L63a8/baX7Vf9Hc8UlVXgpV/2YM+pvJo1TkeiLWb868Yu6NA4LtBNMRSrD5zFv3/dgwoDRQBc2rYRnhrWDmEmU0DbYQkzoUVSlCEiAIzIlW+swP7TBYFuhpORPZrgzoszAj4AEGUxI6OhcaZTe3fZAby6cG+gm8H4QON4K16+qStS4qwBbceiXVl4ffE+xEeG49PxvZ3RoIxK9qkTGNKnU0jpKBbdtYRFN8MwelLs8Foev1AU6KYgIsyMMf3S0YY9lh7ZfOQC5m09gQp74D3M7VPjcPtFzWHmkEbDs+1YDr7bdCzgA6CFpTb8sv2kYaZSA4ARXRvj1Zu7IsoS2IDMA6fzcfVbf6DMZsfVXVIDnpbFeCchyoK7Ls5Ao9iIQDcFFTY7rntnNXYbaODcaFTkncWJ9+8IKR3ForuWHD16FM2bN8eGDRvQuHHjQDeHYRiGYRimVmw7dgFv/b4fZwsCnx52vrAMNruM1HgrkgMsoE7lluBMfin6t0rEG7d2N4z3nTE+e7PyMPWn3Sgqqwh0UwxJWd457Hp/Io4cOYL09NAIwWfRXUs2btyIvn37BroZDMMwDMMwDMMwwrFhwwb06dMn0M2oF1h015KKigps3boVKSkpMAU4b44h8vPz0bFjR+zevRuxsRwGa1R4P4kD7ysx4P0kDryvxID3kzjwvhID9/1kt9uRnZ2NHj16ICwsNOp6s+hmgoa8vDzEx8cjNzcXcXFc7Mmo8H4SB95XYsD7SRx4X4kB7ydx4H0lBryfAHbRMgzDMAzDMAzDMIxOsOhmGIZhGIZhGIZhGJ1g0c0EDREREZg6dSoiIgI/XQTjHd5P4sD7Sgx4P4kD7ysx4P0kDryvxID3E+d0MwzDMAzDMAzDMIxusKebYRiGYRiGYRiGYXSCRTfDMAzDMAzDMAzD6ASLboZhGIZhGIZhGIbRCRbdDMMwDMMwDMMwDKMTLLoZhmEYhmEYhmEYRidYdDO6MG3aNPTp0wexsbFITk7GDTfcgL1797qsk5WVhbFjxyI1NRXR0dHo2bMnfvjhhyq3u3LlSlx77bVIS0uDJEmYN29elevff//9kCQJM2bMqLbNy5cvR8+ePREREYHWrVvj888/r7TOu+++ixYtWsBqtaJfv37YsGFDtds1OqLtqz/++AMXX3wxkpKSEBkZifbt2+PNN9+stF6w7atA7qc77rgDkiS5PIYNG1Ztm0PxnBJtP4Xq+QQE/tq3Z88eXHfddYiPj0d0dDT69OmDo0ePVrnt7du3Y+DAgbBarWjWrBleeeWVSut89913aN++PaxWK7p06YJff/21akMYHNH20969e3HZZZchJSUFVqsVLVu2xPPPP4/y8nKX9YJtPwGB3Vfu1z7l8eqrr1a5be6njL+fgqafkhlGB4YOHSp/9tln8s6dO+Vt27bJV199tZyeni4XFBQ417nyyivlPn36yOvXr5cPHjwov/TSS7LJZJK3bNnidbu//vqr/Nxzz8lz586VAcg//vij13Xnzp0rd+vWTU5LS5PffPPNKtt76NAhOSoqSp40aZK8e/du+e2335bNZrO8YMEC5zqzZ8+WLRaLPHPmTHnXrl3yhAkT5ISEBDk7O9tnuxgR0fbVli1b5FmzZsk7d+6UMzMz5a+++kqOioqSP/zwQ+c6wbivArmfxo8fLw8bNkw+deqU83H+/Pkq2xuq55Ro+ylUzydZDuy+OnDggJyYmChPnjxZ3rJli3zgwAH5f//7X5U2zc3NlVNSUuQxY8bIO3fulL/55hs5MjLSZV+tXr1aNpvN8iuvvCLv3r1bfv755+Xw8HB5x44dtTOSARBtPx08eFCeOXOmvG3bNvnw4cPy//73Pzk5OVl+5plnnOsE436S5cDuK+1179SpU/LMmTNlSZLkgwcPet0u91Ni7Kdg6adYdDP1wunTp2UA8ooVK5zLoqOj5S+//NJlvcTERPnjjz/2aZtVCbnjx4/LTZo0kXfu3Ck3b968WiH35JNPyp06dXJZNmrUKHno0KHO93379pUffPBB53ubzSanpaXJ06ZN86m9omD0feWJG2+8Ub799tud70NhX9Xnfho/frx8/fXX16h9fE4RRt9PngjF80mW63dfjRo1ysXGvvDee+/JDRo0kEtLS53LnnrqKbldu3bO97feeqs8YsQIl+/169dPvu+++2r0W0bG6PvJE4899ph8ySWXON+Hwn6S5fq/n9By/fXXy5dffnmV63A/RRh9P3lCxH6Kw8uZeiE3NxcAkJiY6Fw2YMAAzJkzB+fPn4fdbsfs2bNRUlKCwYMH1+m37HY7xo4di8mTJ6NTp04e1xk8eDDuuOMO5/u1a9diyJAhLusMHToUa9euBQCUlZVh8+bNLuuYTCYMGTLEuU6wYPR95c7WrVuxZs0aDBo0CEDo7Kv63E8AheAlJyejXbt2eOCBB3Du3DmXz/mc8ozR95M7oXo+AfW3r+x2O+bPn4+2bdti6NChSE5ORr9+/SqFYt5xxx0uv7N27VpceumlsFgszmVDhw7F3r17ceHCBec6VZ13wYDR95M7Bw4cwIIFC5znFBAa+wmo/+ufQnZ2NubPn4+7777bZTn3U54x+n5yR9R+ikU3ozt2ux2PPvooLr74YnTu3Nm5/Ntvv0V5eTmSkpIQERGB++67Dz/++CNat25dp997+eWXERYWhocfftjrOunp6WjcuLHzfVZWFlJSUlzWSUlJQV5eHoqLi3H27FnYbDaP62RlZdWpvUZChH2l0LRpU0RERKB379548MEHcc899wBASOyr+t5Pw4YNw5dffoklS5bg5ZdfxooVKzB8+HDYbDbnOnxOVUaE/aQQyucTUL/76vTp0ygoKMD06dMxbNgwLFq0CDfeeCNGjhyJFStWONdr3Lgx0tPTne+9nVPKZ1WtEyz7SoT9pDBgwABYrVa0adMGAwcOxP/93/85Pwv2/QTU//VPyxdffIHY2FiMHDnSZTn3U5URYT8piN5PhQW6AUzw8+CDD2Lnzp34448/XJb/4x//QE5ODn7//Xc0bNgQ8+bNw6233opVq1ahS5cutfqtzZs346233sKWLVsgSZLX9b788stabT/YEWlfrVq1CgUFBVi3bh2efvpptG7dGqNHj65VW0SjPvcTANx2223O1126dEHXrl3RqlUrLF++HFdccQUAPqc8IdJ+CuXzCajffWW32wEA119/PR577DEAQPfu3bFmzRp88MEHTu/NtGnT6vCPghOR9tOcOXOQn5+PP//8E5MnT8Zrr72GJ598slZtEZH6vv5pmTlzJsaMGQOr1eqynPupyoi0n4TvpwId384ENw8++KDctGlT+dChQy7LDxw4IAOQd+7c6bL8iiuu8DmnCR7yRd58801ZkiTZbDY7HwBkk8kkN2/e3Ou2Bg4cKD/yyCMuy2bOnCnHxcXJsizLpaWlstlsrvR748aNk6+77jqf2mt0RNlXnnjppZfktm3byrIc/PuqvveTNxo2bCh/8MEHXj8P9XNKlP3kiVA6n2S5/vdVaWmpHBYWJr/00ksuy5988kl5wIABXrc1duzYSjn7S5culQE4C+Y1a9asUl2MKVOmyF27dvWpvUZGlP3kia+++kqOjIyUKyoqZFkO7v0ky4G9/q1cuVIGIG/btq3abXE/JcZ+8oSI/RSHlzO6IMsyJk6ciB9//BFLly5FRkaGy+dFRUUAKOdCi9lsdo4u14axY8di+/bt2LZtm/ORlpaGyZMnY+HChV6/179/fyxZssRl2eLFi9G/f38AgMViQa9evVzWsdvtWLJkiXMdURFtX3nCbrejtLQUQPDuq0DtJ08cP34c586d8xj+pRCq55Ro+8kToXA+AYHbVxaLBX369Kk0Rc++ffvQvHlzr9/r378/Vq5c6TL11OLFi9GuXTs0aNDAuU5V552IiLafPGG321FeXu5sTzDuJ8AY179PP/0UvXr1Qrdu3apdl/spMfaTJ4TspwKn95lg5oEHHpDj4+Pl5cuXu0wNUFRUJMuyLJeVlcmtW7eWBw4cKK9fv14+cOCA/Nprr8mSJMnz58/3ut38/Hx569at8tatW2UA8htvvCFv3bpVPnLkiNfveKqIPXbsWPnpp592vlemjZg8ebK8Z88e+d133/U4bURERIT8+eefy7t375bvvfdeOSEhQc7KyqqllYyBaPvqnXfekX/66Sd537598r59++RPPvlEjo2NlZ977jnnOsG4rwK1n/Lz8+UnnnhCXrt2rZyZmSn//vvvcs+ePeU2bdrIJSUlzu3wOUWItp9C9XyS5cBe++bOnSuHh4fLH330kbx//37nVEWrVq1yrvP000/LY8eOdb7PycmRU1JS5LFjx8o7d+6UZ8+eXWnanNWrV8thYWHya6+9Ju/Zs0eeOnWq8FNRibaf/vvf/8pz5syRd+/eLR88eFCeM2eOnJaWJo8ZM8a5TjDuJ1kO/P1Ebm6uHBUVJb///vset8P9FCHafgqWfopFN6MLADw+PvvsM+c6+/btk0eOHCknJyfLUVFRcteuXStNT+DOsmXLPG53/PjxXr/jScgNGjSo0neWLVsmd+/eXbZYLHLLli1d2qrw9ttvy+np6bLFYpH79u0rr1u3rhpLGB/R9tV//vMfuVOnTnJUVJQcFxcn9+jRQ37vvfdkm83m8r1g21eB2k9FRUXyVVddJTdq1EgODw+XmzdvLk+YMKFSR8bnFCHafgrV80mWA3/t+/TTT+XWrVvLVqtV7tatmzxv3jyXz8ePHy8PGjTIZdmff/4pX3LJJXJERITcpEkTefr06ZV+/9tvv5Xbtm0rWywWuVOnTlXeJIuAaPtp9uzZcs+ePeWYmBg5Ojpa7tixo/zvf/9bLi4udvlesO0nWQ78vvrwww/lyMhIOScnx+N2uJ8iRNtPwdJPSbIsy9X7wxmGYRiGYRiGYRiGqSmc080wDMMwDMMwDMMwOsGim2EYhmEYhmEYhmF0gkU3wzAMwzAMwzAMw+gEi26GYRiGYRiGYRiG0QkW3QzDMAzDMAzDMAyjEyy6GYZhGIZhGIZhGEYnWHQzDMMwDMMwDMMwjE6w6GYYhmEYhmEYhmEYnWDRzTAMwzAMwzAMwzA6waKbYRiGYRiGYRiGYXSCRTfDMAzDMAzDMAzD6ASLboZhGIZhGIZhGIbRCRbdDMMwDMMwDMMwDKMTLLoZhmEYhmEYhmEYRidYdDMMwzAMwzAMwzCMTrDoZhiGYRiGYRiGYRidYNHNMAzDMAzDMAzDMDrBopthGIZhGIZhGIZhdIJFN8MwDMMIyK5du3D77bejSZMmiIiIQFpaGsaMGYNdu3a5rLdx40ZMnDgRnTp1QnR0NNLT03Hrrbdi3759AWo5wzAMw4QWkizLcqAbwTAMwzCM78ydOxejR49GYmIi7r77bmRkZODw4cP49NNPce7cOcyePRs33ngjAODmm2/G6tWrccstt6Br167IysrCO++8g4KCAqxbtw6dO3cO8L9hGIZhmOCGRTfDMAzDCMTBgwfRtWtXpKenY+XKlWjUqJHzs7Nnz2LgwIE4duwYtm/fjpYtW2LNmjXo3bs3LBaLc739+/ejS5cuuPnmm/Hf//43EH+DYRiGYUIGDi9nGIZhGIF49dVXUVRUhI8++shFcANAw4YN8eGHH6KwsBCvvPIKAGDAgAEughsA2rRpg06dOmHPnj311m6GYRiGCVXY080wDMMwAtGkSRNYLBZkZmZ6XScjIwMVFRU4duyYx89lWUazZs3QqVMnLFy4UK+mMgzDMAwD9nQzDMMwjDDk5ubi5MmT6NatW5Xrde3aFcePH0d+fr7Hz7/++mucOHECo0aN0qOZDMMwDMNoYNHNMAzDMIKgiOjY2Ngq11M+z8vLq/TZX3/9hQcffBD9+/fH+PHj/d9IhmEYhmFcYNHNMAzDMIKgiGlvHmwFb+I8KysLI0aMQHx8PL7//nuYzWZ9GsowDMMwjBPO6WYYhmEYgUhLS4PVasWhQ4e8rpORkYHy8nIcP37cuSw3NxeDBw/G0aNHsWrVKnTs2LE+msswDMMwIQ97uhmGYRhGIK655hpkZmbijz/+8Pj5qlWrcPjwYVxzzTXOZSUlJbj22muxb98+/PLLLyy4GYZhGKYeYU83wzAMwwjE/v370a1bN2RkZGDlypVISkpyfnb+/HkMHDgQhw8fxvbt29GqVSvYbDaMHDkSv/76K/73v//h6quvDmDrGYZhGCb0YNHNMAzDMILx3XffYcyYMWjYsCHuvvtuZGRk4PDhw/j0009x9uxZfPPNNxg5ciQA4NFHH8Vbb72Fa6+9Frfeemulbd1+++313XyGYRiGCSlYdDMMwzCMgOzYsQPTpk3D8uXLcfbsWSQlJeGyyy7Ds88+i86dOzvXGzx4MFasWOF1O3wbwDAMwzD6wqKbYRiGYRiGYRiGYXSCC6kxDMMwDMMwDMMwjE6w6GYYhmEYhmEYhmEYnWDRzTAMwzAMwzAMwzA6waKbYRiGYRiGYRiGYXSCRTfDMAzDMAzDMAzD6ERYoBsgKhUVFdi6dStSUlJgMvHYBcMwDMMwDMMwTHXY7XZkZ2ejR48eCAsLDTkaGv9SB7Zu3Yq+ffsGuhkMwzAMwzAMwzDCsWHDBvTp0yfQzagXWHTXkpSUFAB0sDRu3DjArWGY2rF83xk8/f12AMCL13fC0E6pAW4RwzAMwzAME8ycOnUKffv2deqpUIBFdy1RQsobN26Mpk2bBrg1DFM7Cg6WIiyuIQBg+wUz7uZjmWEYhmEYhqkHQilFN3T+KcMwlcg8U+h8vXzvGVTY7AFsDcMwDMMwDMMEHyy6GSaEyTyriu7c4nJsOZoTuMYwDMMwDMMwTBDCopthQphDDtHdOjkGAPDUD9tx31ebcORcYVVfYxiGYRiGYRjGRzinW2dsNhvKy8sD3QzGT4SHh8NsNge6GX4hr6QcZwtKAQAPDGqFx7/7E5lnC5F5thBtU2Lx+FXtAtxChmEYhmEYhhGfoBTdK1euxKuvvorNmzfj1KlT+PHHH3HDDTe4rLNnzx489dRTWLFiBSoqKtCxY0f88MMPSE9P90sbZFlGVlYWcnJy/LI9xjgkJCQgNTUVkiQFuil14rDDy90wJgIjezZBSpwVX68/gt92ZiGniAeKGIZhGIZhGMYfBKXoLiwsRLdu3XDXXXdh5MiRlT4/ePAgLrnkEtx999148cUXERcXh127dsFqtfqtDYrgTk5ORlRUlPACjaGBlKKiIpw+fRoAhJ8qTsnnbtkoGpIk4ZI2DfFXVh5+25mFvBIW3QzDMAzDMAzjD4JSdA8fPhzDhw/3+vlzzz2Hq6++Gq+88opzWatWrarcZmlpKUpLS53v8/Pzva5rs9mcgjspKakGLWeMTmRkJADg9OnTSE5OFjrU/JCjcnnLhtHOZXGR4QCAvGIW3QzDMAzDMAzjD0KukJrdbsf8+fPRtm1bDB06FMnJyejXrx/mzZtX5femTZuG+Ph456Njx45e11VyuKOiovzZdMYgKPtV9Fx9xdOdoRXdVofoLqkISJsYhmEYhmEYJtgIOdF9+vRpFBQUYPr06Rg2bBgWLVqEG2+8ESNHjsSKFSu8fu+ZZ55Bbm6u87F79+5qf4tDyoOTYNmvHkV3JAW/sKebYRiGYRiGYfxDUIaXV4XdbgcAXH/99XjssccAAN27d8eaNWvwwQcfYNCgQR6/FxERgYiICOf7vLw8/RvLMDpy7EIRAKB5kidPN4tuhmEYhmEYhvEHIefpbtiwIcLCwiqFh3fo0AFHjx4NUKsYpn4pKbc5K5SnxqkFBOOdOd0cXs4wDMMwDMMw/iDkRLfFYkGfPn2wd+9el+X79u1D8+bNA9Sq4OXw4cOQJAnbtm0LdFMYDWfyqShgRJjJGVIOqJ7u4nIbyirsAWkbwzAMwzAMwwQTQRleXlBQgAMHDjjfZ2ZmYtu2bUhMTER6ejomT56MUaNG4dJLL8Vll12GBQsW4Oeff8by5csD12iGqUey80oAAClxVpcc9RireknILylHUkxEpe8yDMMwDMMwDOM7Qenp3rRpE3r06IEePXoAACZNmoQePXpgypQpAIAbb7wRH3zwAV555RV06dIFn3zyCX744QdccsklgWy24SgrKwt0ExidOO3wdCfHuopqs0lCTAQJ73yuYM4wDMMwDMMwdSYoRffgwYMhy3Klx+eff+5c56677sL+/ftRXFyMbdu24frrr9etPbIso6isIiAPWZZ9bufgwYMxceJEPProo2jYsCGGDh2KnTt3Yvjw4YiJiUFKSgrGjh2Ls2fPOr+zYMECXHLJJUhISEBSUhKuueYaHDx4UA8zMn5E6+l2J87h7eZiagzDMAzDMEygWblyJa699lqkpaVBkqRqp3oGgOXLl6Nnz56IiIhA69atXXRgIAjK8HKjUVxuQ8cpCwPy27v/byiiLL7v5i+++AIPPPAAVq9ejZycHFx++eW455578Oabb6K4uBhPPfUUbr31VixduhQAUFhYiEmTJqFr164oKCjAlClTcOONN2Lbtm0wmYJyTCcoyM4jT3ej2Mrh43GR4TiZW8LF1BiGYRiGYZiAU1hYiG7duuGuu+7CyJEjq10/MzMTI0aMwP3334+vv/4aS5YswT333IPGjRtj6NCh9dBiD8hMrTh27JgMQD527JjL8pKSErlbt26y2Wx2eUAy1fphjmskN/n753KzR7+VwxKb1uy7bu2o6gFABuB8bzKZZEmSPK5X3XaeeeYZWZZlefr06dWuH8hH8+bN5RMnTsgVFRXyxRdfHPD2VPewWCzyv//9b1mWZfnxxx+Xw8LCarUdyUTHpMlUxWd1bGubNm3kCxcuyNnZ2XJGRkbAbVfdIzY2Vv7hhx9kWZblCRMm+GWbYWFh8sSJE2VZluVZs2bJ0dHRfm3z2LFjZVmW5Z9//lmOi4sLuA2D6cG21eeh9zkRyo/w8HD5ueeek2VZlj/88EM5MjIy4G2q7tG5c2e5qKhIzszMlJs2bRrw9nh7WCwW+eWXX5ZlWZZfffVVOSIiIuBtqu7Rt29fuby8XN65c6ecmprqcZ0rr7xSttvt8tq1a+WkpCTZbDbLrVq1ks+dO6fnLbSu2O12+brrrgu4/QP1GD58uGy32+VVq1bJiYmJAW+Pt4fJZJIByLt375Zzc3Odj5KSkmr3MQD5xx9/rHKdJ598Uu7UqZPLslGjRslDhw6ty+FVJ1h01xJvonvHjh1O8erPx6w538pLV6zUZdt6PLp16ybLsiwPGTIk4G2p7jF37lzn/hTh0adPH1mWZblFixYBb0t1j6VLl8rz588PeDt8fdx9992yLMtyQkKC37aZmpoqy7Isjxkzxu/ttVqtsizL8oMPPhhw2wXbg22r30PPcyLUH+3bt5dlWZZHjBgR8Lb4+ti8ebM8a9asgLejusdFF10ky7IsX3zxxQFvi6+PgwcPyu+9916V61y4cEH+5z//6bJs0aJFOt5B60tJSUnA7R7oR35+vvzCCy8EvB21eUydOrXafQxUL7oHDhwoP/LIIy7LZs6cKcfFxdX+4KojHF7uZ+x2mmapQYMG2LRpE6zWyjmzNeHGG2/Ehg0bYDGbYDFTuHbz5s2xZs2aOrfVnZtuugmdOnXC//3f/wEAxowZg8jISDz33HOV1k1JSUFUVBQuvfRSNGnSBH//+9+RmpqK33//Hf/85z+ddlCeH3vsMTzxxBN+b3NduOGGG7Bx40bIsuxsp8ViQWZmZpXfKykpwbFjx9CsWbM679+a8Pvvv2P8+PHOPH2lzfPnz0f37t1rtK3bPlyLQ2cL8c7onujbMhHY/RPw22Sg2UV4IWIyft2ZhYevaI3bL2pRq7Zedtll2Ldvn4ttu3Xrhl9//bVW29Obt99+G9OnT69k21WrVqFly5a12ubu3btx5ZVXVtrmCy+8gAkTJtSpvSdOnEDfvn0rbduI55loHD9+HP369atk20mTJuHxxx8PZNOEZ8+ePRgyZIgu50Sos27dOtx0002VbPvGG29g1KhRgWyaV3r37o1Tp0659BOXXHIJ5syZE+CWubJo0SLceeedlWz76aefYtiwYYFsmlfatm2LwsJCF9uOGDECH330EQCgvLwcLVq0AACXdRTkGtQDMhra/7J3717ExMQEsDX1R0lJCVq1agXAdZ+OGzcO06ZNC2TTPHLq1Cn07t0bu3fvRpMmTZzLIyL8M2tOVlYWUlJSXJalpKQgLy8PxcXFiIyM9Mvv1AQW3X5GuVCZzWakpaXVWZQpB5/sKAYHkDBMS0urW0O9/FZMTIxz2wMGDMAPP/yAfv36ISys8qFy7tw5HDx4EJ999hkGDhwIAFi2bJmzvVri4uJ0aXNd8GRbk8lUbTtLSkpQWlrql/1bExo2bAhAta3ynJycXGPb5kgxCIu1olOb5khLiQW2HwViTUDJXqRlpCHsSDlM0Um13mcWi8XZRqWdERERhjsGFOLj4wFUtm1qamqt26wUHHTfZkJCQp3tYLPZPG7biOeZaFRUUC0Dtq3/OXfuHAB9zolQp1GjRgAq2zYxMdGwtlXuK7T9RGRkpOHam5SUBKCybZOSat9H6o1SU0dr26ioKGd7leuc+zraZaKibXtaWlpIiW4F7T6NjY015HGqDArExsYiLi4uwK2pH7jSlZ9RDnLt3Md1QdmO9gTy17ar48EHH8T58+cxevRobNy4EQcPHsTChQtx5513wmazoUGDBkhKSsJHH32EAwcOYOnSpfjqq69ctlHfba4JgbRtbdC2V/tc0zaXlNuQ55gOLFmpXn56l+PDXKSZzgOoW/XyULWt3tusj22HOmxb/WDb6oeIthWlnwhG22pfB7PoNvI+8jfe9mko2UBLamoqsrOzXZZlZ2cjLi4uIF5ugEW33wkm0Z2WlobVq1fDZrPhqquuQpcuXfDoo48iISEBJpMJJpMJs2fPxubNm9G5c2c89thjGDt2rLO92mcjnvSidPgK/ur4Tzsql0eEmZzTgyF7t/Pz9IrDAIC8YhbdLLpDD7atfrBt9UNE24rSTwSjbVl0Bx8sul3p378/lixZ4rJs8eLF6N+/f4BaxOHlfsffB7kSImS323U/gZYvX15pWZs2bTB37lyv3xkyZAh271YF23fffQdADcdSQriV3CEjoe2UlDAXI09z5q+OPztfnaNbkiSg8CxQeNr5eVrpIQBJTm94XdsqwoWfRTejwLbVD7atfohoW1H6iWC0bXWi2z3HWyS0bTfyPZ2/0f7X+tAM9U1BQQEOHDjgfJ+ZmYlt27YhMTER6enpeOaZZ3DixAl8+eWXAID7778f77zzDp588kncddddWLp0Kb799lvMnz8/UH+BPd3+Jpg83bVBpM4pVG2reLpT4hzFKk7vdvm8YRFd1NjTzaI7FGHb6gfbVj9EtK0o/UQw2pY93cFHsHu6N23ahB49eqBHjx4AqLhpjx49MGXKFABUmO3o0aPO9TMyMjB//nwsXrwY3bp1w+uvv45PPvkkcHN0gz3dfsffFypROiUFkTqnULVtdh55upNjHfncSmh5eDRQXoj4vP0A/JPTLcpoq7a9gD6iW9m2P0W3P9vLEHrut1CHbasfItrWU7SZEdsrsm196YO16yiw6BaPYBfdgwcPrvK4/Pzzzz1+Z+vWrTq2qmawp9vP+DtMWTRhqA2HB4wtBuozdN8feOv4a3qsKeHlyU5Pt6OIWodrAACReQcRhgrkFdc+vFxbOVWE0H1tewH/nMfezgV/2MF9GyLYWBTcjwV/7rdQR89zItQR0bae+mAjtldk21bVB3taR4FFt3gEu+gOBox7xRAUDi9nT7de+D+83M3T3XYoYImFyV6ODCmLq5ejbm2uj/ByZbsi2FgU2Lb6IVL/IBoi2laUfiJYbetpHQUW3eLBfZfx4fByP+PvgzxUvbH1gafQNiO2U8FfHf+p3GIAjpxuux04vYc+SOkMJHcAjm/AJ+Gv4QwSUF50EcKjEurUVhGPWxbdoQvbVj9EFC+iIKJtRekngtW2VYluLqQmJpIkOe9pRThOQ43QOhrrAfZ0i9M5haptT+VSeHlafCSQewwoLwTMFiCxFZDWHQDQ3HQavU37ULJ/VZ3bGkq21Xub7ttWtiuCjUWBbasfIvUPoiGibUXpJ4LVtuzpDj5EOadCFRbdfsbfFyrRTiCROqdQtK0sy6roTogELmTSBw1aAOYw4NLJwBVTkCk3BgCUFObWua2hYtv62Kb7tpXtimBjUWDb6odI/YNoiGhbUfqJYLUti+7gQ5RzKlRh0e1n2NMtTucUirY9V1iGsgo7JMmR033hMH3QoAU9xyQDAx/HcXNTAEBpUX6d2xoqtq2PbbpvW9muCDYWBbatfojUP4iGiLYVpZ8IVtuGgugONUQ5p0IVFt1+xt+5waLldLtX+TRyrrS2rUZup4Ky3+ti25M5lM/dKCYCljBTZdHtoNwcBQAoK6qdp1s02+pRnVbPc0G7DVFsLApsW/0QqX8QDRFtK0o/Eay29bSOQjDkdBt5/+iFaJoh1Ai9I1Jn9PR0i3Ah8XVEuEWLFpgxY4bL9+bNm1fn367JNkQbEfQ2lVFN2nwyh0LLGydE0oJqRLe9tKBWbRXNtt5G/NnTHXqwbfVDRI+hKIhoW1H6iWC1bbB7uo28f/RCNM0QanD1cj/D4eW165xOnTqFBg0a+PQbL7zwAubNm4dt27bVehvubQ1m22pRKpenxTumC6tGdMulhXVuq2i2ZdEd2rBt9UNE8SIKItpWlH4iWG3Lojv4EOWcClVYdPsZf1+ojHwClZWVwWKxuCyrbeeUmppa5/bUdBtGtq0n/CO6HZ7u+Ko93bYwEt1SGYtuf2xT+8yi29iwbfVDRPEiCiLaVpR+Ilhty6I7+BDlnApVOObAz3g8yGUZKCus1UOSbQAAe1kx5DLyUkqowfZqcOEcPHgwJk6ciIkTJyI+Ph4NGzbEP/7xD+d/atGiBV566SWMGzcOcXFxuPfeewEAf/zxBwYOHIjIyEjcfvvtAACbzdFuR3jLv/71L0RGRiIjIwNff/11pd92Dw0/fvw4Ro8ejcTERERHR6N3795Yv349Pv/8c7z44ov4888/IUkSJEnC559/7nEbO3bswOWXX47IyEgkJSXh3nvvRUGBGi6teMoXLlyIK6+8EgCQm5uL8vJyn21Wn/ij41dyutMSrEBxDlB8gT5IaO6yni3cIbrLObzcH9vUPrPoNjZsW/0QUbyIgoi2FaWfCFbbViW6gyGn28j7Ry+0tX9EOE5DDfZ0+xmPB3l5EfDvtFptz7S/iLb729OQ42ib0smtvm/v2ZOAJdrn3/viiy9w9913Y8OGDdi0aRPuvfdepKenY8KECQCA1157DVOmTMHUqVMBAAcPHsSwYcPwz3/+EzNnzsT8+fPx2GOP4fjx49Ruhz3Onj2LZcuWITw8HA8//DBOnz7ttQ0FBQUYNGgQmjRpgp9++gmpqanYsmUL7HY7Ro0ahZ07d2LBggX4/fffAQDx8fGVtlFYWIihQ4eif//+2LhxI06fPo177rkHEydOdIp0hdOnT+ODDz7ALbfcgqKiInz++efO/2sk/Cu6I4GcI7QwuhEQEeOyni2MjhlTeVGd2yrChZ9FN6PAttUPEcWLKIhoW1H6iWC1bbB7ukMxl1lb+0eE4zTUYNHtZ/ye061sF6rTWs/zp1mzZnjzzTchSRLatWuHHTt24M0333SK0MsvvxyPP/64c/177rkHY8aMwaOPPgoA6Ny5MwDgwoULKCkpQXExibyHHnoIF110EQDg008/RYcOHby2YdasWThz5gw2btyIxMREAEDr1q2dn8fExCAsLKzKcPJZs2ahpKQEX375JaKjSUC+8847uPbaa/Hyyy8jJSXFue5tt92G5s3J0xsZGYklS5YErehWw8utwPlMWugWWg4AdguJcHNF3cLLRRlt9dRe7fK6bFPZnj9H37XbEMXGouAuukPZa+Jv9DwnQh0Rbatts5HbK7Jtq+ofvPV7QHCIbiPvH70QZSArVGHR7Wc8VgsMjyKPcy2QdtwO7JkH+ap/QU5LA+aMhtS0N/DsEt824AgT9pWLLrrI5QTt378/Xn/9dWe4eO/evV3W//PPP7F9+3ZnyLiyHgBkZmY6RXebNm2cy9u3b4+EhASvbdi2bRt69OjhFNy1Yc+ePejWrZtTcAPAxRdfDLvdjr179yIlJaXSzTUAmM3mKr3wgcRbx+/raG6FzY7sPBLdaQmRwNHD9IEH0S2Hk93MFbXzdGtHW0WooOmpvdrlddmmP6ch89QuUWwsCu5ThoWy18Tf6HlOhDoi2tbT9EZGbK/Itq2qf/DW7ynLRCWUxSaLbmPDotvPeLxQSVKNQrxdvmoOp+2aLZDDImiZyVzr7dUVrYgFKBT8vvvuw8MPPwwAWLduHcaOHYvWrVujVatWtTrpIyMj/dfgKvB2cTJqLlNdPd2n80thl4Fws4RGMRFei6gBgBRB+zm8jp5uUS78eoaXK3B4uRiwbfVDz3Mi1AnWEGgjEKy2DfbwciPvH70Q5ZwKVYw7TCco/j7I63ui+/Xr17u8X7duHdq0aQOz2exx/Z49e2L37t1o3bo1WrdujaZNmwIAwsPDYbFYEBFBAwWHDh1yfmfv3r3Iycnx2oauXbti27ZtOH/+vMfPLRaLi0fdEx06dMCff/6JwkJVNK5evRomkwnt2rVzWdfooW0Kde34lenCUuKsMJmkqkV3OIWXh9uK69xWES78eotuf9uBhaF+sG31g22rH8EqDI1AsNo22AupGTkSQS/qWzMwNSP0jkid8XtOdz13SkePHsWkSZOwd+9efPPNN3j77bfxyCOPeF3/qaeewpo1azBx4kRs27YNJ06cAABkZWUBgHNKsf/85z9Yv349Nm/ejHvuuadKb/bo0aORmpqKG264AatXr8ahQ4fwww8/YO3atQCoinpmZia2bduGs2fPorS0tNI2xowZA6vVivHjx2Pnzp1YtmwZHnroIYwdO9Ylnxvw3OEYkbp2/CdyHKHl1UwXBgCS1SG67VxIra7bdN8ui25jw7bVD7atfgSrMDQCwWpb9nQHH6KcU6EKi24/I7roHjduHIqLi9G3b188+OCDeOSRR5xTg3mia9euWLFiBfbt24eBAwfinnvuAQCnZ1xpc1JSEgYNGoSRI0fi3nvvRXJystdtWiwWLFq0CMnJybj66qvRpUsXTJ8+3bnNm266CcOGDcNll12GRo0a4Ztvvqm0jaioKCxcuBDnz59Hnz59cPPNN+OKK67AO++841ynqg7HiNS14892FFFLjbfSgoJseo5tXGldkzUWABBhZ093Xbfpvl0W3caGbasfbFv9CFZhaASC1bYsuoMPUc6pUIVzuv2M6KI7PDwcM2bMwPvvv1/ps8OHD3v8Tp8+fbBo0SIAwKpVq3DppZc6i6Apbf7nP//pnAsbAMaOHeuyDfcLfPPmzfH99997/L2IiAiPn7lvo0uXLli6dKnHbQBAv379cPjwYRfbJicnY/ny5V6/E0jq2vGfLyoDACRGW2gOd2U6sOhGldY1O6YQC5fLAVs54KgtUJu2inDhZ9HNKLBt9YNtqx/BKgyNQLDalkV38CHKORWqsKfbz/g7N1i0/AylbYodjJxbo22rkdup4G7bmh4POQ7R3SDKAhSepYXmCCAittK64VGaebvLCurUVtGOW71EN08ZJgZ67rdQh22rH+7CUATbatts5PaKbNvaThnGOd1iIppmCDVC74jUGT093SJc6LVTUGifjdhmbVuN3E4Fb6PtvnYsFwrLAQANosNV0R3dEPDwnyMskSiVHYEwZTWvYM5Thuk79ZQ38WJkG4uCtwENtm3d4enY9EPkaa2MfiyIbNuq+geeMiz4EE0zhBocXu5n6iO8XK8LvT/CqkUKwxItDMfTIAHge5svODzdCVEWoEgjuj1gDTejCFZEoKBWols024oWXq5sR9muCDYWBQ6B1g+2rX6I1PcqiNJPBKttObw8+KhPzcDUHN4Tfkb0nO66IlLnJLJtayMMc4ocnu6ocKDwDC2M8iy6Iy0mFMJRcK209uHloWLbqrbpvt1QvTaIBNtWH1h064dIfa+CKOdZsNqWRXfwIco5Faqw6PYzoX5jLVLnJLJt6+Lpdsnp9lBEDQAiwswokmmO9brkdIeKbavapvt2Q/XaIBJsW31g0a0fIvW9CqKcZ8FqWxbdwYco51SowqLbz/j7IBetKIK3giNGDG/xlPtixHYq1EUYyrLs9HQnaD3dXsLLIy1mFMIxnzeHl9dpm+7bZdFtfNi2+sCiWz+CVRgagWC1bVWimwupiYlomiHUCL0jUmdC/cZapM5JZNvWVBgWldlQZqOOqEGUBSg6Rx9UkdNd6PR01150i3Lh99Re7fK6bBPQp6iJaDYWCU8DcmzbuqP3ORHKeBvwNrJtRTnPRLZtbauXs6dbTES7rw01WHT7Gb1urEU5gbxV+TRim0UbEfR0gwL4NpqrhJZbzCZEWcxqeLmXnG5rmAlFjpxue2l+jdvqqeibkUed61qkrqptum/XX3YQzcYiwbbVB73PiVDG28whRratKOdZsNrWW7+nLBMVEe7n9EI0zRBqGPeKISjs6WZPt17UxdOtDS2XJEkTXu45p5vCy0l0lxfXXHSL4sFQ0KO92u/rMbAjmo1FgqMI9EHvcyKU0R6zgLH7XgVRzjORbVtV/+BtIF9ZJioi7B+9EO2+NtRg0e1nOKfbWDndkiRh3rx5Xj8DQiOn26WIGlB9eHmYGYUyiW5bSd1Et2jHrb/ayznd4sK21QfO6dYPdxuKYFtRzjORnAkKnNNt7Ps5vRBNM4QaPE+3nwn1G2ujdU6nTp1CgwYNPH4msm1rLro1RdRkudpCaiaThBITFVKzheiUYSy6Qxe2rT6w6NYPEW0rynlmtPsaX6ir6GZPt5iIck6FKqE3DKQzdT3IZVlGVl4JcovLXbYjyglUn51TWVlZteukpqYiIiLC42ci27bm4eUaT3dZIVBRQh94yekGgDKH6LaXsOiuyzb9vV337YtiY5Fg2+qDiMJQFES0rSjnGYtusRBh/+iFKOdUqMKi2894ulDJsozCwkKfHmcv5OFw1jkcPHUWhYWFqKioAACUlpaipISEkt1u93l7Nb1wfv/99+jSpQsiIyORlJSEIUOGoLCQqld/8skn6NChA6xWK9q3b4/33nvP+b3Dhw9DkiQsWbIEAJCVlYVu3bo52yxJEo4cOYJrr70WDRo0QHR0NDp16oRff/3VuY0VK1agb9++iIiIQOPGjfH00087/z8ADB48GBMnTsSjjz6Khg0bYujQodX+H214udLGuXPn4rLLLsPMmTMBAEePHhXi4lQnT3chDeI0iNZMFxYWCViivX6nzBRFv8ee7jpt09/bdd++KDYWCbatPogoDEVBRNuKcp6x6BYLEfaPXohyToUqHF7uZzwd5EVFRYiJianTdqdPn+58/dtvv/m8vYKCAkRHexdWWk6dOoXRo0fjlVdewY033oj8/HysWrUKsizj66+/xpQpU/DOO++gR48e2Lp1KyZMmIDo6GiMHz/euY23334bAJCUlIS2bdti165dAMgeDz74IMrKyrBy5UpER0dj9+7dzv9x4sQJXH311bjjjjvw5Zdf4q+//sKECRNgtVrxwgsvOLf/xRdf4IEHHsDq1at9+k+eeO655/Daa68hJSUFc+bMwbfffovhw4c722lUlLbVZlorJac7wX26sCq+WxEWBZTVTXSLklekR3vdb4J5yjBx0N64cJE6/6H3ORHKiGhbUc4zd9Ft5LYq+NI/eLunUJaJigj7Ry/4vsDYsOj2MyIXcDh16hQqKiowcuRING/eHADQpUsXAMDUqVPx+uuvY+TIkQCAjIwM7N69Gx9++KGL6L7zzjsxdepUSJKEF198Ed9//z0AssfRo0dx0003ObfZsmVL5/fee+89NGvWDO+88w4kSUL79u1x8uRJPPXUU5gyZYrTnm3atMErr7xSp//5xBNPYMSIEZg/fz4AIDc3FydPnnS206h4ukEBfGuzGl4eXm0+t0KFmTzdUnnN5+nWTkUiwjmhR3u139d2gHpMGSaCjUXCUzEatm3d0d78sW39i97XGz0Q5TzzNhWqEduq4Ev/4GkdhWDwdBt5/+iFp2ngWHQbBxbdfsbTQR4VFYWCAt+8hcfOFyGnuAySJKFzWjwmT56M999/H0888QSaNWuGRx55BNdccw1mz57t0/aioqJ8bnu3bt1wxRVXoEuXLhg6dCiuuuoq3HzzzbBYLDh48CDuvvtuTJgwwbl+RUUF4uPjXbbRrl07AGSHxo0bO5dLkoSHH34YDzzwABYtWoQhQ4bgpptuQteuXQEAe/bsQf/+/V3sdvHFF6OgoADHjx9Heno6AKBXr14+/x9vKL+p/a2cnJxKy4xGXeaSVgupWaqdo1vBFh4NFANSWc1Ft2ghThxezmhh2+qDiCHQoiCibUU5zzi8XCxE2D964ck5E4qDD0aFRbef8XSyS5Lkc4h3WKGMKCkcAAlmi4WmeAoPD3d57ev2aoLZbMbixYuxZs0aLFq0CG+//Taee+45/PzzzwCAjz/+GP369av0HS3h4dR2WZYr2eCee+7B0KFDMX/+fCxatAjTpk3D66+/joceesjnNvrjfyttdPe6uC8zGn4rpHau6jm6FWxhNGBjKuec7rrColss2Lb6IUmS065sW//Bols/WHSLhQj7Ry9EOadCFR7+8DN1vVCVVaghPjLq/wSSJAkXX3wxXnzxRWzduhUWiwWrV69GWloaDh06hNatW7s8MjIyKn1faa+n5c2aNcP999+PuXPn4vHHH8fHH38MAOjQoQPWrl3r8r3Vq1cjNjYWTZs21e2/KohwcapTITWHp7tBVLgmpzupyu/Ywynf3lRR5Je2imZbf7RXTzuIZmORYNvqB9tWH1h06weLbrEQYf/ohSjnVKjCnm4/U5eD3Ga3o0KTVyPLcr1OdL9+/XosWbIEV111FZKTk7F+/XqcOXMGHTp0wIsvvoiHH34Y8fHxGDZsGEpLS7Fp0yZcuHABkyZNcm7Dm+g2mUx49NFHMXz4cLRt2xYXLlzAsmXL0KFDBwDA3//+d8yYMQMPPfQQJk6ciL1792Lq1KmYNGmSbqExnkS3kcNw6ia6NYXUCn3zdCuVzcNYdNdpu3p59USzsUiwbfWDbasPLLr1IxRFdzAUUjPy/Zxe1KdmYGpOUB6RK1euxLXXXou0tDRIkjpllCfuv/9+SJKEGTNm+OW363KQa73ctK367ZTi4uKwcuVKXH311Wjbti2ef/55vP766xg+fDjuuecefPLJJ/jss8/QpUsXDBo0CJ9//nmNPN02mw0PPvggOnTogGHDhqFt27bOaceaNGmCX3/9FRs2bEC3bt1w//334+6778bzzz+vy3/VtlXbXiNfnGoruitsduSX0NRrVEjNt5xuf4huUS78erXXU36Vv0W3KDYWCT33W6jDttUH9/5MBNuKciy439cYua0KvvQPVa3Dnm4xEWUgK1QJSk93YWEhunXrhrvuustZbdsTP/74I9atW4e0tDS//XZdLsbuottez6K7Q4cOWLBggdfP//a3v+Fvf/ubx89atGgBWZaxf/9+AGSHhIQENGzYEGfPnoUkSc7pxLwxaNAgbNiwwevny5cvr/5PuKHtOJQ2Kih2fPrpp9GxY0eXZUaktqI7p7jc+To+MtxnT7cUQeHlZrkcqCgDwix1aqtotuXw8tCFbasfbFt9YE+3foSSp9tkMsFms7HoFhRRzqlQJShF9/Dhw53zLnvjxIkTeOihh7Bw4UKMGDHCb79dlzDlUlvlKRtEO4G0Fba1z0Zss2hTK3jyCmiXe0MpohZnDUOY2eRzTrciugEAZQVAWKLPbRVtOiu92qvnlDii2VgkRJnKSETYtvrAU4bpRyhNGcaiW2xE0wyhhnGvGDpit9sxduxYTJ48GZ06dfLpO6WlpcjLy3M+8vPzPa5XkwuVLMuo0AjtSuHlEC8/w1sYlh6d09dff42YmBiPD1/2qycRa+ROtLYX0xyliFq0hXIWfPR0R0REoFx2VKcvL66XtgYK9nQzWti2+sG21Qf2dOtHKHm6lRlpOKdbTETTDKFGUHq6q+Pll19GWFgYHn74YZ+/M23aNLz44ovVrleTg/xMfimy8kqQ0TAasdZwDznd4nm667Nzuu666ypNYaagTAtWFSLbtibtzSsh0R0fGQ6U5gM28nxXl9NtDTOjGBEIRxGLboNtV+9thzpsW/1g2+oDi279cG+TkduqUFfRzZ5uMRHlnApVQk50b968GW+99Ra2bNlSowPxmWeecanSfeLECWcesJaaHOS5jlzb/JIKj6K7vnO6/UF9iu7Y2FjExsbW+vui2hao3XEWZ9Xkc4dHA5aoKr9nDTehGBbEoQgor1kxNVFty6KbAdi2esK21QcW3foRrLb1ltOtLBMVEfaPXohyToUqIRd7sWrVKpw+fRrp6ekICwtDWFgYjhw5gscffxwtWrTw+r2IiAjExcU5H97EnvYgryo8xy7LKHGI7LIKO2x2GWWOUPMw50VPvBNIpDCs2to2UGFX2rbVpGBfXjFVLo+LDPM5nxsAIi1mFMsR9IY93Ybart7bDnXYtvrBttWHYBWGRsDTgLf7cqPBotvY+0cvRDmnQpWQ83SPHTsWQ4YMcVk2dOhQjB07FnfeeWedt689yE+ePIlGjRrBYrFUOuiLyypgLy+l18U25BXIkCvKYDaZYIIEucKG0qI82CpIMJWVlaGsjMKC7XY7SkpK6txWPSgtpf8kyzJKSkqc9igrKzNcm202GwCgvLzcaVul3Z6QZRllZWU4c+YMTCYTLBbfq3n7g9qLbk14eWE2LaxuujCo4eUAgPLCWrVVlLwivdqr7QB5yjBx0HO/hTpsW31wF4Yi2FaUY8FT3+u+3Gj40j94WodFt9jwfYGxCUrRXVBQgAMHDjjfZ2ZmYtu2bUhMTER6ejqSkly9fOHh4UhNTUW7du3q/NvKBTk2Nhbh4eE4efKk5zaWVjgLXEkSUHguHBeKymENM8EOwGorwOlThci7QHMq5+Xl4dw58lIWFhYiMzOzzm3Vg6ysLAAkaDMzM1HhGDQ4ceKE4Ypa5OXlAQBycnKQnU1itKSkpFrbRkVFIT09vd7/j/bCqQwY+NIGJafbJby8miJqAGC1mFEMx8BCDT3dnirDG23/a9GrvXraQTQbiwTbVj/YtvqgtaEothXlWNC2Sel73ZcbDV9s62kdLqQmNqLNyhNqBKXo3rRpEy677DLneyUXe/z48fj88891/W3thSs9PR0VFRUuF2mF1xbuxW87TzvfD2jVEGsOnsXInk1x4HQ+3jxzPwCgQSatExsbi4SEBABAXFwcMjIydP0ftUXr/c3IyHCe7Onp6VWG7weCBg0aACDbNmxInt/o6OgqbWs2mxEWFhaQi5j7lDBADXO6I8OBIhrEQbQvnm6TX8LLjezBUNCrvXqOOotmY5Fgb4F+sG31wd0bK4JtRTkWRPZ0V9U/eFqHC6mJjad9GoqDD0YlKEX34MGDa3TBOHz4sN9+W3uyS5KE8PBwj5W0Vx/Ow4l8VYz/tPMMSivsaNowDttOFsJacAwAEA66AJpMJoSF0e4KCwuD1Wr1W5v9SWRkJACyg9VqddrDarUars3KflGEtPLaaO1UqHNOtzUMyK2B6A7XeLrLahdeLspoK+d0M1rYtvrBttUfUWwryrEguujmnO7QQZRzKlTh4Q8/48tBXlxmw75smue7a9N4AECpo6hau9Q4RJjV70rWOOd2RTiBQqGQWqCotegu0Xi6Cx2i24ec7kiLNqebC6kZabt6bzvUYdvqB9tWP0SzrSjtZdEtFkY+lvRGlHMqVGHR7Wd8Och3n8qDzS6jUWwELmqp5pdLEtA2JQZJUr5zmckaA8D44VcK7qLbyOEtooXh+Ed01yCnO0xbvZynDDPSdvXedqjDttUPtq1+iGZbUdrrqZ6K+3KjUVfRzTndYqLdf0Y+p0KV0DsidcaXgzzzLIXqtkuJRYukaOfy9MQoRFnCkGY/5VymbMXonZICe7r1o7ai22We7hrkdEdaTJpCarUT3aJc+PVqr55516LZWCQ4X14/2Lb6IZptRWmvyJ7umlYv55xusRHtvjbUYNHtZ3zpOE5coFDdpg0i0aJhlHN5uxSa+zu14oRzmWSnfFxRTiBt24zeZtEuTp5G22uS0x0fGaaGl/sguiPCzCgG5bf/uuUQXvpld43bKppt2dPNAGxbPWHb6odothWlvSKLbg4vDy1EOadCFRbdfsb9wuWJ4xfIa9i0QSQyGqqe7vapJLpTBBbd3qYtMWKbRQvD8dTxVxc+ZbfLyHdOGRZWo5xua7gZxTJ5us/n5ODr9Ud8bqsoU8Eo8JRhjBa2rX6wbfVDNNuK0t5QmjKMRbfYsOg2Nsa9YgiKLwf5ifMFuM28FB3MJ5ASa0VkOIXztEulommNyjSiWybRLaIwNPpJL9rFqTbh5YVlFbA7+s44qRiwkwD3LbxcrV4eKZWhpNzuc0csStiggl7t1TMEXDQbiwSH7usH21Y/RLOtKO0V3dNdmynDgiGn28j7Ry9EOadCFRbdfsaXgzzj3HJMD/8EVyy7HiaThMvaN0KcNQx9MhoAAJLKjjnXNcliebrdRbeRC1p46nCM2E6F2ohuJZ/bEmaCtewCLbTEAOGR1f5etMWMJMfc8JEoBQCU2XzriEUd0ODwcgZg2+oJ21Y/RLOtKO0VXXSHani5ke/n9MJT9IKRj9NQIyjn6Q4k1R3kNruMqKJTquXLCvHu33qitMIOa7gZkGU0KDnuXF+08HJ30e1puVEQpcNXqE1OtzpHt7ZyefVebmXbE67oDPykiu6Scjsiwsw+t1U027LoZgC2rZ6wbfVDNNuK0t5QEt1cSE1sRDmnQpXQGwbSmeoO8uy8Epyzq3ncOLkN0t7fYP3jFUCWgcKziLAVOj+WHOHAopxALLr1ozaebmW6MJciaj7kcyuYLFToL1IqAwCUltuqWr1SW0WzLYtuBmDb6gnbVj9Es60o7fU04O2+3Giwp9vY+0cvRDmnQhX2dPuZ6g7y4xeKESGVqwsyVwBr3wXKCoCWgwHJdRxEZE+30Tsn0S5OtRLdxbWbo9tJOInuaIk83aUVNQsvFyWvSK/26pl3LZqNRYLz5fWDbasfWtu6LzMiohwLInu6azplGItusRHtvjbUYE+3n6kuN/hEThGsKFMXKIIbAM4dAC4cBgAcsqfSduRy53ZFOIFEqvKprV4uWk63Ytvq2us6R/c5WhiV5PuPOjzdUQ5Pd4mPnm5RqtIqcPVyRgvbVj/Ytvqh2NHofa+CKMeCSPc1CrWtXh5MhdSMvH/0QrRZeUKN0DsidaZaT/f5Ylih8XQrghsAzh8Cco4AAA7ITWg7st25XRFOIJFGhEUbEaxdeLkjpzsyHCjJoYWRCb7/qMPTHekYKKqpp9voHgwFvdqrpzdaNBuLBEcR6AfbVj+0tnVfZkREORZEuq9R8KV/8LQO53SLjWj3taEGi24/41t4eZnHz3AhE7hAonu/IroldbsijN6J1DmJdnHS7veahpfHR4YBxY7q5ZENfP9RR5Vzq7OQGud0G2G7em871GHb6gfbVj9EFd1GPxZEuq9R4JxuY+8fvfA0kGJkzRBq8J7wM9Wd7Cdy3DzdgNObqPV077c3pe1otivChYRzuvWjNp5ul/Dy4hxaWCPRrXi6a5fTLZptWXQzANtWT9i2+sGiW3+CybYsuoMPEc+pUIJFt5+p3tNdhAglp1syA5CAQU/R+/OHnTndR+VklCEcJkHCrxS8jQgbcaRNtBHB2oWXawqpOUV3gu8/6hDdESgFILOn2yDb1XvboQ7bVj/Ytvqh2NHoA94KIh0LwWjbqkQ353SLCed0G5vQOyJ1pqqD3G6XcSKnWBXdg58GnjkG9LuP3pfmArnHAADH5EYolSwu4eUinEAihWGJ1OG743t4uWae7jqEl5sgIwLlKCnn6uW12a4eedei2VgkOF9eP9i2+iGyp9vox4In2xoZX/oHT+twTrfYiHxfGwqw6PYzVR3kOcXlKLfJ6pRhYVYgIpaETWyacz27OQJnkIBSWDi8XEdEvDi5d/y+errjI2sruqOcLyNRitIK9nQbYbt6bzvUYdvqB9tWP0QW3UY/Fmra9wYaDi83/j7SA5HOqVCERbefqSqsRREsUYrodngRAQCJLdX1YpsBkEh0C+bp9lTsCzBmm0UMw3EPcasufEqdpztMrV5uTfD9B81hgNkCgCqY++rpFmUqGAWeMozRwrbVD7atfvCUYfrhblujtlOhtlOGsegWGxbdxsbYVw0BqeogL3UIlkinpztC/TCxhfNlRWw6rY9wp6dbFGHInm59qbGnW6leHmYDyotoYU083YBzcChSqp2nWwTPgF7t1TMEXDQbiwSH7usH21Y/RPV0i3AsiOzprs2UYaKE0XtClH2kByKdU7Xh3XffRYsWLWC1WtGvXz9s2LChyvVnzJiBdu3aITIyEs2aNcNjjz2GkpKSemptZVh0+5mqDvIym7vo1ni6G2Q4X9rimwEAimULTIJ5ur3ldBtxVNhTh2PEdmqpeXg55XQnmByCWzIBEXE1+1FNBfOa5nSLdtxyeDkDsG31hG2rH6KKbhGOBZFFd6iGlxv9fk4PPEUvGP1Y9ZU5c+Zg0qRJmDp1KrZs2YJu3bph6NChOH36tMf1Z82ahaeffhpTp07Fnj178Omnn2LOnDl49tln67nlKqF3ROqML55u6/+zd97RcVRnH35mq3q3JfdesbFNM6ZDTA81hB5aIKElHwECIdRACIEkBEIPoQRSgNADxBSD6baxDe69N0m2ZXVtn++PKVu0KivtrOZK9znHx9rZ2aurd9r+7tuSerqj4eVq4XAAfKp44eVtiW47zlmkB75BYnh5e/ONRFQa/ZrozlMbtY1ZRZDqgyhOdMucbjuMa/XYfR1pW+uQtrUOKbqto6+IbllITWxEuqZS5cEHH+SKK67g0ksvZeLEiTz55JPk5OTw7LPPJt3/q6++4tBDD+X8889n+PDhHHfccZx33nkdesetRIruNNOu6NZDc7MUvXp5XE73iJifhwPQorqFLqRm9we/iDenVB78wRj7e4N12g+ptAszMES3EpB9um0yrtVj93Wkba1D2tY6EhdlY7fZEZHOhVQWvO2A9HTb/xhZgUjXlEFDQwP19fXmP7/f32qfQCDAwoULmTlzprnN4XAwc+ZMvv7666TjHnLIISxcuNAU2Rs2bOC9997jpJNOsuYP6QRSdKeZ9kW37ukmiac7JrzcUTwM0MLLjWECobAQF5DM6baWVER3IEYguwOG6C5O/ZcaOd1d8HSLkldk1XwzIbpFsbFIyHx56xDxvisKInu67X6dierpTrVlmBTdYiPi/XXixIkUFhaa/+67775W++zevZtwOEx5eXnc9vLyciorK5OOe/7553P33Xdz2GGH4Xa7GTVqFEcddZQML+9NtHdDNkSQ2ac7Nqc7uwiGHQYlI3GWjwPAh9vM6V5dWS/MBWRg9wqqIud0d2a1PVZ0u9IkuqWnu+vjJm5L59gi2FgkpG2tw8proq8jsui2+3Umqujuqqe7NxRSs/v3OSsQsSvPihUrqKurM//dcsstaRl3zpw5/O53v+Pxxx9n0aJFvP7667z77rvcc889aRm/K7h67Df3Utor4GCEl3uSeboBLnkH1AieiHaB+NRon+7K2hZ21LYA9r+AHA4HkUjE9g9+EW9OiQ/+9h4qwbD2N7kcCo6utAsz8OQCenh5Jz3dIrWCAetbhlmxACWajUVC2tY6RGtrJRKi2Vak66yvtAyTOd1iI9JClkF+fj4FBe0X+C0rK8PpdFJVVRW3vaqqioqKiqSfuf322/nRj37E5ZdfDsDkyZNpamriJz/5CbfeemuPXMP2vmsISGfCy71qkpxu7UPgcOJ2KigK+GL7dKPy6WrtZFu3q8mayacJUVbbRbw5Ja5Cd8bT7XE5oGWvtrEHPN0ieAasmq+V14JoNhYJGbpvHaI8H0RENNuKdJ2J7OnuSsswKbrFRMQIzs7g8XjYf//9mT17trktEokwe/ZsZsyYkfQzzc3Nrf72nj6/pac7zXSmerlH1YsEJHq6dRRFweN04MeNW9/mcTpo1itRz15ZzeY9TQwrzU3v5NNEsge/HRFBdH+zqYa9TQGO20dbyUspp1tvUed2OqClVtuY4ZxuO9s2FqvDy60W3SLYWCSkba1DNGEoEqLZVqTrTGTRLQup9R1EuqZS5frrr+fiiy/mgAMO4KCDDuKhhx6iqamJSy+9FICLLrqIQYMGmTnhp5xyCg8++CDTpk1j+vTprFu3jttvv51TTjnFFN+ZRoruNNOu6A5HABU3Sfp0J+BxOfCFPHj0YaYMKiCrrIQPF0FYhfveW8WTP9o/3dNPC6nkHfckItycrnxxIXubA8y/dSZled4uFVLrvqc7Gl7uC0nR3ZVxrSgqKJqNRULa1jqsvCb6OqLZVqTrTJTvNQYyp9v+x8gKRIoeSZVzzjmHXbt2cccdd1BZWcnUqVOZNWuWWVxty5YtcZ7t2267DUVRuO2229i+fTv9+vXjlFNO4d577+2pP0GK7nTTvqc7jNcQ3ADurDbH8boc+IIeivRhstwOxvTP40N97FnLK/lq/W4OGVWWzumnhVTyjnsSu4fhhMIR9jRpqQh7GgMpi+6g7un2OGNFd1HqE9E93SU08MOap2G7Bwbt1+5HRLvxWzXfTHi6RbGxSMjQfesQzRsrEqLZVqTrTFRPd1+uXm6n73OZIlmevt3P1VS49tprufbaa5O+N2fOnLjXLpeLO++8kzvvvDMDM+scfe+MtJiOcrrNyuUArrZFt8fp0HK6ab0Suc+gQgDe/HZ7uqadVkRZEbb7KntzTCh3g09brOlKeLnH5QCjkFqXPN1an+4LXLM5vflVeObYDj9id9smIsPLJbFI21qHaMJQJERJ7TIQ6ToTVXSn6unu6ZzXdGD3c8lKRLqm+iJSdKeZ9jymgVAk2qNbcYLT3WofA4/LEV9ILeYCKsnVcsE7W9Qq06RS7KsnsXv18mZ/jOjW8/kTFzTaW8k1wsvdTqV74eWenPjXkVCHHxGpKi3Ez82K6uWxX4Jl9XL7I21rHVZeE32dRNva3a4iXWe90bbt7SNFt5hI0W1v7H3XEJAOPd2K0aO7bS83gNflxI/bbBkWd1M0L6r0zDndSE93emgKRMVtgy9edKfs6TZEd1dahiVW2e8EdrdtIrFzs6J6uczpFgtpW+sQLe9YJER59hqIdJ31RttKT3fvQ6Rrqi8iRXeaaV90h6Oe7nbyuUH3dKseHPow8Tk3uvCy6U1R5nSnh1hPd2NXRLfRos4B+Oq0jd0IL4++7rhqvki5emC96JYtw8RC5stbhwwvtw5RQ6BFuM5EtW17z4f2vgOJkqKQDDt+n8sUdo/g7Ov0vTPSYjqd092Bp7u98PLojTJNk04zojyc7L4iGOvpbvTH53R3ZrXdKKRW4PCBqj9Au1FIzSSvf4cfsbttExFddItgY5GQtrUOKbqtQ5Rnr4FI11lvtK30dPc+RLqm+iJSdKeZ9k5yLae7k6Jb79PdXni5KJ5uu17wdr85NXc3vFz3dBc7mrQN7tw2e8O3S6JnO+Tv8CN2t20isXNLZ/igDC8XE2lb65Dh5dbRG0Og7UJvtG2yfWROt9iIdE31RaToTjMd53QbPbo7F16e3NOth4/Y9KYoysPJ7jenpthCal0Q3YanuwhddHfFyw2tPN1qqKXDj4gUNghie7pFsbFIyNB965CebusQZcHbQKTrTFTb9uWWYXY/RlZg9++1fR0putNMezdkfzAc9XR3kNPt1cPLk+V0K2ZOd5omnWYSRbdd82psn9OdJk93rsM453La3LddEj/Xyz3dooluUWwsErG2Tdwm6R5SdFuHqMJQhHtYb7Rte55umdMtJjKn2970vTPSYtpre6HldKfg6cbTQfVye6pu2TIsPcR6utvK6W7voWK0lPMqunjvSmg5tG4ZFmzpsKCASK1gIH5u6VwsMsaIDaW1omVYusfu68i2VtZh5TXR10m0rd3tKtJzojfaNtk+MqdbbERayOqL2PuuISBpy+lOUkitVXVJm94TRVkRtvvNKZmnO5UFjWBY+5uyHPqX23b6wrdLXjmqw02DqoWZK6gQDrb7EZHCBkFsT7f0GKYfaVvrkLa1DlGevQYipciIattUq5dL0S02do/g7OvII5FmOmoZ1tmcbq+R0x0zrqgtw+x647O76G4K6DnxRDh3z2Ow+KUuhZdnOXTx7uyipzunBOUnn3CO+rvotg7yuu1u20Sk6JbEIm1rHdK21iHKs9dApOdEb7StLKTW+xDpmuqLuHp6Ar2NjgqpdTan2+N04MNteroj4bBwLcNkIbXu0ezXxPIEZQun+t6Gj77pUiE1L930dANUTKbStZNIWMGhqB3mddvdtonEzk206uWyCnT6kba1Dmlb6xDl2Wsg0nOiN9q2t+d02/0YWYFI0SN9EenpTjMdh5cbnu7sVu/HUprnxR9TSE2NhIX1dNs1tMXuYTiGp7tAadY2NNek5uk2RLfp6fZ0az5etxM/unAPds7TLcqNX3q6JbFI21qHtK11iOyNtfucRbVtX65ebqfvc5kiWZ6+3c/VvoT0dKeZjluGGTnd7Yf6XnLocAYUeFH+bIwbc1NUjIsqTZNOM6I8nOy+ym7kdOfg0zaE/SiKdt6kEl7uoZuF1HSy3E78PjfZBHq1p1uKbom0rXVI21qHKM9eA5GeE73Rtsn2kTndYiPSNdUX6XvLQBbTnsfUHwpHPd3u9j3dBVlufnjgUBSX5p2M9XSbN3+b3hQTq3za9YK3+4qgUb081xDdoBUxo3MVVE1PtyG6uxNeTrSNHdBhTrdIVWkhfm7pjHqwspK/rLBtHdK21iFKdwsRSbSt3c9ZkZ4TvdG27e0jRbeYSNFtb+x91xCQDj3ddM7TbaA4xRPdouQ+2T0E2vR0K1GvsiG6U/F0u5VuFlLT8bqd+FVduPdiT7eVOd3ptIHMjbUOaVvrEOX5ICKi2Vak50RvtK30dPc+RLqm+iJSdKeZjnK6o6K7fU+3gcOthxPHFFIbtH0Wd7mepzy4LQ0zTj8ypzs9tOfpTqWQmhle3t2c7lhPdydzukXI1YPMhZdbIbplmG76kba1DtHCdEVCNNvafeE7FlFtm2rLsN5USM1O3+cyRezxs/s11ReROd1ppkNPt9MopJaip1uNoDbuAqB077dc4lrBlLoa4Ifdn3SaEeXhZPcVQcPTnUtU4CpqfHh553K6jXOue6I7K7aQWi/2dEvRLZG2tQ5Rng8iIpptRXpO9EbbSk9370Oka6ov0veWgSymrZM8FI4QjqgxLcM65+mOy+n21QEQ8JYAMDnwLTTXpGPaaUWUh5Pdb06Nhqe7i+HlhqfbnSZPd1YKOd12t20iMrxcEou0rXWIFqYrEqLZVqTnRG+0bbJ9ZE632Ih0TfVFpOhOM22d5P6QUdQqRU+3KbojqC2a6G4snsjKyBBchGHVu+mYdlqRojs9GJ7uYlfA3Kag2bQztjXOuXSJ7ric7qCv3X1FChsE6emWxCNtax2iPB9ERDTbipSGJKpt+3LLMLsfIyuw+/favk6vFN2fffYZp5xyCgMHDkRRFN58803zvWAwyM0338zkyZPJzc1l4MCBXHTRRezYsSMtv7utk9wI9c1KMadbya8AIBJoQfXVAxDMLufd8MHaDive7OaM048oK8J2vjlFIirNep/uEnfQ3G6El6fi6XZZ4ununOi2o22TIUW3JBZpW+sQTbyIhGi2Fek50Rtt256nuzfkdNv9GFmBaA6PvkavFN1NTU1MmTKFxx57rNV7zc3NLFq0iNtvv51Fixbx+uuvs3r1ak499dS0/O62CjgYXscsJbWeyY5BUwFQA42ofk10B3L7815kurbDhjm2CzFPbBlm12IWsQ8XuxXeaAlGw1qLkni6O9UyzPB0q6lVzG8Lr9sRk9PdvugWqRUMJBfdVrQMS6cNZFsr65C2tQ7RWi+JhGi2Fek50Rttm2yf3pTTbfdjZAV2b4Xb1+mVhdROPPFETjzxxKTvFRYW8uGHH8Zte/TRRznooIPYsmULQ4cOTfo5v9+P3x/NrW1oaEi6X9vh5ZpIylZSzOkeOE0bN9CCGtbFbHY/NgQGstE5ghHhjbBmFkw9v1PjZQJRVoTtvMrepIeWKwoUOKIC15FCeLnRp9ulpqdPd5bLib+XerpBm5+qqjKnWyJtayGiREKJiGi2Fek50RttKwup9T7ai16Q9DzySAB1dXUoikJRUVGb+9x3330UFhaa/yZOnJh0vw7Dy5UUc7oLBmjjAhHjHqiHCa9xjdFe16cnND5dSNHdfZqNImoeFzmxLcNSCS8Pafu6VP2c6254eRdyukXI1TOw4ryV4eViIm1rHaI8H0RENNuKFAorqm272jJMim4xEfG7V1+iz4tun8/HzTffzHnnnUdBQUGb+91yyy3U1dWZ/1asWJF0v44KqWWZhdQ66ek2LyDtH4DDoa+4qvrhU+2VeyPKirCdRbfh6c7xOMlSY/t0azZNxdPtTJPo9vbinG6w5ryVnm4xkba1DlGeDyIimm1Fek70Rtu25xWVoltMRLqm+iK9Mry8swSDQc4++2xUVeWJJ55od1+v14vXG/VO19fXJ92vo/ByrxLQ3NburE7N0cwjUsG4BToUfRv677Cp6LZ77lN7q7w9jVFELdfrwhuK7dOdQnh5KL2iO75Pd++qXg7S0y2JIm1rHaJ5DEVCNNuK5JUT1bZdrV7eGwqp2eX7XCaJPX6ifPfqS/RZ0W0I7s2bN/Pxxx+36+VOhTZFd1BvGWYWteqc6DYfSsR6urX/w4bojoRbf7AHEeXhZOcVwSZ/1NPtCXRRdBue7khqKQ1t4XU7aFR7v6dbim6JtK11iPJ8EBHRbCvSc6I32lbmdPc+RLqm+iJ9UnQbgnvt2rV88sknlJaWpm3stm7I/rDRp7uLotudg+oIA6EYT7cML+8Odr45mZ5ujwt3uNncnkp4udEyzBHRz7m0FFJLPafbbrZtCym6JQbSttYhmngRCdFsK9JzojfaVoaX9z5Euqb6Ir1SdDc2NrJu3Trz9caNG/nuu+8oKSlhwIABnHXWWSxatIh33nmHcDhMZWUlACUlJXg83QvBbataoObpVvGYOd0piu7c/qgjx8Oq92Jyuu0ZXp7YMsyuF7ydw3AMT3eeB5zh2EJqqbcMc5rVy7vn6c72ODud0y1SKxgDK1rCZLplmF3OX9FJtK20a/oQrfWSSIhmW5GeE73Rtu3tI0W3mEjRbW96pehesGABRx99tPn6+uuvB+Diiy/mrrvu4u233wZg6tSpcZ/75JNPOOqoo7r1u9vL6fYaghtSz+mORKI53brojnq67R1ebteHkwg53cWuYNx2RT8LOrOgYYhuRyQ9Od35Wa6Uc7pFuvGLXkjN7vYVCWlb65C2tQ7RbCvSc6I32ra98HKZ0y0mdnYmSXqp6D7qqKPaXaWzcgWvverluTGtn1L2dMeuRBqiVpBCana94O38wDeqlxe5tNDwkOpAVRzGEe/QtpGISkjvMaeE9f7yru6Lbl8Xcrrtfh4YiB5ebnf7ioS0rXVI21qHaLbtTLEvuyCqbTvbMsxA5nSLjZ2/10pky7C0016f7nGOrdqLomGdzq9NfgHp4dtGyzCbrUiK8nCy883J6NNtiO4msmggB2N6HdnWKKIGoKTN0+3uEznd0tMtkba1Dmlb6xDNtiI9J3qjbRP/JpDh5aIj0jXVF+mVnu6epD1P9xRlvfZi4LROj5e80IV+o5Se7m5h55uT4ekucGhe6iayCKieTnu6g7GiO2wUUuteTnd+lsvM6VZDPtqzlEgeDAPp6ZYYSNtah7StdYhmW5EiokS1bWdbhhlI0S02dv5eK5GiO+20l9M92bFBe9EF0R3XR9HYpto7p9vuK8J2vjkZnu4Cpya6m9UsmsjuvKc7lMzT3b3q5XneaE63GmzplOi2o23bQopuiYG0rXVI21qHaLYV6TnRG23bnujuDTnddj9GViCiw6MvIcPL00xbBRwCoQj7Khu1F4P26/R47VWXtKunO7F6uV2LWcTa1m6FNwxPd56iie5GsqhTc1p5utuabzCsnSsuh4IS0nO6uxlenutx4dc93ZEOwstFqkprIHr1crvbVySkba1D2tY6RLOtSM+J3mjbZAK7N+V02/0YWUGySAUpuu1D3zsjLaatlSVHyx6GOHZpLwZM6fR47a1WmqI7Yk9Pt91XG+28ImhUL89TNHHbrGZRH5PT3VEUgeHp9rgcEDba1HVPdDscCk5PtvYi2NLuviJ5MAykp1tiIG1rHdK21iGabUV6TvRG2ybzdPcm0W33Y2QFyY5pX7SDXZGiO820dbGX1q0AYE/WUMgq7PR4yXKejJuiWUjNZp5uUR5Odn7gN+ue7my94n0TWdSrOTg6G16ui3JNdKfH0w3g8uRoPxje8zaws23bQhZSkxhI21qHtK11iGZbkZ4TvdG2spBa76O9lAFJzyOPRJpp62Ivb9REd3X+xJTGS15ITXsvGl5uT0+33R9Odn7gh/V2X96w5lFuJot68lDo3IJGIKR93u1QIE2F1ABcXs3TrfTilmFSdEukba1D2tY6RLOtnaPNEhHVtp1pGSZFd+8h2THti3awK1J0p5m2LvYBTSsB2FOwT0rjJWt079RbhkVUQ3Tb6+aYuNJm11W2ZA8lu8zV6LHtjmii26donu5OF1LTq5fnOGOiILpZSA3A7dU83Y6wv93zTqQvUwYyvFxiIG1rHdK21iGabUVanBXVtl2tXt4bCqnZ5ftcJkl2/Ox+rvYl+t4ZaTFthpf7tR7d9YVjUxovuadb2xYyDp/M6e4SdvZ0h/RCaO5QEwA+JUfL6dbf72zLsBxXzIPT1X1PtzdL93QTieaKJ8HOtm0LKbolBtK21iFtax2i2Vak50RvtK3M6e59yJxueyNFd5ppM4xHDwF3uLNSGi/5jTPR022vFUlRwrDs/MA3Pd3hZgD8jmzq1dzOe7r1Qmq5jpgFmTTkdHuyc2Mm2XaIuZ1t2xZSdEsMpG2tQ9rWOkSzrUjPid5o2/Y83VJ0i4kU3fZGiu4001arAkUXxm6XM6XxOuXptpnoTgxvsesFnyx03y5zDemeapcuugOObLapZaanOxzSvMxthU8Z4eXZLl10K05wpHbuJSM7Oydmkm2LbpFawRjIlmESA2lb65C2tQ7RbCvSc6I32jZZKLIU3WIjc7rtjb3vGgLS5sVuiG5nasInmTB0GCtZpqfbnuHldu/TnXxBwx5zNQqpuUJaTrffkc236mjTWx1u3AN07OnOMTzdafByA+Rnu/Gpem54L/V0y0JqEmlb65C2tQ7RbCvSc6I32jaZQDPCy2VOt5jInG570/fOSItp8+amb3e5XCmN154wDNk8vNzunm47F3ExwstdYS2nO+jMQcVBMLsUgEij1vO9Q9Ht1B+m3ezRbZDvdeFHF93Bzoluu9m2LWR4ucRA2tY6pG2tQzTbdqbYl10Q1badqV4uc7p7DzK83N5I0Z1m2rrYVfQc3XSIbmN10gg2loXUuoSdV9mN8HJnMCq6AYJZZQBEfHVA2/M1CqllKWn2dGe58aGPJT3dPTJmJsbu60jbWoe0rXWIZluRnhO90bbJPN0yvFxspOi2N1J0p5m2Pd3aBeBJR063MzG83F43Rym6u4/h6XaGtJzuoEsT3WGXVj08Eml/vq083WkT3S78KYSXi+DBMJCebomBtK11SNtah2i2FSkiSlTbtie+ZCG13ofM6bY3UnSnmY5yunO8qYmfZOLF9HSrRiE1e3q67b4ibGfRbeR0O4Ka6A7roltVjQUXbb+OPN1e6enuNFJ0Swykba1D2tY6RLOtSM8JkW2buC3xtRTdvQfp6bY3UnSnmWQFHPyhsCm6+xWk1jIsWQVKp77NDC+3WU63MWe7F1KLta3dCm8Yotmp9+k2Rbf+viG625qvX/d0ZztC2oY0ie68TuZ0i1SV1kBWL5cYSNtah7StdYhmW5GeE6LaNpmgbm8fWUhNbGQhNXvT985Ii0m2wlZZ50PR5VJhjjel8ZKuBBstw4zwcpnT3SXsHAJteLoVPac7kiC6wx14uo2WYVmONBdSy3JJT3cPj5mJsfs60rbWIW1rHaLZVqTnhKi2leHlfQvp6bY3UnSnmWQX+/baFhzoN+oUeyUneyg5FW2MaHi5vVYkRXk42fmBHzREd1jrx6249MUaRV/F7Ci8PKTt4FXS6+kuyHKnlNNtR9u2hSykJjGQtrUOaVvrEM22Ij0nRLVte7m97e0jRbeYtFccT9LzyCORZpKK7r0tOAwfpZKayZMVGnHonu6IcfhkTneXsPMDX/N0qyj6sXXqnurE8PK2Pd3a56I53alFWLRFXpaLFrQUiZC/sc39RCqQYyBFt8RA2tY6pG2tQzTb2jnaLBFRbZuK6FYURYpuwZGF1OyNFN1pJtnFvqPW12XRHZufEc15MkS3kdNtr5tjoqfbrqtsyRc0en6uqqoSjqi4iVmpdLv197TXHRdSS/R0u9Myt/wsF/Xo7csa9wJa+kSDLxi3n0hfpgxkeLnEQNrWOqRtrUM024q0OCuqbVMJL1cUJWlOsGjY6ftcppE53fam752RFpNcdLeYOd2kePInDy83Cqnph0/mdHcJu3q6jXZhLkLmNofp6dZt25GnO5RQvdyVHk+32+mgSckFINi0l71NAY764yec9/TcuP3satv2kKJbYiBtax3SttYhmm1Fek6IattURbf0dIuNzOm2N66enkAigUCA6urqVqtsQ4cO7aEZpUZS0V0XzenuTni5ObbRp9um1ctFCcOy6wPfKKIW6+l2twov76BPt15IzYPugU5TTjeAz5kPKoSaa9m4uwlfMMLaqvhQc7vatj2k6JYYSNtah7StdYhmW5GeE6LaVoruvoUU3fbGNqJ77dq1XHbZZXz11Vdx21VVRVGUuPwEO5MsrCVdOd3xnu5ITE63vUR3YniLXS/4ZKH7dphr1NMdPeed7uQ53W2FTxmebo+Z052e8HKAkDsfAhBprmV3ox/QWpQFQhE8LkfcvERoBWMgW4ZJDKRtrUPa1jpEs61IzwlRbZtKyzCHwyFFt+DInG57YxvRfckll+ByuXjnnXcYMGCAsCdJ4sWuqqpWvdzZPdHdfk63vRYkpKe7e4R0L7UpuhUnHpdWsd54DoaNwIkOwss9iuHpTk94OUDQUwABwFfLnsaAub3JH8Kje+Ttatv2kJ5uiYG0rXVI21qHaLYV6Tkhqm276unuDTnddj9GVtAbjl9vxjai+7vvvmPhwoWMHz++p6fSLRIfHDVNAfyhCIrTuABSuwnErkyanm5ddIdt6ulOvOjtuiKcrIiLHeYaMsPLo0XQDA+ysdDSUZ/uoBlent5CagARbyE0guKvNz3dAI3+EMW54otuWb1cIm1rHdK21iGabUV6Tohq21Srl/emPt12+D6XaUSJNO2r2OaMnDhxIrt37+7paXSbxAfHjlqtl7FD6Z6nO9nYqiyk1i3s+sA3crqzHPpiisON22k8CLVNnS2k5jZyutNUSA3AlVOkTctfx54Y0d3gixZ+S7agYQfbtocU3RIDaVvrkLa1DtFsmyySz65zFtW2smVY30KU7999FduI7vvvv5+bbrqJOXPmsGfPHurr6+P+iULixb69thmg2znd0NpzHPV02+vmKMpFb1fRbXqpDdHtdMV4ujU67tOti27V8HSnr5BaTmEpAK5AA7ubYsLLA61FtwhfpgxkeLnEQNrWOqRtrUM029r1GZwMUW0rC6n1LUQ7T/satgkvnzlzJgDf+9734raLVkittej2ofmkuy+6DRs4jYtK5nR3C7s+8KOebv24OqLh5al6us22Y2kU3QVFZQB4ww3sbogJL2/D020n27aHFN0SA2lb65C2tQ7RbJtM4Nl1zqLaVoruvoUo37/7KrYR3Z988klPTyEtJN6Q43p0Q8qiOzYnJdHTrQrSMsyueTV2zekO6gnbXtPT7cZrhJfr+3Te053+lmHFpf0B8KgB6hujrcIa/FJ0Z2LMTIzd15G2tQ5pW+sQzbadEYZ2oTfaNtnflKzquWjY6ftcppE53fbGNqL7yCOP7OkppIXEAg51LcH40mkpXgDJw8t1UavaM6dblIveri3DDE+31xHR4skdMeHlqhK3T0ctw0xPtyt9ortfaSkRVcGhqAQba4A8IN7TLVIrGAPZMkxiIG1rHdK21iGabTvT1sou9EbbypZhvQ/RFof6GrYR3QC1tbU888wzrFy5EoB99tmHyy67jMLCwh6eWedJvNgjqoqDmBXDdISXO50oSmx4ub1WJEW56O3qjQ3pdstyhDXR7YwppIZxXmn7tjVfvyG6LfB0DyjKoYFsCmlG9dVhim5/0NzHrrZtD+nplhhI21qHtK11iGZb6em2Dhlebv9jZAWiRJr2VWxzNBYsWMCoUaP485//TE1NDTU1NTz44IOMGjWKRYsW9fT0Ok3rPt0xRdQgLaJbURQcikLELKRmL0+3KA8nuwrDkB5e7lGi1ctTLqTWSnSnr3p5aZ6XenIBKKDZ3N5bcrpl9XKJtK11SNtah2i2TZxv7Da70Rttm+xvkqJbbEQ7T/satvF0/+IXv+DUU0/l6aefxuXSphUKhbj88su57rrr+Oyzz3p4hp2jtehW057TrYluiBjh5Tb1dNt9pS2ZMLTDXI0+3cmql7cqpNbGGP6QHhVhiu709el2OhSaHfmg7qJQaTITzZPldEciEdsvvhhI0S0xkLa1Dmlb6xDNtlJ0W0dXRbfM6RYbUdI7+yq2Ed0LFiyIE9wALpeLm266iQMOOKAHZ5YarcPLSVt4eWIIkNkyzGY3RxE93XaaazSnO6Z6ubMNTzfJV6OtDC8HCLjyIRjv6W6ShdQyMmYmxu7rSNtah7StdYhmWxlebh3JbNvRPtLTLT6inad9DdssAxUUFLBly5ZW27du3Up+fn4PzKhrJM/pTk94edyNEZnT3V3sKgxDeuVxr6KLbmdMeHmip7uN1AIjvNwZ0UW3K33h5QBhTwEABUqTua1Riu6MjJmJsfs60rbWIW1rHaLZVopu6+jMfKXo7n2Idp72NWwjus855xx+/OMf8/LLL7N161a2bt3KSy+9xOWXX855553X09PrNK3aemUkp9teotv42+0ehhUbemSnm7QRXu42c7pdeNsQ3Y4OPN2OSPrDywHI0kU3zejF9GmQ1cszMmYmxu7rSNtah7StdYhmW1m93Do6M9/2qpeLjJ2+z2Ua0dIg+hq2uWv88Y9/5Mwzz+Siiy5i+PDhDB8+nEsuuYSzzjqL+++/v6en12la5XSjolgQXu5QYj3d9iykZveLvq0FjZ7GqF5uerodLrN6uXEmhc3w8tbHPhSORFuKRQLaxjQWUgNw5hQBUKA0M6g4G5Ce7kyNmYmx+zrSttYhbWsdotlWerqto7uebhDX2y3Kdw4rEO08TZXHHnuM4cOHk5WVxfTp05k/f367+9fW1nLNNdcwYMAAvF4vY8eO5b333svQbFtjm5xuj8fDww8/zH333cf69esBGDVqFDk5OT08s9RoFV4e6Z6nu+1CajE53TbzdCde9HZdEU62oGGHuRrVy91JwssTn4FKkmMfCMd4DSLW5HR78koAKKCR4aW5bK1pobGld7QMk4XUJNK21iFtax2i2TZxvrHb7Ibotm1PdCcrpAba9yKn02n1VNOOnb7PZZreXEjt5Zdf5vrrr+fJJ59k+vTpPPTQQxx//PGsXr2a/v37t9o/EAhw7LHH0r9/f1599VUGDRrE5s2bKSoqyvzkdWwjug1ycnKYPHlyT0+jy3Sc053aBdBmTrcS7dlMxJ6ebrtf9G3ZtqcxC6klaxnWSnS3Pvb+YIzXIKx7ul3pFd3ZBbroVpoZVppD1bqtvNxwL3x9C8y4Ok5028m27SFFt8RA2tY6pG2tQzTbStFtHV0V3dLTLTaifP/uCg8++CBXXHEFl156KQBPPvkk7777Ls8++yy/+tWvWu3/7LPPUlNTw1dffYXbraVYDh8+PJNTbkWPiu4zzzyT559/noKCAs4888x293399dczNKvu0Tq8PNbTraRNdDscsZ5ue90YRbno7Sq6g5FET7crWr08UXQnWXAxK5c7FBQzvDy9ojuvqAzQcrqHl+bicSyjmHpY+36c6I5EIsI8AGV4ucRA2tY6pG2tQzTbyvBy65Dh5fY/RlYg2nkK0NDQQH19vfna6/Xi9canRAYCARYuXMgtt9xibnM4HMycOZOvv/466bhvv/02M2bM4JprruGtt96iX79+nH/++dx88809FsHRo6K7sLDQPCEKCgqEODk6Inmfbv2BkmJoeew40F4hNenp7gp2Fd1hfS6m6I7xdCcu2iQNL9dFt8flgLA14eUFRaXa/0oTw0tzCSm1AKi+ehTiz39zrjawbXtI0S0xkLa1Dmlb6xDNtlJ0W4cU3fY/RlYgWkQGwMSJE+Ne33nnndx1111x23bv3k04HKa8vDxue3l5OatWrUo67oYNG/j444+54IILeO+991i3bh1XX301wWCQO++8M61/Q2fpUdH93HPPmT8///zzPTeRNJK8T7dR9Sr1k7/tnG5sW7088aK3a15NsgUNO8w1aU63M/m8khVS84e0bV6XA0J+fYz0im5XTjEAxY5mBg4ppE7ZC4DaUidFt8VjZmLsvo60rXVI21qHaLaVots6uiq6k33nFA2Z0y3OeQqwYsUKBg0aZL5O9HJ3lUgkQv/+/fnrX/+K0+lk//33Z/v27fzhD3/oMdFtmzPymGOOoba2ttX2+vp6jjnmmMxPqIskXuxxOd3d9HQnrkZGVP09m+V0i9gyzE5zNVuGGRES7Xi6HUkWXIzwcq/LCWZOd3qrl5NVBMDQ7CD98ryUG55uf502L4FawRjIlmESA2lb65C2tQ7RbCvSc0J023alZZj0dIuHaItDAPn5+RQUFJj/konusrIynE4nVVVVcdurqqqoqKhIOu6AAQMYO3ZsXCj5hAkTqKysJBAIpPeP6CS2uWvMmTMnqRF8Ph+ff/55D8yoa7QOLwdHN8LLY4kT3cS2DLPXaqQoF71dw8ujojua0+10KDgdChA/v6SF1OLCy42c7nT36S7Ufr9f82xXOOr01w3a/wJ5MAykp1tiIG1rHdK21iGabUV6TohuWxle3jcQ7TztLB6Ph/3335/Zs2eb2yKRCLNnz2bGjBlJP3PooYeybt26uPvLmjVrGDBgAB5PeqM/O0uPVy9fsmSJ+fOKFSuorKw0X4fDYWbNmhUXdmB3klUvV5Sue7qNsVRVjQuBjm8ZZi9PtygXvV1Fdzhs5HTrfa8dmmD2OB1Jcro7CC/3WVNIzRDdREIQbKYfWni5I+yHkL9VigHYw7btYUUulJX5VSLmbomCtK11SNtah2i2Fek5Ibpt2xPdsnp570GU9M6ucP3113PxxRdzwAEHcNBBB/HQQw/R1NRkVjO/6KKLGDRoEPfddx8AV111FY8++ij/93//x89+9jPWrl3L7373O37+85/32N/Q46J76tSp5oWeLIw8OzubRx55pAdm1jWSe7rTI7pjXzsUCAmS023XG1+ynG47zLW1p1sX3S4HqXi6s12qJooBnGkOL/fkguLUFnwaqymkMfqer77dB7xdSZxfOkV3OsfMxNh9HWlb65C2tQ7RbCvSc0J023bG1omiW/ScbrsfIysQ7TxNhXPOOYddu3Zxxx13UFlZydSpU5k1a5ZZXG3Lli1xiwxDhgzh/fff5xe/+AX77rsvgwYN4v/+7/+4+eabe+pP6HnRvXHjRlRVZeTIkcyfP59+/fqZ73k8Hvr3799jpd27QuuWYSpKN0W3w+FoFX6lKArhWAEWiYBNVrQSPd12XWlL9nCxw1wN0e0yRLdDu0zdTkfrG6oaavV5o3p5niPmPXdWeiepKJDbDxorYefi+Pd8dShKbpKP2PvmL0W3xEDa1jqkba1DNNtK0W0dXRXdsd+BRPd02+H7XKZJ/Jvtfp6myrXXXsu1116b9L05c+a02jZjxgzmzp1r8aw6T4+L7mHDhgHirqgl0iq8PNK96uWxY8W+djhiqpeD7u22xw1Ghpd3j5AZXh7v6fa6Wh/f9jzdec6Y91zZaZ4lUDZGE92bEmou+OtQnHmtdreDbdtDim6JgbStdUjbWodotrX7/GIR3bZd8XSLLrrtfoysQLTztK9hD5UG3HfffTz77LOttj/77LPcf//9KY312WefccoppzBw4EAUReHNN9+Me19VVe644w4GDBhAdnY2M2fOZO3atd2ZvsZHvyHSojV4j69e3r1CaklFt6IkiG775HXL6uXdI+rpTsjpdrXO6U5avTyo/S15rpge3Vas+JaN1f7fmCC6ffXtVkq1K4nzS2f18nSOmYmx+zrSttYhbWsdotlWpPmKNFfo3HyT7SNFt9hI0W1vbHPXeOqppxg/fnyr7fvssw9PPvlkSmM1NTUxZcoUHnvssaTvP/DAA/zlL3/hySefZN68eeTm5nL88cfj8/m6NHdAS97+4kFUvVp0bE53d8PL2xbdMdttlNctc7q7RzgxvNypBaRovbo7zukO6J7yHEUX3VZ4uQH6jdP+3706frtf5nRbOWYmxu7rSNtah7StdYhmW5HmK9JcQXq6wf7HyApEO0/7Gj0eXm5QWVnJgAEDWm3v168fO3fuTGmsE088kRNPPDHpe6qq8tBDD3Hbbbdx2mmnAfDCCy9QXl7Om2++ybnnnpv0c36/H7/fb75uaGiI38FXZ44P8Tnd3S2klixHQ1GIVi8HW4puO+VJJ8OuOd3BcGJOd6ynO37fZDnd/qARXq6/l+58bgPD052ILKRm6ZiZGLuvI21rHdK21iGabUWar0hzhfTkdIua9mmn73OZprfndPcEkUiEdevWUV1d3eqaOOKII1Iayzaie8iQIXz55ZeMGDEibvuXX37JwIED0/Z7Nm7cSGVlJTNnzjS3FRYWMn36dL7++us2Rfd9993Hb37zm7YHbtoNYMjrqPBMU/XyxNcORUGNK6Rmn/BymdPdPcL6XMzwcj2n2+1UaOXpTnLcTU+3w/B0p7lyuUG/1pEpgPR0WzxmJsbu60jbWoe0rXWIZluR5ivSXEF6usH+x8gKRDtP7c7cuXM5//zz2bx5c6vrQVGUuHaHncE2ovuKK67guuuuIxgMmq3DZs+ezU033cQNN9yQtt9j9AE3SswblJeXx/UIT+SWW27h+uuvN19v376diRMnRndo2gVo4eQQK7qtyelWsL+n2+43vmQPFzvM1cjpdib1dCeGl7ft6Y6KbovCy/MrwFsAfq2OgV914VVCevVyKbqtGjMTY/d1pG2tQ9rWOkSzrUjzFWmuIEU32P8YWYFo56ndufLKKznggAN49913GTBgQLftaRvR/ctf/pI9e/Zw9dVXEwhoedFZWVncfPPN3HLLLT08O/B6vXi9UY9hfX19/A6G6NZfZian296iu63XdsGuwjBkhJerCTndLied8XT7Q9q2bEW7jiwLL1cULcR8+wIANqoDGK9sleHlFo+ZibH7OtK21iFtax2i2Vak+Yo0V5CiG+x/jKxAtPPU7qxdu5ZXX32V0aNHp2U82yQ8KIrC/fffz65du5g7dy6LFy+mpqaGO+64I62/p6KiAoCqqqq47VVVVeZ7XaINT7eqWpfTbfdCagZ2zatJdjOyw1yjnu6E6uVOR6Kju90+3dmK/p5Vnm6IFlMD1qt6TQYZXm7pmJkYu68jbWsd0rbWIZptRZqvSHOFrovu2P9lTrd4yJzu9DJ9+nTWrVuXtvFs4+k2yMvL48ADD7Rs/BEjRlBRUcHs2bOZOnUqoHmt582bx1VXXdX1gfWcbl0vxbQMIya8vGsnf1uebs2HrmiedBvldIt00SuKEreaa4e5Gn26nWp8TrfWpzthQaOdPt1ZWOzphrhiauvUQdoPvjrZMszCMTMxdl9H2tY6pG2tQzTbijRfkeYKXW8ZBtHvRdLTLR6iLQ7ZnZ/97GfccMMNVFZWMnnyZNxud9z7++67b0rj9ajoPvPMM3n++ecpKCjgzDPPbHff119/vdPjNjY2xq1MbNy4ke+++46SkhKGDh3Kddddx29/+1vGjBnDiBEjuP322xk4cCCnn356V/8UaG6rkJqKy4rwcn0oVXFobaNs7Om280VvR9FttAwzRbdDu0zdTqVzOd16eHmWEV6eAU+3isKmiB4pIj3dlo6ZibH7OtK21iFtax2i2Vak+Yo0V0iPp1uKbvEQ7Ty1Oz/4wQ8AuOyyy8xthm5QFMEKqRUWFponRGFhYdrGXbBgAUcffbT52iiAdvHFF/P8889z00030dTUxE9+8hNqa2s57LDDmDVrFllZ3fAIthlenv6cbofDoXu69THVsPbPJoh00dtxrq0KqTljCql1pnp5Jj3dA6aCw01L/jD27srTtslCapaOmYmx+zrSttYhbWsdotlWpPmKNFeQohvsf4ysQJT0TlHYuHFjWsfrUdH93HPPJf25uxx11FHt3iwUReHuu+/m7rvvTtvvbLtlWEz18sQmy52kvWIXqiHkpae7S9hxrqFIQnh5qtXLddHtNUS3y0LRXTAArvyCDXsVGp5/V9smC6lZOmYmxu7rSNtah7StdYhmW5HmK9JcQYpusP8xsgLRzlO7M2zYsLSOZ7ucbmFpx9NtRSE1h34dqYpT+8FGOd0irbTZca5G9XJHQk63x+lsta8SaUd0qxkQ3QD9x+OK1FNPrj4BKbqtHDMTY/d1pG2tQ9rWOkSzrUjzFWmu0D3RbdYkkoXUhEOkmkp25e233+bEE0/E7Xbz9ttvt7vvqaeemtLYPSq6p02b1ukTYtGiRRbPppu01TIMFYeS/pzu6BZDfWfu5hgKR5i3sYapQ4rI9bY+hUR6ONlxrm3ldHfW022El3vM8HILc7p1slxOGlT99/jquxjT0bNI0S0xkLa1Dmlb6xDNtiLNV6S5gvR0g/2PkRWIdp7akdNPP53Kykr69+/fbq0v4XK6Y/8Yn8/H448/zsSJE5kxYwYAc+fOZfny5Vx99dU9NMNOEg5Bcw0QrV4eDS/Hsj7dEOPpzqDofnvxDq5/ZTE/OWIkvz5pQqv3Rbro7TjXYMTwdLfO6VZa5XS3XUjNkylPN5DldtJAjvYiEkQJ++Pet4NdO0KKbomBtK11SNtah2i2FWm+Is0VpOgG+x8jKxDtPLUjsREe6Y726FHRfeedd5o/X3755fz85z/nnnvuabXP1q1bMz211GipwfBxG/eoaMsw1WLRnXlP9/a9LQDsqG1J+r5I4S12nGvYCI1Sg9oGs0+30srT3V7LMLeqC98MeLqz3U6ayCKsKjgVFUegIX6eAoR5yZZhEgNpW+uQtrUO0Wwr0nxFmit0v2UYSNEtIlJ09wyTJ0/mvffeY8iQIe3uZ5u7xn/+8x8uuuiiVtsvvPBCXnvttR6YUQrooeWQJLzcopxuU2v3gKe7JagJPSOMORE75km3hR3nGjRyuo08/djw8gSUSLDVNuO4uCO66M6Ap9vr1iqrN6IJfMUfL7pFuPFLT7fEQNrWOqRtrUM024o0X5HmCjKnG+zxfS7T2NGR1BfYtGkTwWDr7+OJ2OaMzM7O5ssvv2y1/csvv+xeK69MECu6WxVSi6le3sWTv11Pt3EIM1hIzRTd4c6Jbjtf9Haca9gML9cvYKcuup2pVS93ZVJ0uxwoCtSrWjE1JSBFt1VjZmLsvo60rXVI21qHaLYVab4izRVkeDnY/xhZgWjnaV/DNtXLr7vuOq666ioWLVrEQQcdBMC8efN49tlnuf3223t4dh2gtwuDZC3Duu/pTiq6jU5hPdAyzBfUfldnPd12vujtONdQYk632TLMCZ3J6dYXRUzRbWWfbmMeioLX5TDzuhV/fav37Y4U3RIDaVvrkLa1DtFsK9J8RZorSNEN9j9GViDaedrXsI3o/tWvfsXIkSN5+OGH+cc//gHAhAkTeO655zj77LN7eHYdEOPpNoiKbotzug1Pd5LcXqvwpRhebueL3o5zDekRBKagdsb26Y7fV4mEIBKBmJAiIwLBaXq6rc/pBr2YWkgPL5eebsvGzMTYfR1pW+uQtrUO0Wwr0nxFmitI0Q32P0ZWINp52tewjegGOPvss+0vsJNh9uiO3qDic7qN8PI0tgwzxu8BT3dLQIaXW0k0vDy+ZZjbqdDK060A4QA4ot5svx6J4AxnztMNWtuw+qD0dFs9ZibG7utI21qHtK11iGZbkeYr0lwhPaJb9Jxuux8jKxDtPO1r2CanG6C2tpa//e1v/PrXv6amRmvBtWjRIrZv397DM+sAXXRHYhYFjWIGWk53+gupOfTryBTdGbw5dlRITaQqn3acqxFenujp9ibp0+1QgJAvbpuR0+0IZ9bTne1xxoSXy+rlVo2ZibH7OtK21iFtax2i2Vak+Yo0V+he9fLY768iYszb7sfICmQhNXtjmzNyyZIljB07lvvvv58//OEP1NbWAvD6669zyy239OzkOkLP6Y69PVmd021siYaXZzKnW3q6rSQaXh7fMmxwcQ6tPN2gebp1VFU1j4sjrIvxDHm6vS4HTar+uwJNcba0g107Qnq6JQbSttYhbWsdotlWpPmKNFeQ4eVg/2NkBaKdp3YmGAzyve99j7Vr13a471NPPUV5eXmH+9lGdF9//fVccsklrF27Nq5a+UknncRnn33WgzPrBM2aVz72/mRlTrfD4Yjp0y1zuruDHecaimgV783zRvd0TxpUyLiKgrh9lQRPtz/mmDiM7RnM6W7Gq70INkvRbdGYmRi7ryNtax3SttYhmm1Fmq9IcwUpusH+x8gKkukFSddwu90sWbKkU/uef/755ObmdrifbY7GN998w09/+tNW2wcNGkRlZWUPzCgFfLVAck+3isU53Ybnswf6dKtBP2z6EsLxvelEejjZca7hiIqbmKrkjmjphaPG9Y/bVwG27tprvo4V3Yrh6XZ5LZlnIlluBz4pui0fMxNj93Wkba1D2tY6RLOtSPMVaa4gRTfY/xhZgWjnqd258MILeeaZZ9I2nm0KqXm9Xurr61ttX7NmDf369euBGaVASy0AanYpoOWyxt60ouHlXTv5O5fTnfk+3WeH34bn/wUn3A8HXxk3v1jsvNJmx7kGwyouYo6n7ukGyPHGX7KKAhu2bGfImClAQvRB0Agvz6CnW9VFd0CKbqvGzMTYfR1pW+uQtrUO0Wwr0nxFmit0T3Qb34NEL6Rmh+9zmUbmdKeXUCjEs88+y0cffcT+++/fypv94IMPpjSebUT3qaeeyt13380rr7wCaCfKli1buPnmm/nBD37Qw7PrAMPTnV0CbAJiw8st6tNterp7rk/3gEiV5mqt29pqfu29thN2nGs4EokX3Y6o6G41XxSO+Px8qDwOzngSf0hLzfC6HChmeHnmqpe34NFeSE+3ZWNmYuy+jrStdUjbWodothVpviLNFaSnG+x/jKxAtPPU7ixbtoz99tsP0JzAsXTFtrYR3X/6058466yz6N+/Py0tLRx55JFUVlYyY8YM7r333p6eXtuEfGZOrZpVbG6Oz+lOf3i5sZjVIzndesswb8QHTuIKeRnza++1nbDjXENhFW+c6HaaPybOr0otQqEJ1r4PzxxL+MS/A5DtAoxCbBnzdDtokeHllo+ZibH7OtK21iFtax2i2Vak+Yo0V5CiG+x/jKxAtPPU7nzyySdpHc82oruwsJAPP/yQL7/8ksWLF9PY2Mh+++3HzJkze3pq7dOih8QrTiKefHNztOUC3Q4vT3YRmaLe9HRn7uZohJfnoHtSQ/6490UKb7HjXEMRlVwjp9vhjjtvEud3X/hCVuZM4Gn3n2DPOso+vhH4GfnOEKZuz5CnO9vjpN6oXh5sibOtw+FgXXUjf/pgNT//3hgmDChoY5SeQ7YMkxhI21qHtK11iGZbkeYr0lyhey3DpOgWFym6rWPbtm0ADB48uMtj2OKuEQwGcblcLFu2jEMPPZSrr76am266yf6CG8Bfp/2fVRj1OhPv6bamT7d+UzQ93ZkJLw+GI2Yf6WxDdHfg6bbzw8mOcw1HVFyKrphj8rkhyQ1UUfi4tpzABW8BkLNjLiXUU+CKKcSWIdHtjQ0vT9Iy7M63l/G/ZZX8Y+7mjMwnVaSnW2IgbWsd0rbWIZptRZqvSHMFmdMN9vg+l2ns6EgSmUgkwt13301hYSHDhg1j2LBhFBUVcc8993Tp+rCFp9vtdjN06FDC4cyFSKcNn+7pzi5GVVqHAasqOBQLwsv1TWZOd4YKqRntwgByFd3DneDpFunhZMe5BiMR3Iab2tG+6M5yuwhHVDar/RlTsS9K5RKOcX7LNteB4AecHsjQgyfL7YwJL2+Jm2tEhS/X7QFgd6M/2cd7HCm6JQbSttYhbWsdotlWpPmKNFeQ4eVg/2NkBaKdp3bn1ltv5ZlnnuH3v/89hx56KABffPEFd911Fz6fL+X0Z9ssA9166638+te/pqampqenkho+3dOdXRRt30X8TSvdfboVRTF/U6Y93S0xojsbXTzJnO60EYmoqCrRQmrOxGrl8fOrKNS82GurG2HciQB8z7GIPIeez52hHt2g53Qb1cuD8Z5uX0xV9ZqmQOJHbYEU3RIDaVvrkLa1DtFsK9J8RZorSNEN9j9GViDaeWp3/v73v/O3v/2Nq666in333Zd9992Xq6++mqeffprnn38+5fFs4ekGePTRR1m3bh0DBw5k2LBhrcqyL1q0qIdm1gFmeHkRari16I6oRKW4BdXLI2af7gx5ugNR8ZSbkNP9h/dXMbg4R6iL3m5zNUL33bE53TG0Ft3Z7ALWVTfChBPh0/s5wrGEj50t2g7uzISWg94yjOQtw/yh6MN7jxTdacFu525vQtrWOqRtrUM024o0X5HmClJ0g/2PkRWIdp7anZqaGsaPH99q+/jx47vkJLaN6D799NN7egpdwwwvL0Jt9Jmb43O60x9ebt4UjZD2nvB0G+HlYT/ba1t47JP15HicXJwrzkVvtxtUSM8RiXq62xfdA4qyWdqsi+5jpuLL6k+ur5qDQgu1HTKUzw2Q5XLEtAxrIWa5CRTYZ2ABy3fUS093mrDbudubkLa1Dmlb6xDNtiLNV6S5ghTdYP9jZAWinad2Z8qUKTz66KP85S9/idv+6KOPMmXKlJTHs43ovvPOO3t6Cl3DH83pjjRUmZujN63uVy9PXkhN+9n0dGcopztWdEc93QEafZpntjkQJpQdf6O2czGLxBtST8/V8HSbotvRfnj5gMJsaNbDyxWFneVHMGLzq+wXWKDtkKF2YaBVL28xq5c3E3sWlOV5ef7Sgzjw3o+obQ4SCkdwOe11XkjRLTGQtrUOaVvrEM22Is1XpLlC90S3LKQmLrKQWnp54IEHOPnkk/noo4+YMWMGAF9//TVbt27lvffeS3k8252RCxYs4MUXX+TFF19k4cKFPT2djjFyurOKUGNObrNlGNbkdJvVyzE83ZlZkTQKqTmIkK3oHsuwP67AWjDhPm3ni95uN6hQ2Agv75yne1BJDgBrqhp449tt7M0ZqW0PbtF2yKSn2x1TvVw/8w1cTicluR5z3WlvczBj8+ossmWYxEDa1jqkba1DRNsqSb432RHRbCtbhvX897meQLTFIbtz5JFHsmbNGs444wxqa2upra3lzDPPZPXq1Rx++OEpj2cbT/e2bds477zz+PLLLykqKgKgtraWQw45hJdeeqlbfdEsJbaQWtKWYRZVLzeKlmc4p9vwdJtF1ABCAVN0j1B2kvv1n+I+Y+eL3m43KCO83K10Lqe7vCCb06cO5M3vdvCLlxdzUZGD/YAs1cjpzpynW2sZ5jVfqzG+bkVRcCqwT1YNy1qKqWkK0C/fm2yYHkN6uiUG0rbWIW1rHSLaVlEUIUSSaLaV4eX2P0ZWINp5KgIDBw5MuUp5W9hmqe7yyy8nGAyycuVKampqqKmpYeXKlUQiES6//PKenl7bGDndWUXR9l0k5nSn19PtcDhicrozW73cF9DEdU6s6A77zerUBzpW4Qw2xH2mo4u+wRekqt7X7j5WYasbVCRCWA8v9zr045lQvTyZZ/7Bs6fykyM0D/eiurz4MV2ZE7ZZbgcRHPh1b3fs81pBhX+fyzvqtfzdfT+1e3dlbF6dRYpuiYG0rXVI21qHiLaNnaOd5yuabaXotv8xsgK7pUyKzqxZs/jiiy/M14899hhTp07l/PPPZ+/evSmPZ5uj8emnn/LEE08wbtw4c9u4ceN45JFH+Oyzz3pwZh0QW0gtacswui2628/pzmyfbsPTnaPEiOQYT3chTSTe5tq76FVV5awnvubIP3xCbXPmC2zZ5kFatw3+MJLcOXcB4DWiIzrwdGtRDwq3nDiefQcXskMtix83oy3DtFQHn+7tjn1gK027YM0sAI50LmHiez+IXjs2QYpuiYG0rXVI21qHiLaVotsa+mpOd+z3jr4oOO2WMik6v/zlL6mv176rLl26lOuvv56TTjqJjRs3cv3116c8nm3OyCFDhhAMts7zDIfDDBw4sAdm1EliCqmpCaLaXG2zono5Rk63EV6eIU+3nrCd6On2657uQqWpVb249i76nXU+Vlc14AtG2LynOe3z7QjbPEi3zoeWvWRt/gQAr6NzOd2xK9PXzRxDDfm0qJ7oDhluGQaYIeaxa+SKGoKcUp4tv5VdaiH5DRtgdepFKKxEim6JgbStdUjbWoeItpWi2xr6qqc7brHf5sfICkQ7T+3Oxo0bmThxIgCvvfYap5xyCr/73e947LHH+N///pfyeLYR3X/4wx/42c9+xoIFC8xtCxYs4P/+7//44x//2IMz64DYQmrE37D0SOG0h5fHebrN8HJrPN11zUE+WlFFMKyJatPTTaynO1pIrYDmVp7u9i7677bWmj/v7WFPd4/enJq0cGuH3vfdY4SXd1C9PPb10eP6Awo71NLoDhn1dGvnok8PL1dic7oBKiazvuJEPo9M1jY2VmEnpOiWGEjbWoe0rXWIaFvbPIM7QDTbStFt/2NkBaKdp3bH4/HQ3Kw5BD/66COOO+44AEpKSkwPeCrYRnRfcsklfPfdd0yfPh2v14vX62X69OksWrSIyy67jJKSEvOfrYgppGaEejtM0a1d/OkW3bHbrPZ0//GD1Vz+wgLeXbITiFYvz1FiPd0B/EZ4udJk/v2Jc01GrOiu7YGK1rGhOD16c2qsBsChR054lc55uhPn/8EvjiCQNyi6QwY93dm6p7tJ9ZrzMXAogCeP0lwPe9QCADZu3sQ/5m7O2Pw6QlYvlxhI21qHtK11iGjb2Dnaeb6i2bavVi+XoluK7nRy2GGHcf3113PPPfcwf/58Tj75ZADWrFnTpQLftqle/tBDD/X0FLqGGgIcCZ5u/a1Wnu6unfzJVoJNYW9xy7DNNdoKz7a92v8tgbY83ZroLyC18PJvt0QLEUhPNzhCzbgI4U4hpzuWseX5MG4CfKtHjPRATnez6gElfkVPUQBvASW5HnaqhQAsXrWW25YsY/9hxUwYUJCxebaF9HRLDKRtrUPa1jpEtK1tnsEdIJpt0+HpFjGnO3bOdj9GViDaeWp3Hn30Ua6++mpeffVVnnjiCQYN0pxa//vf/zjhhBNSHs82ovviiy/u6Sl0HYcbPLlmTnc0vDw9Od3JvLHRQmrGD9aEl9e3aN7nep/Wwsr0dMfmdKth/AFNMBcqnS+kFgxHWLq9znzdE72b47yxPbly3RSt5p1Ps1a9PEync7rjKBwS/TmT1ctdhug2PN0J4eXePEryvCxDE9jFqnbsv1y3W4pum43d15G2tQ5pW+sQ0bZSdFtDd0S38V1IdE+33aMRrEAWUksvQ4cO5Z133mm1/c9//nOXxrON6Baa7CJQFLNlWNue7u6Hl0dvirqwt7hlmCG6G3za/0ZOd64S3+IrGNBeF9DcaU/36soG00MO4N6zGqoj0H98WubeGWzzwI8R3QVKMx4jvNzhjNutUw/+ohjRnck+3XpOd7NeSC12ZoqimOHlu3VPd6mihdJ/sW43lx8+MmPzbAspuiUG0rbWIW1rHSLa1jbP4A4QzbYyp9v+x8gKRDtPRSASibBu3Tqqq6tbRX8cccQRKY0lRXc6yCoCotWajVNcJSGnu5UPuHMkeyhFhb3FhdQMT3eL5uk2RHd2rKcbCOmiO5mnu62L/tuYfO5i6rli9S9gcxbcsDpjHlrbPPD1nG7QFi7cpujuiqc7Js/Elbmcbq/LgaJEq5c7YqamebrzKcn1sFvP6S5TNE/3vA01BEIRPK6eXZWWoltiIG1rHdK21iGibW3zDO4A0WwrRbf9j5EViHae2p25c+dy/vnns3nz5lbXg6IohMOpaa++F3thBdlFAG1XL1fS1zIsMac7bKGnW1XVqOjWPd2+Njzd4UALkJqne+VOzdM5sDCLk53zyFJboGUvNNek7W/oCNs88Jt2mz8WKE3RPt1dCi+PEd0Z9HQrioLX5aBFTebpBrz58Z5u6gGVlmCYD1ZU8sf3V7N0mybEP15VxZOfrjer5mdq/u29tsuYmRi7ryNtax3SttYhom1t8wzuANFsK0W3/Y+RFYh2ntqdK6+8kgMOOIBly5ZRU1PD3r17zX81NalrFenpTge6pzuSEF4ezenuXnh5ezndqoU53c2BMCF95cDI6TYKqRU64/OvQwE/XgJ4lWCnc7p3NfhRiDBxYAGnrf8y+kbLXigYkJ4/ogNskdMdaIJgk/ky3tPd+ZZh0QFiqpdn0NMNWgXzloDWMszRKqc7n+JcDzV6TrdbCXPy6CzeXefn5//+logKf/tiA2ftP5h/zN0CwOY9zfzujEkZeXBI0S0xkLa1Dmlb6xDRtlJ0W0M6crpFL6Qmc7rtf57anbVr1/Lqq68yevTotIzX42fk0KFDufbaa/nggw8IhUI9PZ2ukV0MRAWw4YU2nM9W5HQr+u8yhL4Vnm7Dyw2Q37wFqleaOdhFrvhK4+GQjwKajEnGvdfWRT9p13ss9V7OxYGXONCxJvqGr7b7k+8ktmgZFhNaDlpOt7sLLcNMXF7Iq9B+zqCnG7QK5kZOtysmvtxoGeZ2OsjKyqZezQHg+6O1vy+igtup4AtGTMEN8O/5W3j+q00ZmbtsGSYxkLa1Dmlb6xDRtrJlmDV05ruCbBnW+xBtccjuTJ8+nXXr1qVtvB6/a7z44ot4vV6uueYaysrKOOecc/jnP/9JbW1tT0+t82Rp4bLR6uXaZiOnu7ue7uTh5drrzIhulYebboanj0ENNAJQ4IwX3WrQT6GiiW4/nrj32rroD2z+lDzFx+Hb/xb/RsvepPtbgS1W2WNCy0Fru+bpTk43QIlemExPfcgUWW4nPtUQ3dHtRng5QFmel116iPmMcpXCbDfjK/L5+IajOGXKQBwK/GLmWG49aQIAD8xaTSgDYebS0y0xkLa1Dmlb6xDRtrZ4BneC3mhbGV7e+xDxPLUzP/vZz7jhhht4/vnnWbhwIUuWLIn7lyo9Hl5+5JFHcuSRR/KnP/2J5cuX8/bbb/PII4/w4x//mEMOOYRTTz2VU089lZEje766cZtkaeGyZvVywwttQfXyxJVIs2WYhaI7nxZKqYMgZPt3A/nkO+ILqUWCfgrQe3kTXwStrYt+ZGhDXOJvWFVwKmofFN1JPN3oYjvB093p0KGTHoANn8KII9M2zc7gdTlMT7czScswgNI8D3vqCxjFTooie5n361NxOx04HQqPnDeN+86cTJ7XRSSi8scPVtMSDLO9toVhpbmWzr29h9XKnfUMK80hx5PaLVOKbjGRtrUOaVvrENG2tngGdwJRbWuIUCm6+wadioaUdJof/OAHAFx22WXmNuO6UhTBC6nts88+3HLLLcydO5dNmzZx3nnnMXv2bCZNmsSkSZN49913e3qKyfEaotu4YWmbjZzuqOju2g0geU53Qni5BTndRruwQqXR3OYJ1gGQpySKbp/p6fYp8Z5uR8zf/e2WvWzc3YSvtooKpYaIqhAaPIMNkQF8Epmq7dRSm+a/pG1skdMd0y4MNE93t3K6ASomwyHXthLtVhMbXu6MmZrm6daukx8fNgJHXn/tjabdZLmdOGNC0fO82t/scCgMK9XC0DftabZ87m3ZdvbKKk58+HPueWdl2sZMByJ+CRQFaVvrkLa1DhFtK0W3dXTV0y1zusVF5nSnl40bN7b6t2HDBvP/VOlxT3dbVFRUcMUVV3DFFVfQ3NzM+++/j9ebmTZSKaOHzSaK7mifbiuql2uvMxFeXki0yFdWSKs4npPQMkwN+839jPBic84hbd/qBh8/fPJryguyeOukIFnAZsoZftl7nHXvbH7mf1r7QF/zdDcmiG6lGTea2OxS9fIeJMvtMI9/nOgG8Gie7hMmDYBNY2HB51o++7f/1MLhh81oNd7w0lzWVDWyaXcTR47tZ+nc27Ltu0t2AvD1+t2tPtPVMdOB3c8FkZG2tQ5pW+sQ0ba2eAZ3gt5oW+np7n2IeJ7amWHDhqV1PCGWgXJycjjjjDOYOXNmT08lOWZOt37DIv6GZVZxTqfodiSGl6fH0722qoGTHv6cv3+1yRTdRTGe7mxddGcT3zJMDQYoMD3diaK7SR+7kVBEZXttC40bFwKwwTkSxeGgKMdNPXr4cF8T3YanO1fz/hbQjItu5nT3ENkxnu646uUKZng5AIane9U78NbV8O9zIRSADXPgxTOgZgNEIvy08TFucL3Cpj3RhR+rSGbbSETls7Wa2N5c00yTP7Vij1J0i4m0rXVI21qHiLa1xTO4E/RG20rR3fuRNug+L774IoceeigDBw5k8+bNADz00EO89dZbKY8lhOi2PXrYbERN8ELr177TiurlpqfbEN3d93S3BMJc869FrNhZz8vfbI2Gl8d4urPDDQB4VV/874/1dCfmdAe10OCtNdEQ4cD2xQBs82pl+ItzPNSquujuQ9XLwxGVhStWA6CWabYoUJpwK7q4c7YfXm638Kkst5MWvZCemVaBfk24Y3Kyc8u0/6tXaP/7amHTZzDr17D+Y/jsT7DxU/avfp2fud6keWdMdXuLSFbJdWVlPbsbtUgNVYXVVQ3dHjNdiFZNVySkba1D2tY6RLStrF5uHR3ZVlYv752IspAlAk888QTXX389J510ErW1tWYOd1FREQ899FDK49n/riECWYk53boXulVOdzrDyw1Pt1N7Iw053fe8u4I1VZpXu7LeF+PpjoruvIguuiMtADQpuvcyHKBA0US1T4nvDW2I7i0xojtvr5YfuztvLADFOW7q1L7n6f5geSXB+ioA6nO0MJYCmnEL6unOcjtp0cPLFWIffk6IfcDrXv045vweqpdrPy9/A76JVrUfvXu2JfONJZltP18bH1K+amdqolt6usVE2tY6pG2tQ0Tb9vQzuLP0Rtt25OkWPadbhGNkBaJcUyLwyCOP8PTTT3PrrbfidDrN7QcccABLly5NeTwputOBmdOd2DJMo7vh5ckLqWmvo57u7q1I7mrw86950f7INU0Bqhs0D1+sp9sQ4K6wJrobHdrfrsR4uhOrlzuC2nZDdHsJUB7cCkBDkdYWqijHQy26gO8h0d0TK9dPf76BUrSQ/ZUBPbxciQkvFyyne0hJjnn8FWIefom2zU2Sn73tm+jPwSYt9FznUN9nlrcNS2bbz9Zoof9FOdpxWFVZTzii4g/FL3LtrGvhgVmruOOtZTz00RrzfSm6xUTa1jqkIHya2QAAxatJREFUba1DRNuKIhB6o207KqQmPd1i0tMRnL2JjRs3Mm3atFbbvV4vTU2ppz3aTnSvW7eO999/n5YWTdQJcdEnVi/XN0ci1nm6zbZkaSqkZuTMDi7OJtvtpIw6DtzyDKXUxVUv1/K7VVN01yua6HaEYjzdHYSXj1O24iTCbrUAT9FAQPN016qG6K7t1t+SCj35wF+4eS+LttRSptQB8PneIkCrXh7N6e5i9fIe4mfHjObPFx4CgBL78HM443fMi/F0Dz4Qsoujr4dGC6qpRcMIqQ4mOjZTvWmFJXM255hgS18wzIJN2gLQjw7WohBW7Wzgmn8u4oB7PmL9Lu262LCrkR88/hWPz1nPC19v5qGP1vLmt9uTjilFtxhI21qHtK11iGhbKbqtoy/ndItwfKxClGtKBEaMGMF3333XavusWbOYMGFCyuPZRnTv2bOHmTNnMnbsWE466SR27tQqBv/4xz/mhhtu6OHZdUCr8HJts3G/clpQSK11Tnf3wssNQTy0JIcBBV4edD/OZYF/cpnrf5Q6o2HhBTRR5A6ZocN1unfaoQZNT3czieHl2vatezWhPs6heblXRYZQmqcJ9OJcD3V9rJDa3z7fQBZ+ivVFjQ8rtQWMXMWPR9Wrwwvm6XY7HYwfUg4keroTRLeR0w0w/mQYd5L2c145nP549HMHXMa37ikA+Je8Zs2kjd+VYMvttT4C4QgluR5OnDQAgEVb9jJreSUN/hBPzFnP1ppmzn7qa3bU+RjZL5cj9ArrRvG19o7XC19v4pUFW9M2X7udCyIjbWsd0rbWIaJtRREIvdG2UnT3TkS5pkTg+uuv55prruHll19GVVXmz5/Pvffeyy233MJNN92U8ni2Ed2/+MUvcLlcbNmyhZycHHP7Oeecw6xZs3pwZp3Ao4eX66LaOMVVrM/pDqfq6X7/Vi13NoEtMaL7OO8yjnBquQojlEoq3C3mfkVKIyWuoPnaqDjuIRitXp7o6Q400eALUtMUMMcE2KAOjIruHA91hqfbVwcZyiXqqZvTsu11/G9ZJbe6/glAtVLKxkjU+5sX1rzfouV0A+DRrt/YmbUS3d4CyCrSfh57Ahz0E8irgCNv0tqH7X8JlI2DqRewouhoAHI3f5yW6fmCYaobfK22J9qy3qcVs+uX52V0/zxcDoWQGb0SYebSG1n31PnsbvQzviKfV346g58foxXD+2rdbiIRtc3jtbWmmTveWs5Nry5hTYrF2dqary3PBUGRtrUOaVvrENG2ogiE3mhbKbp7J6JcUyJw+eWXc//993PbbbfR3NzM+eefz5NPPsnDDz/Mueeem/J4thHdH3zwAffffz+DBw+O2z5mzBizRHu6CIfD3H777YwYMYLs7GxGjRrFPffc07UbjDvXLBBlfNr0Qht9upVI/Bsp0qmc7s4UUmvaDV8/qonucDDuLVN0F3u4pDFawGqQspsyV4zopokit/bZZtVLi6pVqvYQolBJ7ul2BJvYWhMdY5guujepFZTmaZ8vznFHPd2o4K/r+O9JAxnN6Q4HYcFzsHcTf/xgNSc45vMj10cAfDD6DoK4aFQ12+WFa7XPCObpBsCti+6YqbUS3YoC57wIP3we+k+AgVPhxtVw4OXa+6c8DNfOh7x+hMo1T3du46a0TO8XL3/Hob//mLUJYrct0V2c68bjcjC6vx7VocDppds4wTGfo/2fMNRdz5MX7k9ZnpcpA7KY4d3IiJblrFm/rs3jtXR79Pz+62cbuvR3CHEuCIq0rXVI21qHiLYVRSD0Rtt2lNMtciE1EarLW4XM6U4fLS0tnHHGGaxdu5bGxkbmzp3L9ddf30qrdhbbnJVNTU1xHm6DmpoavF5vkk90nfvvv58nnniCRx99lJUrV3L//ffzwAMP8Mgjj6Q+WEzvYeP2ZFYWt7J6udmnOwVPt98QGSoE4gsAbNNF8X7Bb6kIbCKoaiJpsLIrrk93kdLIgGxN4DfjxadqOcceJUgBbVQvDzSxda/23siyXEYoWrXuTWo5ZbnasS3K8RDERYsh2DMUYp7Rm9PS/8A71xH460wK1r7Jg+4ntO2HXse4Q08FoB7tGigKaAsTcbnPSeZoyweL0w0Ot7kwBOBwOlvvN+II2OeMDofLGzAGgNxQLfjquzW12uYAH6yoIhiO9t8255hgy7oWTXSX5GoLQxMHamkkp00dxHWD15r7/Xq6m+FlubD+Y9xPHMy/lVt53XsXI146Coc/Xtgbv2P5jqjofuu77eysayGR6nofP31xAc99uTHp3yJiCxtRkLa1Dmlb6xDRtrJlmHX05ZZhfVlsirKQJQKnnXYaL7zwAgCBQIBTTz2VBx98kNNPP50nnngi5fFsc9c4/PDDzT8MtBMlEonwwAMPcPTRR6f1d3311VecdtppnHzyyQwfPpyzzjqL4447jvnz56c+mB5aDkmql+sXf7RPd9dO/uQ53Xp4uWqI7k54uoPR3OxE0W14ugeHtwHwVWQfAEqVBopCUXFSQBNTyzXva7PqpSWiie4sooXUWud0N5o54xMH5DPcEfV0l+Ubnm7t/9oM53Vn4uYUCke48T+LmfvpewB4WnbxF89j5Ch+GPU9OOY2DhxewmtXzSC/sFSbCyq4smHIwW3O18o5dxtPTvvh5SkwbcxQdqua4F289NtuTevTNbsI6yEoS7fVxr2XaMs6w9Otn5s3HJjNP0Z/wh1H92NIdTTU/bjyBlj1Hrx4BtRuxu8qwKe68YYbUSoXJ/0dy3doiwcuh0IwrPLM5/HCenttC2c/9TXvL6/i9/9bRXMg1OpvEeZcEBBpW+uQtrUOEW0rikDojbaV4eW9E1GuKRFYtGgRhx9+OACvvvoq5eXlbN68mRdeeIG//OUvKY9nG9H9wAMP8Ne//pUTTzyRQCDATTfdxKRJk/jss8+4//770/q7DjnkEGbPns2aNWsAWLx4MV988QUnnnhim5/x+/3U19eb/xoadA+WJ+rpTqxerprh5enzdBsrkV1qGRaM8abFCHBfMEyVnuNaFtgBwHJ1OPVqNgDZwVpzX68SYt9iLTe7mSxTdJcpUQ9kcVFR/PwDjaaon5DfQg5+wqrCVrW/6UUs1lsy1UYM0V1LJkjrzWn7Qmisjt/21aPsefka3li4mdw9Wp68T9X+1tDAA7UQaz2EfP9hJeQXxRQYG34YuBNC9RNWpm17Q3XnJISXu9retwPGlufTlKdVD39z9het2nWlwuyV0eMTG+INbYeXG+fooIUPcNi2pyl++TSUvVGR7NizTusrDjDhFLZePJ/Xw9pNOrzju6S/Y9l27Xq5+qhRALyyYCu+oPZ3RSIqFz0zj017tGvGH4owZ/WuVn+LiF8CRUHa1jqkba1DRNuKIhB6o22l6O6dZDRtspfT3NxMfr7mXP3ggw8488wzcTgcHHzwwV1KfbbN0Zg0aRJr1qzhsMMO47TTTqOpqYkzzzyTb7/9llGjRqX1d/3qV7/i3HPPZfz48bjdbqZNm8Z1113HBRdc0OZn7rvvPgoLC81/EydO1N7ISiK6E3O6M1FIrTM53bHe7Zift9e2oKqQ43HibdI83VvVfmxXyxJHAGBsliYYmvHSEtE8mP2o1d5053DYuAHx848R3eM9mnjYrpaRlZWF16V9vsjwdKuZ7dWdtgf+ziXw9Pfg3+dFtwWa4cM7KF/zL450LGa8olWq/m2/B/Cd+BCui14HT278OFmF0Z9Hz2x3vt2es5W4EzzdycLLU2DAcK01Q3bDZv7+1aYujREKR5izOiq6N+xuotEf9SC38nTHhpdHIrBhjvbGHj20XHFGX+/QPfD7XcyowRWs8miRIoFt8Z55RVGorvexu9GPQ4GfHjmKgwpqCfoaeX+5FgGycMte1u9qIt/r4rSpWku9WcsqW/09wpwLAiJtax3SttYhom2l6LaOVG2bKLpFzukW4fhYhSjXlAiMHj2aN998k61bt/L+++9z3HHHAVBdXU1BQUHK49lGdAMUFhZy66238sorr/Dee+/x29/+lgEDBnT8wRR55ZVX+Oc//8m//vUvFi1axN///nf++Mc/8ve//73Nz9xyyy3U1dWZ/1as0HsGJwsv118bOd3mKd9F0d1eITXV9HR34uYYG14e83NsuzBlr7Zys0Xtzza1X9zH96ja31oU1IRLs+rFr3ttK5QabaecUg4aWRr3OTXQyDI9lHeomc9dQVleNFff43KQ63FmvG1Y2lYEN38FqLB9AdRspLLOx+eff2SG/f/U9Q5uJUwkq5i7r7qErOmXmq3m4ugtort4WNo83QCeflpV8GFKFa8s2JbSCvzqygYO/f3HnProl9T7QhTnuCkv8KKqsDzG251oy9oWrWBgSa4HqpdD8+74vumTf6j9v3Mx7Fmn/TxgKoqi4B9wEADZe9fGDomiKGZo+ch+eeRWzuelwLXc736al+ZrizL/W6oJ7JkTy7loxnCGKzs5edWvCGz4otVY7b2WdB1pW+uQtrUOEW0rikDorbZNXjNILw4ssKe7L3t4ZSG19HHHHXdw4403Mnz4cKZPn86MGTMAzes9bdq0lMfr3jfhNPLZZ5+1+/4RRxyRtt/1y1/+0vR2A0yePJnNmzdz3333cfHFFyf9jNfrjSvoVl+vh1N72/Z0m326jRJr6ezTTYKnuzM53XGe7mZqmgL8e/4W6nRxMbgoG7ZportSKY/zdNerOdSoBZQqDSj12wEtvDygn0KxojvHE39auQjT0lQPZFMe1Dzpm9QKSvWwXYOiHA+1jbro9tV2/PekgbQ98KuWRn9e/R63r53O8NX/43C9+Ph0xyoAHAOngrOd88AQ3UXDoLR1hIcwD/7BB6Lwtvmyu6KbkpEAjHBWsa66keU76pk0qLCDD2n89t0VbK9tYXutll5x9Lj+NPpDfLCiiqXb6yjJ9dA/P6sNT7dby+ne8Km2ceTR2mLI5i+19mZLXoJGbSGJgsGQpy1UDRg+jsptxbiIL9amKArLdKE/aWABfPs3HEQ42vEd123YxcbdTabH+4RJFUwrd/Ns1p8ZqW6j8a1f4LluvnmDEeZcEBBpW+uQtrUOEW0rRbd1dFZ0J4Zky/BysRHlmhKBs846i8MOO4ydO3cyZcoUc/v3vvc9zjjjjJTHs43oPuqoo1ptiz1ZwuGu53Em0tzc3GoVzOl0di2UJian2wgnT/R0OxRV6yeWTtGtbwqrqXi6Y3O6m7jp1cV8FJPjOrGgBUI+UByECwaxvT4qumvV3KgXeu8mAJrIwo+mKgsUfezcfkkfTmPzgzhLBpLXrHnzdjoHcOTYeE96ca6b2kYjvLy2478nDaTj5nTvuyu4ZNV8Bumv1VXvMH/LGE53rGu988Cp7Q9WoI8y9oToQW5jvsle24bBB8Z7ursZXm6I7rGuavBrFb87I7q/Wr+bz9fuxu1UuPTQEazcWc9VR41i1rJKPlhRxTNfbOS3765kxshSprcS3UFwZGue7vlztI0jj4KDr9T+qap2/Qf06v4xx3bKkGIWRMYxVmktug1P9+SKHJj7XwDylRZGK9u56h8L2V7bQo7HyZFj++F492eMVLWFqry6NbDxU20OCHQuCIi0rXVI21qHiLYVRSD0Vtsm/34pRbfIiHJNiUJFRQUVFRVx2w466KAujWWb+Iu9e/fG/auurmbWrFkceOCBfPDBB2n9Xaeccgr33nsv7777Lps2beKNN97gwQcf7NKqBd7Y8HIj9Nu4YWnbrczpjqSS0x0TUr5pZ3Wc4AYY592j/VAwiH6FeXHh5XXkUqvqolvPX10bGWR6uk1yy1oX+wLeuHQCr151CI6a9QDceN5J/Ox7Y+L2K87xUK9mNry8u2E41fU+nvt8LWXNMdWnt8zF0bKHabrorldjWuENmNr+gAf+GE5/Eo7+ddK3E+do2xCqQfsntAxzt71vZygZAUBReA9Z+Hnrux1mFfK2UFWVP7y/GoDzDhrKr0+awIs/ns6Y8nwmD9YE+846rYDgN5tqCCcMZ1QvL8lC82wDjDwyuoOiQOno6OsY0T15cCHfRMaZ16k5J2DBZu3cnuFYCr5oePt093pWVWoFGo8e15+smlXw3T9RFQfzIuMBaJgTrZYpYgsbUZC2tQ5pW+sQ0bayZZh1dMa2yfaRoltserPofuyxxxg+fDhZWVlMnz69012nXnrpJRRF4fTTT7d2gh1gm7tGbJGywsJCysrKOPbYY7n//vu56aab0vq7HnnkEc466yyuvvpqJkyYwI033shPf/pT7rnnntQHSyK6jXO8dZ/urp38SXO6Da2dSk53THj5J0s3AXDipAr21QXI5Bxd6BYPp6IwOy683JlTgjtXz9XWQ9m/UyYQUBPEVE5pa2GoAHXbYO9mqNHEqTNJ6HRxjodadE93c03Hf08a6G5O9+qqBkYqO/EqQRrJJtx/Eooa4QLnbAYqNYRVhefCx0c/MLCDHBBvPkw9D7KLOpxvste2IacExRWtvK44uxlUk1MCWUUA7JNdQ3WDn8/WRCt6N/pDvPHtNmqaAua2zXua+XZLLR6ng2uPGR033OQEL3koolJZ74vbFtQvqdK9S7UFq5wy6L9P/LzKYhaOBkSPbVmely25k0k8Ost2NLC70U+/fC/jds/WNjq1NIv/G1dHQZZmp+/vOwCqVwKgDJnOB6NuI6Iq5G+ZDbu1xRxhzgUBkba1Dmlb6xDRtqIIhN5q2/ZyukUupCbCoohV9Nac7pdffpnrr7+eO++8k0WLFjFlyhSOP/54qqur2/3cpk2buPHGG83WXz2J7c/K8vJyVq9endYx8/Pzeeihh9i8eTMtLS2sX7+e3/72t3g8no4/nEhsyzAz31p/rf/vsCCn26xeboaXd6ZlWNTTvXPXHtxOhV+fNIE3Tgiy4qAPGBrUvbVFwxjbPy9OdE8cOZQjp46NjuX0sNo5Jqmnu/XDCXj5Anh4Xy0UV3FA8bBW0yvOcbNL1cVQY+tqzVbQ3Qf+mqpGJihaHvzKyBC+8hwCwHWu17T31SG8HT5Ei0goGAxFQ9M2367OOVMoMfUOui26wQwxP2ekVoPgiTla1MRX63dz/J8/4xcvLzY92wAb92iLTCP75dI/P771Wmmuhzu+P5Hrjx3L98b3B2D73njRraCQ7XbirVygbRh2SHS1yxwouacbIG/IJMIJsvvr9Vo0yYUHDMC55l1t4/SfAlBWu5h3f344j5w3jRMmVYDRlqx4BD86+RjmqNr4Oz99RpufQOeCaEjbWoe0rXWIaFspuq1Dhpf3TUS5plLlwQcf5IorruDSSy9l4sSJPPnkk+Tk5PDss8+2+ZlwOMwFF1zAb37zG0aOHJnB2SbHNqJ7yZIlcf8WL17MrFmzuPLKK5k6dWpPT69t2unTHZfTDWnO6TZEdyqF1KKiO0fxM3NCOUOKs3G+83/kLHke5j6hvVk8jMsPH8n9PzoG1aX16ia72PQ0AprH1pVFgERPdxLRbfzgygZ3Dky9AFxeEinK8VCplmgv6nd2/PekgW6L7soGJji2ALAiMozrNs9grWMELkVbaPk2Mpr16iBWn/gS/Oj1Lkc7tDVHO99Q40R3dwupgSm6Tyrfi8fpYP6mGm57cykX/G2eWSRtxc5ov/gte6JV+eP4381wbwWXrfoJPy/9hmlDiwDYXhcvulH0yuWVS7TXyfLxDdFdMBhy41vsTRjSj61qedy2zTXNuJ0KF5dv1ELL8ypgxs+0N3etYkh2gFOmDNSOq147gZIRDC/LZeuQUwHIWvMWqKpQ54JoSNtah7StdYhoW1EEQm+1rRTdvQ9RrimDhoYG6uvrzX9+v7/VPoFAgIULFzJzZrSrj8PhYObMmXz99ddtjn333XfTv39/fvzjH1sy91SxjeieOnUq06ZNY+rUqebPJ510EoFAgL/97W89Pb22SRperl30aqvw8nR6urXX4S62DMtGC3FlxyKo1Ty1RDQPIsXDyfY4mblPBUrREP0DRZrwNhh6MB6Xo3Oe7pl3wnXL4LZKuHUnnPZo0ukV57ipVPXf0VQN4VDS/dLG+o9Rdq2KzrMLN6fVVQ1M1D3dDUUT2BP0ckHzjWxXtVD8kolHc+a0QYw54FjoN67bUxbpwa/EXBtKd3O6AQYfCEDevIe4ebwWTvSPuVtQVTh8jCZ4N+xqNK+7zbroHlYaI7pbauGbZ7SCgdvmw5tXcWjuDgC21yaIbhSKc91aD3aAiim0YtxJsM8ZMPPOVm9NGVzEega2GvOkyQMo2qh7uSeeCvnlUDxce719YXRXvX2f8V7e5JNpUT0U+7fDjm+FOhdEQ9rWOqRtrUNE24oiEHqrbaXo7n2Ick0ZTJw4MS69+L777mu1z+7duwmHw5SXxzsyysvLqaxMHhn7xRdf8Mwzz/D0009bMu+uYBvRvXHjRjZs2MDGjRvZuHEjmzdvprm5ma+++orx48f39PTaJq5lWHyf7nQVUkvep1v7v6uF1HLxkZ/lguVvtN6vKCb0u1AX3VlFCaJ7hi66O/Z0O4YeBIZ4b4fiXA97KNTaoKkRTXhbyce/Ranbar5MNQdIXfh3flt9rdkO7KCDtbZ21RRzqeNe1DOe4oRzr+HBc6biaq9NWAqI9OBXPLnRn9MRXn7g5TDuZAj7uXTLLYxwaOfHT48YydMXHYCiQIMvxB49r3uzHl4+rDQ6D1a9qy0ulY7RKsQD+2x+AYC9zcGEP0ChIjsS04N739Zz8uTAD5+Hfc9u9dZBI0pw9o9faBnVP5/rjh6uzQM0wQ4wWK+Eufz16M41Rnj5cACmjBzE7Mg0AMLLXhPqXBANaVvrkLa1DhFtK4pA6K22lTndvQ/RcrpXrFhBXV2d+e+WW27p9pgNDQ386Ec/4umnn6asrKzjD2QI25yVw4YNi/s3ZMgQsrKyOv5gTxMjLIxiyoYXOpIm0Z10JVJ/nZKnO6aQWrbiJ9/rguVvahsqYgRFbL71lPO0ENoxx8YX9xoyHY/TQUBN9HSXtq5e3smLvijHQwQHNYou7mNCzP2hMI3+NHu+63fEVdhO6ea08r8o//05k5QNeJUganYxBx50qFmga+DQUShTzgVHN1tlJdBqQcPGDxZHTJuwtMzT6YKznoXBB+IINvH3aat55Lxp/OrE8WS5nQwq0lIhNuzSzvPNNUk83Yao3fdsOEq7sXtWvcnBpc2tQ/8VhUnOrYAKeeWQ1z+l6bqdDo474oi4bdccM4YRdfPBr4eWDzlYe2Pqedr/3/4DFjwHIT/Ub9e2FY8AYGRZLp+4DgMgvOSNVpXR7XwuiIaIlYpFQdrWOkS0raxebh2yennfRJSFLIP8/HwKCgrMf15v6/TTsrIynE4nVVVVcdurqqpatfMCWL9+PZs2beKUU07B5XLhcrl44YUXePvtt3G5XKxfv96yv6c9erRP91/+8peOd9L5+c9/buFMuoG3wPzRuD0ppujWL34LCqkZ/0dSyemO8XTn4GeEbyXUbdXy0n/4PDx1BGSXaALDYN8fav8A3Hp+96ADIKcEj8th9umODlyGosT3p+7sRV+co41VTQn92AMNO4D9ATjnqbls2tPEp788msLsNIQqR8LQWBWnszp9c9q9Ft64EoB/h45macFR/O6nPwRPDnedOpGbXl3ChdNbF4pLByKttlty43dnwcFXwavfMHTH/xh61n3mBTeyXx7b9rawYVcjBwwrZoshukv0hbGmPbD+E+3nfc6EstEw/HDY9Dk/9X7IhyTe6BXGRDZoP1Yk8XJ3AiUhpUBZ8CzU69fhPqdHC7ONOgaOuQ0+/i28dyPklAIquHPNXHGHQ6FxyDE0bX6U3KbtBHetjB/bxueCaIh0nYmGtK11iGhbUQRCb7WtDC/vfYhyTaWCx+Nh//33Z/bs2Wbbr0gkwuzZs7n22mtb7T9+/HiWLl0at+22226joaGBhx9+mCFDOo6+tYIeFd1//vOfO7Wfoij2Fd2eJDnd+mtDdDstzOkOmZ7uzlQvbzF/zMHPgL167+Gxx0PpKLjqK63AWVsXaclIuOprU5R7XA4aY0R3WHHj9OZ3+eFUnKNVj98eLmIfB9Cg5WlEIipLttUSUWHptjoOG5OGUJGm3aBG4mpLd/rWNO8pCDSyo2h/bq+8lBOHDIXCQQDsP6yE2Tcc1f35tYFID37LbvxjT9TE6N5NWg704AMAzRP82ZpdbNjdRFWDj0AogsuhMLBIj5hZ+ba2OFUxWRPcAIf8HDZ9zpF7X2OIcixxjeoUhWFBXXQnCy3vBErBgPjX2+ZBid4lYdJZ8TsffiNs/hrWz4bP/qBtKxkRdz1OHl7Osk0jmK6sQqleFfdxO58LoiHSdSYa0rbWIaJtRREIvdW2UnT3PkS5plLl+uuv5+KLL+aAAw7goIMO4qGHHqKpqYlLL70UgIsuuohBgwZx3333kZWVxaRJk+I+X1RUBNBqeybpUdG9cePGnvz16cEVFZ2JohsLwsuN8J8u5XQnhpcH9T7H/Sdo/ydp49WK8onmjx5nfCE1n6eYXEXp8sOpSPd074gUa4kP9VqBqwZfiIgKpdQx8c3j4eAL4PDrOzVmm+gtyRSXB9DC1pVIsJ0PRAnuWosbeDV8JCFcjO2f1+Fn0kVXQ/d7Astu/J4cGH8SLP2P9s8Q3f00j/aGXY1mEbXBxdnRfHojtHzSD6JjjTkW9jkDx/I3uMD1MYtj5w8MaF6jveiqpzvxeGUXw4l3adfakAMTdlZg33M00W1UTDcKrOnsN6yYNZHBTHesQtmzOuHj9j0XREPEL9iiIG1rHSLaVhSB0Ftt224kpcA53SIcH6tIphl6A+eccw67du3ijjvuoLKykqlTpzJr1iyzuNqWLVts//f2qOjubUSrl2uvjZxuxWwZ1rWbQNJCavqmaJ/uJDfHhkrI7Q8OB1trmunf0mgG0ObgJye4V3uR269L8/K4HATU6KJDwFtKLq1vdp29CPK8LlwOhSqjbViDltNd26IVxjrQsZqSpnVa9enuiu4GLS9EcWcDmkBzqB2L7o27m/BsXskg4PPdmtgeP6Cg/Q+lEZEe/JZ+mZp0lia4l70Ox/0WnG5GlmnHY8OuJrOI2lCjiFpDFWz6QvvZKF6mTQxO+Qvs+I4SJV7EOpUwRY1rtRcVk7s0zVbHa8q5MP0nbX9gzLGgOKPpIgmie8rgIt5XBwPg37Gi3d8l6ToiXWeiIW1rHSLaVopu6+huITWRPd12F19WIlohtVS49tprk4aTA8yZM6fdzz7//PPpn1CK2Ep0b9u2jbfffpstW7YQCATi3nvwwQd7aFadp80+3UZOd+cDmONobyUyWkgtwdO98TP4+yla+Oxx93DuX+fyWkstFfruOfjw+vVg2tzUCkQZJHq6g1klreab7HVbKIpCUY6HquYibYPu6a7Vq0qXKA369m1a66fYwm6pogt6xR0t1qeEA23tDcCO2hZ++NhnzI3sAgXGT5jMfmVDOGpc1xYtuoJID35Lv0yNOkZbLGqqhm9fhAMuMz3dW2qazWJqw4we3Sve0hamBu3fSsiSVQAz70L5+Ly4zX90/xVnJKBV7teLmaVKq+Ole+XbJKcEhs6AzfoCQcJcc70uGgrHQjMou2V4uVWIdJ2JhrStdYhoWym6rUOGl/dNRLmm+iK2Ed2zZ8/m1FNPZeTIkaxatYpJkyaxadMmVFVlv/326+npdQrTs62Hkxu3K2v6dOuiO2IUUkvwdFfpXrCdiwmEImyvbSHbG+1DnKP4cft2ay+64+mOyekOdVN0g96ru9nwdGsh4HubNTFcTEN0x12rYOjBXZm2RqPu6XZ1XnTPWlZJtm8nLm8E1ZXFPRfOjIYcZAiRHvyW3vhdHi0HetbNMOd+2PdcKvLcZLudtATDfLFOO7fNyuXJQstjGXFEqyWx45yLUB1elFMe7vJxbnW8OjPOuBNiRHdrsR8pGw9bILt5e7u/S9J1RLrOREPa1jpEtK0oAqG32laK7t6HKNdUX8Q28Re33HILN954I0uXLiUrK4vXXnuNrVu3cuSRR/LDH/6wp6fXKQxPd7RlmOHptq6QmunpTszpNiqV+2pp8Gme4iyiojIHP45mQ3R3rTCZJrqj6zaR7FJtbt3IOy7O9VClFmsvdG90XYs2/2IlRnRXLe/KlKPogt7hyY7OM9K+6F5VWc9QResNrRQNy7jghq6H7vcElreCOeBSKBqq5ec/Nh3Hvf15KPsZPARZvqMe0Ht0122DLV9rn5l4evKxckpwFA2O2xRxuOGS/2pVxrtIl9rMjDsp+nOiVx4o7T+AXWphXLu7To8t6RQitgcSBWlb6xDRtrJlmHXIlmF9Eym67Ytt7horV67koosuAsDlctHS0kJeXh533303999/fw/PrnMktgwzL/5uiu6kOd1GoYu2croN0d2ylwZfCCdhvEq0z3WB0owS0quZp9h/2CAxvFzNKY2bY7L5d0RxjptKI6fbXw/+RjO8PE50V69I8ukU0EV3bHi5owNP98qdDaboTiaGMoFIq+2W3/hdXjj6Nu3nui2gRjg+8AEvee4hj2YcCkwYkB9tEzb4ILPKfNL59hsb99o9/niU7kRT0MXjVToKpl8FUy/Ufk5geGkOqyODW3nm7XwuiIZI15loSNtah4i2FUUg9FbbtpfTLXIhNREWRayiN+d0i45twstzc3PNPO4BAwawfv169tlnHwB2797dk1PrNK1yuo323BZ4uo0t4bZahgUM0V1HvS9IDv7kg7tzwJPbpXl5XA5UHARUJx4lDDn9Ws032ev2KM7x0EQ2AWcOnnAzNOykVv9TSmLDy6u6KbqN6uXuGE932A/hEDhbXxahcIQ1VQ2cbIjukq7l+HYXkR78Gfkyte/ZEPYDCmQVoL79M/bzrePfB64nctCVDC7OgXn6udJBPrXWU/u96Ot9z+n29Lp8vE78fZtvDSvNZa06mEMcy7o2tqRDRLrOREPa1jpEtK0U3dYhw8v7JqJcU30R24jugw8+mC+++IIJEyZw0kknccMNN7B06VJef/11Dj64e96mTGF6uhNem4XU0im6zUJqRk53Yni53h7MX0dDs4/stkR3F/O5AcrytFroAdx4CFNQWtFqvslet0eR3qu73lVGWXgL1O9gb7MW/l7qiLY8a9i6hJUb9nDQyNKuTT6uerk+T4C6rfGCumYjNO1ik3ci/lCE4V7p6e4sGbnxKwrsd1H05a7V8Mm9TFY2wJAibaMRFWG0xmtrqLJ4T7cy+pg0TC/9x2t4aS7vqUOkp9tCRLrOREPa1jpEtK0oAqG32laK7t6HKNdUX8Q2ovvBBx+ksbERgN/85jc0Njby8ssvM2bMGCEqlwOoGPkw2kWfiZzuUFvh5YanG2ipryFb0UR3o5qNlwBuRRfp3RDdlx46nJJcD+45WRDwUVg2oNV8k71uj2K9V3eNs4wytkBDJXUthQAM9DSD3tUrX23kva8WcdDIY1OfeCQS9XTH5nQrwN6NUdEd8sNzJ0LDTqqPeBFwMsa9W2vrLUV3h/TIjX/AFO3/nYuj24yoiP77tPtRJSs//rXL0+3pWHG8BhZlsZ4hQp0LoiFtax3SttYhom1FEQi91bZSdPc+RLmm+iK2Ed0jR440f87NzeXJJ5/swdl0jVae7jTldCfPudH+jxie7rYKqQH+xj1meHkzXiIouPXe1N0R3UU5Hi4+ZDhsPwa2zoP+E1vNN9nr9ijVveeVagljAWq3UNusieACVSuOFXJ4cUX8qFXLgS6I7pYaiGj57Y3haPV1BTTPtpFGu/RVs5jb0G//CNzEoIgm1rvaQqq7iPTg75Ebf8W+2v+712gLT8Fmra0YQL9x7X7UCttaMabL6aCxcCyhpq4XLJS0j0jXmd3Z2xSg0R9iiN6+T9rWOkS0rSgCobfatj3RLXJOtwjHxypEuab6IrapNHD55Zd32Njc7hj3J8MLbSwSRj3dXTv5kxdS01637emOhmKHGmvM8PIW1UMT0eJh5KWhx/RZz8J1S7V+x3SvyueY/nkALPDpBa92fEttSxAXITwhLRIiOEDLzf1p/V9Q/34aNFanNl+9iFqDs4iP1+yJzlMBajZoL1QV5j5uvje4YTGnOr4iK6LbtWhoar8zTSTeQO1cLKRHqtLmV2h959WIVuHeCC0vHg7evHY/akV1Wqsq3vYvK2OZOjyugrmdzwURib3WpG1TZ111I2c/+TX7//ZDDn/gE77dshcQswq0KIhoW1m93Dq6Wr3c+F9kT7cIx8cqZCE1+2Kbs3LXrl2ccMIJDBkyhF/+8pcsXry44w/ZDNUscmaEl6O/tjCn27gntsrpjnq6w017yVGinu5m1Rvdrxue7phJgcOZdL7JXrfH2PJ8FAW+8Ome5G3fUNsUoJhGfTAHnkmnAjBQ2YOycQ5880xq89VDy7eHClr9GerejSzcvJfAujlQtYxm1cvLoaMAuNf9rLZjXgV4clL7nWlCpNX2HlltVRQYoHu7Kxd3OrRc+6gYnm7QKpjPj0yIy+tWFIWapgDfba01v3iEwuJ5KuyC9Ba0TW1zgGv/tYiv1rVd5PTpzzYwf1ON+RxcuFkT3SLdw0RDRNuKcp31VtvK8PLehyjXVF/ENqL7rbfeYufOndx+++1888037Lfffuyzzz787ne/Y9OmTT09vU7Rqnq5pTnd2v8h1Sik1nZOt9qylxx8ALTgpYVY0d21dmGdnW+y1+2R7XEyrCSH5epwrU9yUzXZzduj7cKyinBOv4Jf5D3AQ6EztW3LXm1dvb09dE93ZaQ4LvpAAXZtXsUPn/iCHW9oraheDR/B/aFz2aaWka/oLdaStHHKFCI9+Hvsxh+b193JImoglugeVprL/Mj4uOAZRVH4v5e+5fTHvuSJT9fz+dpdHHDvR9z0qngLmHZAfnFpm/8s2MY7S3by0Oy1Sd9XVZVP1+wCYMIAbXFze612/xTpHiYaItpWlOust9pWiu7ehyjXVF/ENqIboLi4mJ/85CfMmTOHzZs3c8kll/Diiy8yevTonp5ap1ATosijojt9nm4z/Mf8HcYPbfTpBhTfXrLQ2rE1q16a40R3WZfm1Nn5JnvdEeMq8vHjYXeeVk16dGBl1NOdUwoOJ6FBB/F06GRCDi/sWQc7v+v8L9BFd7VaBMTenKC4eRN/cj/B8OZlNCvZPBk6hRn7juMneY/yt363wLQL4ZjbUvp70kliyJSdb6g9L7qXREV3+cQOPyaS6B5elsM3kXFxnm5VhQWbNG/iA7NWc+lz31DbHOSt73bgD4WTDyRpE/nFpW2W76gDYOPupqTvr6psoLLeR7bbyQ/201KFtu+VottqRLStKNdZb7WtFN29D5kaZV9seTSCwSALFixg3rx5bNq0ifLy8p6eUqeIerrjb1TdLaSWPKe7A093jOh2+OvM8PIWvLSkO7w8ge7mHY+r0Dwja1zjAZiqrIt6unO0FmFj+ufRRDbL8g7Rti99tfO/YMMcALao/SkviOa315KPWwlzhvNLAH4TuJAdlPGzY0bz7o0ncPk1v4LTHoNhh6T096QTkR78PfZlyiimVr0iJry8d4nuYaW51JEXF6lR0xykJRgGVH7l+je3Op4HVPyhCEu31UU/vOo9+PqxtMzDoK4lyDtLdvSqcHZRxEBPsHyHVtRyV4OfBl+QzXua+GR1tLaG8fOMUaWM7JcLSE93JhDRtqJcZ73VtkkjKfXvbCIXUuvLYlPmdNsXW52Vn3zyCVdccQXl5eVccsklFBQU8M4777Bt27aenlqnSKxebmV4uUHE+G2JOd0x4eXuQJ1ZvbyFBE93nr3CywHGV2itm+YGtIr20xzrqHAZnu4SIFpw7T31UG37stdaV3BPIBJR+WD2+7Dpc0I4eC18BBMGFprvb1SGsiT/cAA+Dk/l5fBRjC3PY3xFgW1uXCI9+Hvsy1TxcMguhnBA61fvyoLSjqNlrJijdTndueR6nOZCH8DOOi2F5OgKP1e6/sulrvf58ShNHM3fVEN1g4+3vt2K+sZP4f1fQ/XKtMwF4L73VnLtv77ltUVi3Ks7gyhiINO0BMKs39Vovt6wq4lr/rWIS5/7xszxnrNaCy0/elw/BhVp9S/sKLqf+nQ9+971PqsrG3psDunETrbtLKJcZ73VttLT3fsQ5Zrqi9imZdigQYOoqanhhBNO4K9//SunnHIKXq+34w/aiERPt7FIaEUhNdPTTVue7mjYnydYTzZaa6xm1YtbiTnsGfB0d1V0/69uCDc6YaKyiUWueggTFd3lmuh+uW48t2QXoDTshO2LYMiBbY77z3mbyZ3zF3DCu+GD2eUoY2xe9Bw7cGQZ+/7iLdRtC3jo9SaobOGUfQemNHerEenB32M3fkWB0x6HFW8BKow9HpzuTnws/fO16ng5HQrThhazWnGAfn/ZoYvu4/I2QK223/c93/IMRzFvQw2zV1ZTvWUVp3k1Ic6uVZ3Kde8Mn6/VxNbKnQ3ajS9NXobvttYyojSHwkAVFA6O8+xbjfzikpxVlfVmcTSAxdtqWbZdO6f+u2QnkwYXmkXTjhrXn+Jcrd99bXOQRn8It43uYS99s5V6X4j/LdvJOP25IzIiPR8MRLnOeqttpejufYhyTfVFbOPpvuuuu9i5cydvvPEGZ511lnCCG6LVys2WYfp2Swqp6UOZ1ctjvbyhgNmHGiA7VE92THi5Wb1ccWgewTTT3bzjYaW5ZLkdrA+W0uIpwauEOITF2pt6ePmw0lzcToW6gAPfwOnae9sXRgfZ+g28cjE0VAHgD4V55ZNvOMXxNQB/C53EPoMK8bgSqq47nChDp/PQBQdx3cwxXH54tH+8Hehu6H4m6dFWMONPgjOfgjP/CpN+0KmPWDFfK9vM7De0CDXmnrKzVhPd09RV5rbxdZ8D8MW63SzcvJfxytboALtbF8Gas7qaP7y/ikik81+2Kut8phfz2DV3w58nQnNNSn9LMj5eVcXpj33JR8/dAQ9NgsX/7vaYqSBKK6NMs2JnfdzrN7/dbv784YpK3vx2O+GIyqh+uQwpySHP66IwW1v02r63xTatl3Y3+s2c9BU76jvYWwzsYttUEOU66622TbaPFN1iI0W3fbHNXeOKK66gqKiop6fRLaLh5UbLMP3iVzOc0x2ML26TE6lPHl6eUxbX6itddFcYOh0KY/rnAwqr3Fqrpwnh1dqb2Zqn2+10MKJMyxWszNW9dTu+jQ7y0V2w4k1Y+DwAry7cxtjGBbiVMKudY1iqjuTgkSVtFpwY2S+P62aOJduTfvt0B5FW20W78Yvk6QbYb1gxaswtvLFOC+kd2rjY3JZds5Lx3hqcEa2Q4rh2RLeqqvzy1SU89sl6vt6wh85ieDVdhDiwcTYYUSfd5K3vdgAq+1e/qW1Y8Va3x0wF0c7fTGHkc+d5tYipRVtqzfd2Nwb43Xta2sKPDh6mbdy+kF95XyUHH9trm21zD1ukn7cQ/ZtExy62TQVRrrPealuZ0937kDnd9qXvnpUWoKpGaA6gquaKW7R6edfGTR5err0OG9XLY1ckY/K5AfIijWTrortZjWkZZkFoeeJ8k73uDBP1NjN/a5wR/4bu6QaYpOdjLwrpX+500V1fV4O6da62rXoFoXCExz9ZzyjHDgCGTT6Me8+YxNVHjRbmgW8g0oNfZNuKILqnDSmOC7fObqmkkEZy6nQxrfcmf979O9ZkXcxFzg/ZP3tHdIA98aJ7TVUjuxq0+4RRabozGKJ7lLIDD0FtY/32dj7RMcFwhE9WVTNO2cpwZae2ccvc1l0aLES08zdTGAL1uInxBU69Lu3rhC8YoX++l3MPGqq98eY1nOd7mVtc/2L73hbmbYyPgugp2y6MEd3ba1uoaw72yDzSiUjPBwNRrrPealsZXt77EOWa6otI0Z1G4gqpqREz3Ly71cvbuymGjF8aW0gtGC+6i5SmuOrlZnh5nn1F9xl6m5lZ/snsUEuib8SI7oNHaT+/u0v/8rd7DWu27OSWPz6CYoTXV69kw+4mtte2MNapfXnPGjCBC6YPozDbLdzNSaQHv8i2FUF0F+a4zYgXgAFKDTPzNmkvSsdo7e2AipAmtK/Jnc1U705zf1/lal6Zv8V8/aVeBAuiRdk6w8ItmniZpGyKbuym6J6/sYZ6X4gTnfOjG321sHt1t8ZNBdHO30wQCkdYpYeXf3/KgLj3LjtshPnzlUeOIsvt1LoH7NI83xc4Z7N20RzO/9u8uM/1lG0XxIhugOU763pkHulEpOeDgSjXWW+1rRTdvQ9Rrqm+iBTdacQU3QoQCUf7dluR063/H04WXh4wwsu1fQppIjdGdG9VNEFreMLSTToeTtNHlDBpUAFhnLwUOib6Rk5UgM8YqYnuT3c6ieQPBFQWzZvDoep30f1r1rNhp+ZZGefS+nNTNibp3ES4OYn04BfZtiKIbgCnM3pPGajs4Zic9dqLoQfDvmfDgClERh9LxOGmPLCF4qYN5v5ZkWYefuszmvzaAtVX67WQcgcRjl56E7xxZXwETRJaAmGWb9fEymTnpugb3RTdH67QajGcpIvukKIXwtvydbfGTQXRzt9MsKWmGX8oQrbbyWGj++F2anZxOhSuPGIUI8pyGd0/j/On617uZa8BEMGBQ1E5t+rB1i01e8C2/lDYbKNnFO7sDXndIj0fDES5znqrbaXo7n2Ick31RaToTiPR6uXonm794rdEdGuvzfDy2EJqQT00NE/zAHuVIP0cWouX0w4czck/vAyu+gpm3tml+aQy32SvOzvGj3XPycvho4goem51TIuzISU5DCnJJhxR2VOoLSA0b1rAEc4l0YEiIfZsWYGLEAPCupevVIruTCCybUUR3a6Y3K0Bjhr2Dy/WXgydAbll8NPPcFz4Ko6RR5r7+Zz5VDo1L+VQdQfzNu4hFI4wT8/jPsixin3rPtYKlzVEPePJWLKtllBEpbzAywGeqNec+h1tf6gDVFXlwxVVjFK2M1bZRkB18rrjOO3NzZ0X3aFwhGe/2MjKnV0TU6Kdv5mgWk8/qCjMwuNyMLREawc2viKfwhw3H/7iCN77+eGal1tVTdG9asqvaFa9THRsZowj/pzqCdsu215HIByhNNfDCZMqgNYF4jrCjoJEpOeDgSjXWW+1bXuiW+ScbhGOj1WIck31RaToTiNmOLkCqOHoils3W4a1V0gt2F4htbx+RPT2YOWK5u2dPm4I358yCMr3AZc1FeITL/KuFrQ4efJAygu8VFHCN1N+CzPvgpL4auKGt3sF2vZjG99isLIbv+rGVzYJgODO5QxRduEiBO4cKBiUdK4iFN4Q6cEv2o1fRNHtcUXP2eGOaiqaVoHTA2OOi99x/Mnmj1mDJlMxcl8ARio7+WzNbpZur6NB93h/3xEjbHe1H85t5PdOG1zA6EjUi94d0b1kWx3ba1u4yv0uAF9E9uWtFm2+bJnb6XGe/2oTd7+zgpteXdLxzkkQ7fzNBLsbNdHdT2+1uG9JiB85P2D/QdkAuJyO6Dm58zvYuxFc2ahTf8RqdQgAB+TuihvzH3M3U9scyMwfoPPxqmoA9h9WzD56bZBUPN1zVlcz5tb/xVVutwMiPR8MRLnOeqttkzp19O9CdlxY6gizlpIA3+esQhZSsy9996y0AOP+5FAANWJpyzADU2rH5nQbhdTcuYQ82heK/qqer+nO6dIcUqG7LcMMPC4HD549lTOmDWLiCVfAYb9otc8MPa/743pNSA9RtC90H0b2Y0+B5v3O2ruGUYouAkpHx/UQFu3mlK4FjUwgSisYA6vma+XCTux4LkW/z0z6Qet6DeNOiv7cf4KZYjFK2cFna3fx2Rrt/jCqNIsTnN9E9929pt3fb7QKm5pbQ5Yakwde13Ux8vqibYxStnOm4zMA3im+gO8io7Vol7otULetwzECoQjPfLERgGU76jot6jbubmKu7vFPPB8q63x8vb7zVd27Q70vSDBsPy/Tbt3TXZav9d6+wfEy97if58qsjyAcgmeOg+dO0grerfyv9qGxxzOgfxnr1YEAnDigASXGtu8tq+KRj9dZNudQOEKjP9pCc0+jn+e/3ATA6dMGsc/AAn7i/C8X7XkIn69ztQw+WFFFKKLyr3lbOt45g/TWtlZ2oLfaVrYM632IspDVF7H/XUMgouHlihZeHjFahnXP0518JbJtT/fXq/W2QJ4c/G6tCrgp/D25XZpDKqRzRfjQ0WX8+Zyp5Ge5k74/Y2QZAC/tGs783KN5L3wQVwau4xfBa9jq0qqal7Wsj4rusrFtzk2Em5NIq+0i2zad87XSDnFjGz9M/2nrHfMrYPBB2s8D9jVF91jHNup37eDbT9/iDMfn3D5sOWVKjMevA0+3UeV8gqp5uVdG9FzeQAP4Ug/rDoQifPDdBm5x/Uvr+jD++xSMPoQmstnq1dNCDDHXDm8v3mEWg1NVmLuh477hVfU+Tnv0C857ei5rqxpaHbcb/vMd5z09lwWbut+DvD027GrksN9/zCXPze945wyzu1FbvCjTPd2DG7UogoFNK7QFmq3zYPOXULUUNn2pfWj0TIpz3FR5NE/3tOzquAKAKAqvLtxGSyBm4TgNqKrKe0t3cuQf5nDgbz9iXbWWYvX4nPU0BcJMGlTACftUMGDvN/za/W/Od87m/X8/3Kmxt+zRFrYXbdlr1kSwAyI9HwxEeU70VtvKnO7ehyjXVF9Eiu40klhIzYrq5cZKpJHTHYm9J0YirN/VyDsLNK+B35FFi6sgfjB3dpfmkAqZfDhVFGYxviIfv+ri7D1XcHXwOraUzySIi1WRwQCMVrcyRq9cLrroTlcUQSYQzbbCi24FTVgPnJZ851MfgaN+DVPOM+saHOZYxoKsq3jeeQ9/9jzBUStuB2CPqhWX6sjTvaNOE93Dgto955vIOBqVPO3NzoSY++rhk9/B/34F/72OuqdP5cPIFcx0fouqOOCY2zhtquYhfbbpUO0z8//abuswVVX562daQbmCLC295uv1u9vc3/jMLa8vpd4XQlVhzupdrY7V4q11AK1aXqWbP32whnpfiK/X7+lQiH61bjdH/uGTLnvg65qDvPj1JpoDnROORnh5WZ4X/I3RRZnqlVC1LLrjmvdh+0Lt5+GHoigKZxx7NAD5jRvjbNu/wEtdS5D/Lul6SkIyXlmwlav/uYjttS20BMO8v7ySqnofL87dDMAvjx+PIxJEefdG8zP7bnyGa16cz+V//yaumn8im/ZoKVyhiMp8ixdhUqG3CkM70FttK0V370O0tMm+hDwaaSRarVx7EclkTjeAGmFnrc/syb2hVqXZkRc/mDvznm6rL/qHzp3KgMIsQAtJ/+EBmthe5NMK5AxVqpnq0r5oxVYuT5yrCDcnkR78onyZMhBedBcMgmN/0/bO/cfDUTdrtRwGTotbgNqm9sPfb5L5+tHQ6doPu1a1+/t36OHlpc2ap3uVOpRK9LZ+9dtYtr2O2Sur+GrlVsKf/hHevEYTawbfPA2f3g/znoCFz9Gv6nPyFB97vYNQTnsc+k9g6pAixpXn85/goQRc+VCzAdZ91Oactta0sKaqEY/LwW0nT2QQu5i2/HdJw9KD4Qgvf7OFa/61yMzzBfhi3e4429b7QmaI8pJtte3apDss3VbHu0u1BcKICmuqGtrd/4lP17N5TzOvLeo45N5AVVXzi+mv31jK7W8t56GP1nbwKY040V25BHOpec962LYguuO8JyEShPyBUDwCgIGjpuiDrI2z7bETtaJ+/9TFcLr4YLlWAX9QkbbQPG9jDbOWVRIIRZg6pIgjxpRp89y9GnL70eIqZISjCufKN/loZTV/eD95lEcgFDHPe4Av17a/oJNJRHo+GIjynOittm0vp1vkQmoifJ+zCtHSJvsSffestADVXCUkLtw7mtPdtZM/+Uqk9tqsXg6ghtnT5CdHF90rdoeooTj6vjsX8su7NIeuzjfZ63QzvqKAN64+lOP3KeeGY8cyTm8Bs3SvhxZ3MQ5FZXREy+9sT3SLcHMS6cEvsm2FFN2n/BmGHdK5D3py4Jr5VF+3nYuHzGLJWZ/jveZLuGY+t/d7mFfCR2n7Ne2C5uSePF8wbIYb5zRsAmCjWsG2sHbPee/LRXz/kS/4ywsvMeKlI3B+cg989w+tKrr+ef/aTwFoGXk8H/S7hJuCV3Bm4DfsvnQuTD3P/BvPOXAIzWTxjut72i+f/1Sbf9qG3ZqoH1Gay8yJ5dztfp7TA+/S8tHvCYUj/Hv+Fl6cuxlVVbnjreXc/NpS3luqtRP8wX7agt38jTVxtjWqdgMs225da6k/fBAv9FZVtv276n1BM/98bXVjm/vF0hIIM/PBTzn7qa9ZubOe95ZpAv+/i3eY6VDtscvI6c7zwI5vo2+oYVjxZvR1s+55H3ZI9GFVMgIcLgg0xtn2e+P743E6WLytrsNFhlRYVamN9ZMjtCKbCzfV8MEK7TifOKlCm8PyN7Sdj/41WUf8HIBbC2cBsHxHHb5g60iD7bUtcRFmX7TjEc80Ij0fDER5TvRW20pPd+9DlGuqLyJFdxqJVisH1HDU062mL7w80dMdSvB072kMkKP35N4bdPGrqqN5LnQ8c0bdBNfMBW9+l+bQ1fkme20FFYVZPPWjA/jpkaMYXqp587fVtvBG/2toVvUq7YoDSka1OTcRbk4izNFAZNsKKbpTHVtR6F+Ux99/PIOTJmveRvqNo6lsKk1k0+jVF+jaCDE3vH2FHhVHnVZQapsygB0RTXSvXrOKHHw8lf0oA5QaAqoW6s36j1m+o44jf/8Boc1aNfLTVx7NT7Yex6uRo7ngB2cxpiI+LeaMaYPwOB38ue5IrXbGuo945aOv+GzNLti5OFo8Eq0YGsDwshxKmjfyPacmDutWzeH7j3zBLa8v5fY3l3HhM/P493xt3lceOYp/XT6dP/5wX8ryPLQEw4RinDxV9VHRvb22hT2N0dedpbLOx8LNe9t8vyUQ5vO1WiHImRM026/c2bYInbN6F8Gw9mxZX93YqS/I8zbuYf2uJr7ZtJdznvrajM7aWefj261tz83AzOnO98aLboDGqtYfGH5o9Gen2/R6x/bqLsr1csBw7Zz5bktth3PoDPW+oFnk77SpAynMdtMUCPPlOm0x4HsT+kMoAFXLtQ+MPBrlwMvB4aa8ZT0H5O0hGFZZvLX1fDbroeXlBdpzZVVlgxkB0NP0VmFoB3qrbaXo7n2Ick31RaToTiNmWIvu6Y7mdFtQSE3/P0LMBRXRPN3ZaAWEmvGyJjyAB50/puDIq6FoaJd+f6r0dN5xRYHWQzYYVnl41zSO8j/IulEXw/G/0zx8MYgWhiNSOLwoVWkNrJqvlXawYuwKPVWj0qPfL9ooprajVrvPTMtvQFHD4M4hr2wIO1UtvHyAo4Y3x75PRaSaKkd/fhS4BQB146f84p/zGdi8mlzFz141j02OoZQXeHn8gv35wf6DW/2u4lwPx4zvz1a1nO35WvuwJZ+8wocvPwJPHQEf32Puu8kU3bnw9WPRvyu4ld2V2yjIcuFQMAXYFYeP4FcnjueQ0WUoisKho7XijKGY75vVDdHq52OUbWxYnnqRs2v/tYgfPPFVm/nXa6sbUFXiekfv2boawkFzH38ozI+emcePn/8mrl1Voz8UtzDQFrEF5ep9Wrj82HItBemdJe33ZFdVlV2xLcMM0R3bxlFxxLerG3ZY/CB6pJEj5lbrcDiYNEjrsrFsR127c6is83HeX+dy4L0fmf+O+dOcVh7yNbqXe2BhFkVz/8CHzp8zEM0jPbQkh1H98mDXSgj7IasIiodDdpG5SHBu0QrOcHzOvv/er1Vv+C012gLPvoOL+F7/Bi50fsiHS6PHYmtNM7e+sZRvt7S9iLFyZz13vLXMLMiWLnprhW07kEnbhiMqz36xkT99sLpbwldWL++bSNFtX+x7hxOQ6MWO1jIsTYXUkud0a68jsYdQjVDTFCAb7QviCdNG8dpVh7Dw9mPZb2hMmLnFZDqnOxGHQ2FoiSauq+r97FaKcZ90Hxx8Vat9RRKxINbNVKS5Qh/0dLeBUR9hk6JVm070dC/dVseX63azvVYTDFNy9PDakpE8csH+7D9Za9V3Zt5yxm55GYBthz/AfHUce9QClEATRTWL+V6OVnytePyRrL73ZOb9eqYpNpNx7ETN+/thUMsNPtrxHd8Pvq+9uXWeud9GXciMz/PD4pcACLg0YfmbqXXM+eXR/OW8aXhcDqYOKeLG48fF/Z5DR5XRj1pCoWhocaUeVp3tCPIfz2+Y8sHZbYbdJ8MfCvOd7jX957zkucurdaE4riKfCQPyud71Co/suhT1/V+b+7y6cBufr93N7FXVZg660Rd7bXW88Kyu93HTq4tZtj0qZI1w9EmDCvAS4MChhfzy+PEAvLd0Jws21bC3SXt+NPpD/GfBVmr01w3+EAHd/d/P7Yc9epuvqedHf2nJKBh7vPZzbr9W6TzG61hPt6Io7DNQi2xYur1t0d0SCHPFCwv4esMedjX4zX8bdjXx2CfxLcdWxtiSxf+mf2gn57jmAJqXW1GU6KLBwKnREPixJwBwePArfu3+F9nB2viweWDTbu38GlaSw53uF/it+znWffoPIhGVJdtqOePxr/jnvC3c9d8Vrf4GXzDMRyuqOOuJr3jh6808++XGNv/ertBbvbHppKtCMpltF2+tNRfRqut9/Pzf37aKZvlq/e64a7Ajquq1haW731nBIx+v61Y6i8zp7puI5kzqS/Tds9IC4sLLI+G0FVJLGv5DEk+3Go4LLx87uJz9hxWbX8oyhR0e/MN00e12Kvz5nKkMK01eQE4KQ+sQaa4gRbdBRaFWeGp1WKsabobgAs2BEOc/PZcfPTOPL3RP8Ri3XoCsZCSj++dz1IFTAfC06NsP+gn7HXU6kwcX83lEK9Z2lGspFw7Qi3/FhiC3wzHj++NQ4OW6iQAc5ljKdIde6K0mKl4MT/e+vgWaJ7N8Mp79NGF4csEGSnI9fH/fgXxz60xevXIGXpcTWvbC6v9B0Mcx3hV87v0/CtToF+VK3Yv8g6HNFClNeMLNsPGzTs0bYF11IyE99OmD5VWmkI3F8NaOLc9n3Jqn+LnrTQDU7/4NQR+BUITHP9Gqsjv1Vdd++V6OGNPP/B2x/H7WKl5ZsI37/rcS0ES0IWqfOd7LyuzLeWHgaxwxtox8r4uqej9nPfk1h93/MR+vquLS5+bzy1eX8NBH2qKL0aM7z+sia7deqbxwKAw/IvpLKyZpfeJHHg1H/SoqZg304n2Jonta7m6OcCxm5c56Qm30J//V60tYur2O4hw3L//kYN77+eE89aP9AW3BIDbEe9VOTajsU+41C+id7JgLqHxvvJ42YYruadFfoi8YlNctpp+i2Uqtji8muKVGO7+GleYwOKSNPbhxGfe8u4JznpprzmPx1lrzXNxa08zJf/mc8bfP4vIXFtCkV6U3UiHShR2evamSqeeEqqo89sk6pvzmA95enHql/MS5BcMqFz4zjwufmcfmPU08Pmc9by/ewc2vLTHrI6zf1ciFf5vHRc/O71TNBIB7310ZVxF/5c7Mi27p6RYb0b579SWk6E4jrT3d1uV0G0OF4zzdKnuaAmb1ctzxodSZwg4P/gtnDOOgESX8/bKDOG3qoDb3E+3mJNJ8RZorSNFtUFGgebo/bR6ubdjwidlz+eNV1TT4Q0RU+J9eZXsYelhyqV4voSAmPHz44XD871AUhWcuPpDBB3wfgMvLVlC0S28pNaxzors418MBw0tYpQ5hh1qCV4lpc9VSA746AqEI2/ZqnsiKgC7EhxwU/R2bvtTE9Wd/pHDZ33HtWQNNe+CZ4+Hf58KjB1L230vIUoK4iIo/o5DaqQOiQjyybna7811b1cBJD3/OfxZsZVVMbnYgHOH1JNXGV1dpovmAvD245twLgE914wg0wNoPeOPbbWyvbaEsz8ubVx/KjJGl3HjcWMZVaF78WNFdVe/jv7qwmL+xhkZ/iAWbaghHVIaW5FC+/SMcaojsxc/jrd/ML08Yx8h+ufTL99IUCHPZ8wv4ZpPmsVuyTfuboz26PVqLMICKydB/QvSPKJ8E2cVw0Ztw4OWtjWKK7qhtlcqlDPnPSbzguZ99QivZkESILttex1vf7cDpUHjywv2ZPrKUiQMLOH6fCqYMKSIYVnllwVaC4QjhiGpGDeyXV4tRYX20YwcnlO3hoBEl2qA7vtP+HzA1+otKRkJZfORDKEF0b9YjKYYVe3E0aGHlUxzree7LTbQEwxw+psz8HW8v3sG66gZ++OTXLN+hiSe3U2G6/r7Reixd2OHZmyqZeE5EIiq/ePk7/vD+aup9oaTXX70vyONz1rFhV2Orz4Yjaqu5balppsEXIhxReX3Rdt7T74frqhvNKJT/Lt5BRIWapoDZXrEjjIiY8XpB2BVSdKeEFN3ifffqS0jRnUbiC6llJqdbjfN0a+HlRvXyxPzlTGGHB//R4/rzyk9ncMiosnb3E+3mJNJ8RZorSNFtMKBIE93zWwbyr9DR2sa3fwbBFt6Nyfsdr65nAHsoD+r5rEaRwuLhUDoa+u8DZ7+gFdBC88oecMwPAPDsXQP+evAWaMKtkxw3sRxQ+CQ8rfWbNRvZUtNMRIVcj5OcOr0NVv8JUdFdvVwT1x/fA+/eAI9PhycO0dpGAdRtgWAzfld+nJM2FNHCuPf1Rv/+PYv/xyb9C/ryHXU8MGtVXF/te95dyYqd9TwxZ73prSrJ9QDwr3lbaPLH98Y28pD3QQuV3pA9iefDWrgzS//DM19oiwhXHjmSyYML+fdPDuacA4cyur8mumMrmD//1SazyFowrPLlut1mPvfBI0ui7b3UCHz5Fy6aMZyPbziKz286Wrdx1Ju+tqqBSESNbxdWt1X7fPEwyCrQjjnEC9hk6AJdifkyr8y6GSWgzf0c55ykobjPfbkJgJMnD2D6yNK49y6crtUeePTjdexz5/uc/JfPzcrlYz3xlcWfmLoJT91G8NVFIzgS+9rr3u5mRY+WaqrU+smjCTAjp3tkVj1EtGM4UdmMixA/3H8wz15yIGcfoKVmvDR/C2c/NZfKeh9j+ufx+U1Hs+a3J/Lwudrv3La3hWAbnv2uYIdnb6qk+x62o7aFP32wmnpftBbCnDXVvPld1Lu9aPPeOM9zOKJyzT8X8cCs1XGt4vyhMMf++VO+/8gX1LdEx4P46+3pzzfEdTh46rP1qKpqLnwBrN/V8QJLcyBknl9nTNOcBe11MOgIKbr7JqJ99+pLSNGdRpLndKsxLcPSKbrNLVHhHQmzu9FPtqIVOMpET+5kiPTgF+3mJNJ8RZorSNFtUJbn5dqjR3Pg8GJ+Hz6fSrUYatYT+OQPpgfnNMcXvOO9jVe9d1HUpPXopnS09r/LA9d8A1d+Djkl8YPnV8Axt8GQg2HoDDj+XnA4Oz03o6r3x5GpAPhVF5Xe4dqbezea4bzDSnNRDA9lv/GQ1y/qwXR6YdJZWgg0QGMl5JTCT+bAzLtgxrXsPf89Eq05ojSXrJpofnu/yC5+/cwb1LUE+ckLC3l8znpe+HoToHmXP1ujVSLfsLuJ2brdrj5qFIXZbjbsbuKiZ+ebwqCuOUhlvXbfHuTXQsibiifydngGAOqa99lZpY1x1v6DQVW18PbdaxndT/OIrddFQE1TwOx5bQjy2Sur+FSfz8EjSmD7wugf9t0/oU5bOMlyO3n8gv144Af78vpVh+B2KjQFwmyvbYlpFxYjugv1qIZTHoajb4VRxyQ5ajF486FkZNyChhLyQT9NjJ/snMvqLZVxH6luiHrtLztsRKshT5kykOIcN82BMIFQhFWVDTT6Q7idCgPCuujxaDZSvngQHtkP/jxJ6yOeXdK6wOgBl0HFvnwz7pdUqUUAhKpX63Px4w9FcDoUKiLRvu5ZSpC/n5zPA2fti9vp4Ph9yvG6HOyo81HTFGDyoEJe/ukMhpTkoCgK/fO9ZLkdhCMq2/Z2zgPaGUR69hqk+x52/6xVPPLxOv72eTTlxMizPnPaILLcDup9IbO1IMAD76/ic73feqyYXlPZyPpdTazcWc9Nry2J+z3rqqMiullfbDtybD88TgffbNrL43PWxwntRA96MoxoldJcj1nQcVVlQ1ry0KXo7juI9t2rLyFFdxoxCjhEPd1qXO4arb7GdY7khdRixtK/tAZCIRp8Idt5uu1c0CJ2rnaep4FIN1OR5gpSdMdy4/Hj+M+VhzB26GDuCl4MgGPeYxSE9nBq8Sb+6PkrAIOUPXiadZFUGtOOz+FoW0wf8Uv48ftw2SzY76KU5jW8LJezDxhMaORM1o24kDtCl7IKXYjVbDTDdceVOKBWE55m+PMJv4P9LoarvoKzntFCoK/8Ag69Di55V/N4Hvb/7Z13eFRV2sB/dya9B0ISIIHQe+9NLCjYEfsiiGsXK/vZ1lV314Jt1bViw+6CvaEoHZWmdKT3GkJLJ3Xu98eZmgIhmZvkZN7f8+TJzJ17z5x5z73n3Pe+7R4Y+QTJrbtTbAv1fLFh0CohUmW8BhwhKvFX+5xlXDFlsbs81cw/0zFNk+e8rGXgid3tl9aIjy9vSruwTJbvOsbIFxby6e973C6kzePCCTmsviMmrSfrzZZsozlGaSFn2/6gY3I0cREhKiv7+xfCK33p8u25vBz8EmMLprE9/RjXvL2U7IISWjeJ5KHz1W//bPleNhzIJjTIxvCEbCjIhKAwaN4HSovg1QHw8z+guIAgu40r+qXSIzVOZflGxZu7Ld3RIZDpUrqdyfZanw7D71PjfjKSu/mshIYBXPwKuRGpRBqFRO/4EVAWvomfrGD8O8soKnXQu0UcPVPjyjUXFmxnZo9f+TV1Ch+O70p8hPKsaNMkCvsxp+LV8y/K7V19o/KyAN8kai4atYJbfqHvJXey01APFVavVKXtfnTWNW8eF05Qtq+L8pDwXe5rMDosmJFdVFLA/q0a8cmNA9xeDqCSfbpKW/rTxTzQlW7TNN3JAtfszXRvX71HeU/0TWtE95Q4AFbsUp+v25fFGwu2u/fdfSSfUqcVfIOXlXnpDt8EaZudCnJYsOecnzAkjSv6qXPm2TJzwLYqKN2bnSEm7ZKiaJsYhd1mkOn1QO5Uqa7SLYnU9EYSqdVfAvestADXEzZXyTDT9LJyQ/nFvYpU/CTSewc1jJl5amJ2JVKrq5juui4ZdiroNjnp9JBAl1IwLqRkWHlGdU1mpqMfm4I7EVRawCshL/Nc4eMEU8Je0yt0IyRaZauuBZ65rAfv3zCYghFPMr30DDYVOftxbIdbue0doay6RCRApPPztiPgopcgoa2nseRucPa/fOOSXYREuV8ahkHHxkFwTCnytr7qQcRw22o2eZWrWrk7k8/+2MuynUcJCbJxzcAWnGtbyqyQe+lq20H7RkF0/fZ8frLdzaSoWaRn5XPfF2u4c5pK6tUhOdrt9ty8Q1+iw4L5uaQ3AP1sm1Ss8J5lMPtRZ8ds2DL+5EL7EiYFf86LLz3L+gPZJESF8MY1fRjUujHhwXZ3JY3HRnelceZa9aZpD7joZeUJUJQDi16Gz//qU6KsgzOudJOX0t0kKsydnIy41JOOVzmSu/mWDAsKhaY9ON7lKgD6HvuRn/9MZ/w7y5ix5oDbVfyGYa0rag0yd5O08iVSDi1kWNEi3v9rf7o1j+XawWlw1KlMNe2uPBmunw0P7oGhkyAsFrpfWWk3I0ODCG+mkvZtXLuch79ex7+cGclH92wGmbt9D9i/wuftvy7qwstX9+KDv/YnOiy4XPtupdvCZGqBNu/uOXrcXTrPFUPvcJisdirgPVJj3ZVcXNbv9xftBFToQojdRlGpg/3Oh2iu3ABtE6PKeSpudlq6bxiqzsu4iGCGtEng4Qs6c373pu79zuig5sVtGeXHedHWw8zb6PGYcCVT7JAUTViwndYJ6hzxzglxKkjJsMBEN4NHIFH/Z2SN8HEvd5Rigq+l24KYbrVRWZRcSnek4cyMG8CJ1KqKbpOTTv3Vqa8glu6KUCW8DB7KvRyAAbaNhDjyyW8+hEtKnuKI4XQfb9y62g8Vq0ua64a00KlUe1u67c4484qU6SoSFu6ZPy/o3Jjx7QsAUynyPa8BYLh9LSnGIdonRdErNZY3gp+n+/fnEslxxg9syV+6RjM5+G3a2fYxPvJ3wnNUPLHNUcSdJe8yP+VN4oMK3a7bPRqVKHd3ICi5C0PbJrDCocps9bZtYWCLKKUYO0qgyxj4v61w2btsilEx60Nsf9IsNoxPbhxIO+eN+/D26qb/8j4pKtbYFc/dvC8kdYHblsAVHyq3+00z4MuboFBZ3NonKaV7c3oOh5y1yhMjcPfRbek+FZJ7+J63iZ3BHkzjwcrrob+xngc/nEtGTiHtEqN47vIevHtdP86trJzcig9xJUtj7Wd0Tw7nu/5/cnWr4x6lu1EbFXee2k+5uI94FO7fBT2uOmFX23dR2dETC3fxodNl//qhrbjn7PYepbupKmHHvpU+x8Zv/B8XzjqDsIO+2120TFDnl5VKd6DNu95Zvw/lFJKRU8COI3nkFJQQFmyjQ1I0fVoqpXvF7mMcyytyZzL/69A0WjZWY+J6eOdSuv86pBVNokO9v4ojeUUYBtxyehv+eWFnXh/bh5AgG6FBdl6+qhd3ndWOi3o049bT1UM+b3d2gF1H8hg3dRnXvfc7czceBDxKdzvnddexqfKocVncdx/J55W5W6qc9V7cywMT3eaAQCKorjvQkCifSK2spdsqpVu9dind4ThdkeqJe3l9vuh1m5x06q9OfQVRuisiJT6C7imx/LG3I1+XDma0fREMvpOIsx7l48PHCdmYAfP+rhS4WiYqNIiEqFB25ak4b8eR7azOVm6kLUudruVNOlR2+Emx2T3L4/iOJo1ynQpcYidI7AithmPbsYDHm/5Gk8su4cDiTxlxSCm054as5tbTL6TRgocwDHWD3CnkIBxR8dpEJEBRLi0PL+TXJoc478jd7CqKpn+EM1FbfCsIjeK09k14bp1SutsZ+0guXqziqSMTVRx1WAx0HUOHkCj45DcubbydMXeeQbDds9Y8fkYst4bPotMFznjrfU6lO0UplBgGdL4I7B/A9LHw55dKMW/Wk3Hp21ljO5uN6WcSFqwe7qbYnYpNULiKhT9VyrqXN1OJ9GzxLShN7oE9fTWn21YzO/Qs3r62b6XlHgEoLYGVH3neb58P390Nqz9RmdJdseeNKrCSV+F6CWuqHtr0Cs9gbK/m9EmN4ZK+rdS15gpf6DwaDqyGjPWwazG0HAS5h+Cnh5QHwdI3IKX89dGqcSSRHCfj0EGg60n7UlUCed79fcdRn/d/7s/mqDPrfrfmsQTZbfRuEQeo2O0pC7ZRWOKgc9MYereIJy0hki0Zuew4nMdp7Zu4vSxUpvym+PoyGLRsFEFUaBAThrTy+cRmM9SDGXDnbTiYXchXK/fyn5838/fzOjFvY4bbjf1vn67mh7uGscXpXu7yMOnUNJoFq/PYsu8I6/cnMu6dpRzJK+LF2Vu4dnAaD53XCZvTbcThMFm64yi9WsS5r1VRugMT3eaAQEKUbj9SNpGaw/SPpbsi9x9v9zzTsGMAWXmFGDgIw2Xprh+J1Oqzi5vOrnj1fTLVbeIXpbtiLuuTwpq9WSzpPpnR56VCpFK02idFQ+JtkNZbWUzrgLTGEezMVUq3kbOfosLjdGramMSCnWqHJh2r3baPbI9sg9gw3zYH3wE7FnB67o8Q9zTt977h3v/6xI00zt8Bf0x1b0t17IMjKjM57c5WJbX+dxWRxzbwbZf5fNPiAfqbM9TnTnkOa5fAg8Syy5FIS1sGsUufU593ukAp3C5aDgLDTlDWbsjZ60kOVlJEwjfXkHBoAySHwYCbId3pXl72QUmHUXDNl/DN7SqLe9ZuYoCXgjdx7eE4tocrxTDJ4XTdj0utnndDdDKGisFSsk3u4f7I3n4kpK/mztRt3HrpPypXuE0TcjNg/deQs18lRItpDgfXKoUb4LAz6V1wJEQlnno/wT3WCUX7eGLLaEhvCr3mQ1CoR+luMUhlbT+wCt4dpbwgTIdSuAE2/wQlRbDkVdj5K+QcBEcxo4/ncUXoXhx7DTj4q9+uoUCed393WrobR4ZwJK+I9fuzOeiMh+7hjOVuHBnCFbEbCc3dzVsLRwA2xg9qieHK24CydB/OLeRwbiGGAe2TojivezKTy3xfx2TnNbjK+ZAnpS8c2wnrv4V+10NIJDFhwSRHBTHi+I+8/VU6e4uSuXvaKkqd94upjcLZc/Q4t3y43J0bon1iNGyZzdXrn+GW0GWs3tSWSzb8m8ISk/iIYI7lF/POrzvolxbPqK7Klf2F2Zt5ee5Wbj+jLf83skOVZSsx3Q0P3UL7AgkZDT/isXQbYJY6Ld1ek5Y/63T7WLpVu1n5hR6FG8TSXQUC+QbFanTqK4jSXRnXDGjJrHtOY/JlPdwKt9eXKoXPWwGsRdISIjlMDIVGOAYmqUYG943sgHHImbm8Bu7lPrI9utVTmzrRqXS3OUtlRC/KgdeHEHxsCyWGeo7dMWcJzHsCzFJKminlNrZgH7j61aiNukEfoxLSxe76mfH9U7BnqJhhkpSCmxIfQesmkawwlbXbrbR3OM+3s6HR0FzFfrPjF8/2xa+4k7+x9jPYOEO5pseklM/aDdB6ONy2GEZOhnMex2w3klCjmNfsz2LL2U9YsI0Um7MMlytz+aliGBg2z/N+o6mXlddZrqtl5lLaNg4rf2xRPkwfB8+2gf+0hx/vU9t7/sXXVdwrHp9GNQh9iEpUsd+YcPyYsmZvnKEs7M6M78S3hHFfqSR9AKs+8ij+QeFQmKWS1M3+J2ydrR4MHNpIWO4ebIZJEA5KN8+qXv8qIFDn3UM5hWw/nIdhwFX9VdjDun1ZrHbWve6RGgeHNsFrg3im8N88FvwelwYvokdKLBf3VOW5WiVEMtC2npEbH2LHNvXQpkWjCCJCguiZWqYSg2HQqWmMyrHw9a3w0aUqLOOLG2HWwzDzAfeut0XM4vHgd3nQfAeAImc9+dPaN+HDvw4gOiyI1XuVl05STCixYXb48gbiM5ZiM0x62bbQtHQ/fVvGM//eM5gwOA2AH9aqMI8juYXukoKzNxw8JdmKpbvhodscEEiI0u1HfC3dJphY5l7u/h7AHdOdX+DJXA5qwa8DROm2Dp36q1NfQZTuyrDZDNolRdfLMUxrHAEYbCtVlswRSfmcnuZlhWziJ6X7wGpVogs8dZ1tNpWxG5S1FQg66x8QkYBRmA0bvlXbLnwBgiMxzFLYNlft78r0njYMwuIg/wjsWQIH16ntXlbPq/qlstIZ1w0oy23asPIddm3b6VS6j+2EBc94Pj+4DuY9qV73Glu5IhoaBYNug8F3YFz+LjvsrWhk5HJJyFLeGNeXiHynC3x14rmdGHZPYjEjPM7zQbPeyvW+MBt2Ly5/4Ir3lVzzjwCGenjR9TIYchd0vVTJJjIRxn2tPgeVb6DaHTWgw/mq3dSBnj7k7AezFOwhEJWsSuNd9BJcP8v9wIT253oeBCxzekF0uwLGfg7Xfo854QfeclwIwP5Ny8ioZobq8l0OzHnXlbW8Q1I0g9skMMS2lp7bXmfjgUwAlfn+93c8D6GAZ/tk883tQwkPUfdQXdjBO8HPMih/HrbVHwPQMTkaCnOwf3VD2Z7Tq0Wc5zwtyIRv74C9y9T7FR+ocIOSIi46/o1q37aLCYNa0j+tESF2G387uz1pCZG8fHUvt/di+6RoNX8dPwb2EIqT1cO0/52Zz6c3DyI2PJgLezQDYO7GDApLSnlz4XZ32bKN6TkccSY9LCvbUodJepbveSZKd8NDtzkgkAhYpXvfvn1cc801NG7cmPDwcLp168Yff/xRozbdbi1u93LrYrrV9xiujQDk5BcS7p25vI7cSiR7uXXo5A6vm4uTZC/Xj/O6NaVjcjTpduVi+dfWmRh/fq0+bNyuvGX+FPCRbV46lBQohbtZb89O3S6De/6Eq6fBmLdh8J3QfpTn8/bnqszZLiU7/4izb84M6vZg6HCuer3wOY/rd9Pu7iZuHNaah26+1tNm2zMhuAIrcCun0r3jF/XQ95fnoeS4UsZdfTq6DTCg17iqCSEkkoJ2FwBwQ9sclZTNFSddnczlTmwhnv77nLc2m3K9B+WWDSq+fN6TkHcEFr+mto34J/x9P9y5QpV/i0qEmKbKSn/rbyphmus3J1Q/rh+A0a/B/TudXgmGiht3PYCJTfVdZ1P7w00LYMIPcPm70OlCz2fh8XDes+r3tRqGkTaEXdHqXCrYvYqznl9AflFJzfqKfnOCv/r7xQqVUf+Mjol0aRrNs8FvcLP5GaebfzCkbWNS4sM99eldWeu9H+zkpNNlwY1EOu+hDGd4QofkGFj/DbZNM3y+75WxvRnWLsGTmBBUPgTwhPZ9dxcsfoW4YpWhPN7I5bZ+MXxy4wCW/P0sZX0HTu+QyCMXdMZmwBkdEiHdWRM8sTPBXdQ51PTwEnf8dq/UOJJiQsktLOF/S3fzwWL1kDHC+fBgyXblZl9Wtq/P38rAyXP4auVen+1lX4vSrTeidNdf6v+MbAHHjh1jyJAhBAcH8+OPP7J+/Xr+85//EB8fX6N2fRKpOUqdMd3WuJeDJ67bdFq6s/ILiXQlUaujzOUglm4r0am/OvUVxNKtI62bRDHz7tM4c/RfAUja9LEnjrrXNTVqu0KvokG3l7cQx6Yoxbn75ao2eUcv1+/T7lX/E9r5HuOd2KujUmrZPk9ZTztfrDJte/UjtHl3z5xe1rXcRepAlYE8e69KLrZ6mtp+xkPQ7XLPfu3OPiWFuVNvpcw3znHWHXYp3TWydHtqVpc7b13K8sqPlEv/R5fCgqfhtYEq1jwiAQbcUnH4VHxLT/z2hf+F0/8OA2+tdj+dHYSgENV2mzPUtnnO6N6KXPTtQZA2BILDnZ4MsWr7afeBt1UfGDJUJbdrbeynuCCPfceO16yv6Dcn+KO/+zOPs2CzyjVwRd9U4gr308xQiufFsVt5Y1xfjNIijzI78DbAUA+hcp0lu5ZOwZ53kDxTZSkPy1aJEzsmR8PB9eUu+3O6NFX93edMr+Z9fzfuK3WeHt4Ec/7lc1xiwXaC7Dafuu0AE4a0Yu0/R/LXoa3ggLOfyd2gjTMB4o6F7nJ+NpvBKGcd+H9+t57jxaX0ahGnqhMAi7apEJCysv3DWSbtxdlb3EncTmTp1jmmW4dz3yp0mwMCiYBUup9++mlSU1N599136d+/P61ateKcc86hTZs2lR5TWFhIdna2+y8np3zdxLKJ1Ew/uZdXZo11v3a2m5NfSGNDlZZw16atAySRmnXoNJnq1FcQpVtruo5Rccp5GSqhlWGHHlfXqEkf2YJqv/PFJz+w7QilSA++w5MhvLGX0h3dVLlwu2hzpicUKCIBzn++fJv2IKXAtz0bOl1U8feGRHgUzG9vh9JCZZVvMVA9FHAp7b2vrfj4ykh2Wt0Pb1Yx1Zl+ULpPdN52vEApGwWZ8PYI9R/U2IJKBhdchdCp6CQ4/X7l+u0v+kxQ/7OdlkKvhyMVEhQCl7wJw+9XifPKcO6gnhCZiN0w6WjscZeOqwm6zQkV9beguPSU2vjsj72YJgxs3UglQ9u1yP3ZyMgtRIUGqfCK0iKVdK9pD08Ix+4l4HDAms8A+Dz8CgDSzP3EhtkY0KoRHFxHWUkahgE56epcMGwqxAHU+dtiAEz4HloMVttCYzFbqrJ+ZGz0bcg0YcN3cGgzkaHOXAduj5ce6vqLaKxyR3hZ1V0J1ADaNIlkyjV9GNxGefYs3nbER56u1y7X8l1H8vn5z/QK9wHPvZDOlm4d7uesQjcPzkAiIM/Kb7/9lr59+3L55ZeTmJhIr169eOutt054zOTJk4mNjXX/de7cudw+Jy8ZVr2T/+SWbjWM2ccLSSBLbYxsUq3v8gdi6bYOnfqrU19BlG6tsQerOGQX7UcppasGlJPt8PvU95yMoFC46mM453HPNm9Lt8u13EVIhHJTN+xw4YuVPzAdNgmu+dxXYS/L8Pt8leHBTst8SCRc+o5yy67MUl4Z0ckqTtp0KGUg25lArAbu5Sc8b+1BcMF/AQOKctX/i1+DqCSlfPS9vtrfW2M6XQSjX1eW9h5Xw6CJJz+mwyg44+9KAa+IZFUyrYttJ4dyPUr3jDUHuPiVX92xylVFtzmhbH8/X76Xzo/M9HGBPhHHi0r59A/1IOiqfk7PAy+l2354oyrh5rJIN++jrokWzhj93Utg9yKlPIfGsCblKopNOxFGIZ/+JY3GUaGQUd7SbRiGRwlu0gnOfBgufx8uflVtS+wE1/0A134H1/+M4fo+r5hyANZ9AdOvgffOh+OZapvLIp/cXYUvtHZ6WGz5yX1Y/1aNGNi6EX1axjPtpkEkxYQxoHVjbAZsP5xHelZBOdke8IrnnrJgG6ZpSkx3A0S3OSCQCEile/v27bz++uu0a9eOn376iVtvvZU777yT999/v9JjHnzwQbKystx/69evL7ePr9JdiumjdBsWKN1lLN3Hi2hiOJXu6pZI8QOidFuHTv3Vqa8gSrf29B7vceXtPb7GzfnI9povoc8pWoi9adym4tcuLngB/rbJNwa4OoREwrlPq9dxLaGTl2W+43kw9J5Tz/VhGJ4Y8y0/KWuhYVMW+2py0vM2pQ/0v0m97n+jSvx2x3K4Y0WN4vRrjGGoTOnnPg2XTCkfNlAdnLLtbOzysXS/PHcLq/dmMX7qMmatP+hziMNhustLle+iXnNC2f7+uuUQDhOe+2kzxaUOsvKLOZZXVOGx+zKPc9mURezLPE58RDCjuiqXa3b9pv67suTv/MUTz93c6X3SYpD6v3sRrPlUve58Edef2Z3MMJXNvIP9gHI/zztUsaXb3WZvFVrSZbRvCIFhQKvTVMUDV1LHjI1K0Z/7BGQfgNlO9/O8DJj7mHpAkHMAMDzWeGdWf357CZZMgR/uw/75BKadXcwXtw6mSbRyiY8ND6ZLMzUH/r7zqI9si0tNso4r9/Rgu8HqvVks3XG0wvNFlG69qekc4HCYzFp/sNI5Rqg+AVmn2+Fw0LdvX558UmVy7dWrF+vWrWPKlClce23FN1ahoaGEhoa632dnZ5fbp+I63c6YmGq6lqv2Tqx0u2K6C4uKSbC7LN2idFcF3W9Q6jM69RVE6dae0Gj4y2eqLJfrJrUG+Mg2soYuyt7W7UYVKN32YIjyk3dSx/NVIq/YFGU19gdNe6hyV4udVrzk7lWz+ldClc7bUZOhyyUqORmo8W2IeFm6f3Qq3fsyj7MxXYWwFZU4uO3j5cz92+mkNlIhAlN/28HjMzbw1JhuXNXfN668ItnmFpbw49oDtEqIpG+aH93t/UBl1th9mcd5cfZmPlm6G5thMHvScOLLxEHf8P4fbDiQTaPIEN4Y14ewYDtk74djO9Q9V/erVAm3CpVup+V5/ypId1YN6H4lnZvFQFo32LQbDm/FaUbBiEgAPGGFxq5FsGOBepNSpuZ9RbhKDWash/9dDcePwqKXVbLDsFgoyFLZ1SOcD5Uat/F4tXS9FLbOgTXTYOb9njbXf62Swl3yhtuoc1HUBsYHfcu+9H/7yDa7QCXpiwixc0mv5ny8dDdvLtwuSncDpKb3Bc/9vInX5m+jb8t4Pr91cLX7cSinkM+X7+Xq/qnERVTi6RNgBKSlu2nTpuXcwzt16sTu3btr1K5vIjUHJl6W7hoo3ZXHdPu+MDC9LN3iXl4VJKbbOnTqK4jS3SBoMUBZpP0gC7/KNjTaYxmuyNLtb9KGqMRf/sIV113idE8dfn/l+1aBKsnWZlc14G32Gn1XvSe5BwBdjR1cu3osLH2TuRtV/HqvFnH0ahFHcanJz17W7q9XKRf/j5d67lnyCku4Z/oqSrxztxoGHyzeyaAn53Dv52v4y1tLWeWsW10Wh8Nk3b4ssguK/fwDT0y5uGOv0mmvztvGsfxijuQVuetQuziQdZwNB7KxGfDNxCH0cz1McLmWJ3fzJDXcOEPlJACP0h2b4oy5NsFRrOLzWw5VnyU4H5Id3gwH/1R9Sxvk2+8PLiyvyJ+Ixu3UfWBhtlK4QSncAKOeUg8IMFXSQPBcc6CuAVdYAwa0Gwl9rlMhKWumq1rhAFn7uHbvo1wetJDk7Z/7yDbLOa7JMWHcMKw1hqFKjuV7xc8bhsGqPZks26kSrumcSE2H+zmrqElM95cr9vLa/G0ALN99jKz86s8Hb/+ynadnbuShr9dVu42GRkBauocMGcKmTZt8tm3evJmWLWt2k1K+ZBh+Uboru0FxvXLFdNtxkGx3PomtQ0u3TkkcdOor6PWQIFBL19RWu1a3Hej4XbZDJyn37Nan17yt2sarhBnN+3jKnFUTOW+9aNSa/LBkIgrSaVa4DX75D3Mb9wTgnM7JBNsNVu7OZN7GDK4f2oojuYWs26c87dbuy2Ln4TzSEiL5eOkuvlq5j8JSj3Vyf1YB//z2TxymsnDmF5Vy60fL+e6OoSREKc+9/KIS3v1tJ58s3c2+zOMMat2Y/900sNZ+ftk12GXptttUXemEqBAO5xbx3qKdXD+0ldvavWJXJgCdmsa4PQAAVdINoOUQaDlYZfTPdT6wiE/zDU+Y8D0c2QZHtytXbldfEtqr/4c3Q3G+6pvTI8Hd75imypW8WW9I7HLyHxocpqoWHNmq3p/zOBzeohTq7leqXAt5GbBtrvq8zPdhs6mwhrMf8+QHKC2CVR+rmuCp/eH7ewgpzQOgeeYf2Gwe40tOgVKuk2PDaJUQyaguyfy4Lp09xzwPOUocJjd+8Afb16kka6UaKt1i6a7+A+O8whIe+kopyK7rb/H2wz5J+06FHYfVufjD2gNsOZhDuySPt9KirYd59+c/q9WuzgTkanfPPfewZMkSnnzySbZu3conn3zCm2++ycSJVUiKcgLKZy83sVnpXu7KpOZ0L7dhkmhzur3Xk5ju+j7x6dRX0Ku/OvUVxNIt+OJ32Q64Ca75QsVd60Z8K8+D3LMeqbEngZy3XthsrD7vG24ruhMAM+8Qi7ep8ldndUrkjI5K7kt3HCG3sIRftx72OXzG2gOUlDp4f9Guck1PW7YHhwlD2jZm8YNn0TohkgNZBdwzfRUOh8mPaw8w/Nn5PPvTJnf85uLtR9iYXj58zh8UFJfy4eKdZOR4FD3v8T9e7KDIaaqffEk3zu/elBl3DqNz0xhyC0v46/u/87dPV7P5YA4rditrbO8W8Z4vKClSmcBBhZiEx6tEZsP+T4UqjJzs2yGbHZq0V8nuvBMDupTuI1tV1nOAxM6+5+2k9TBxKVzyetXzJDRxupjHpkL/m+Gil1Q+B5tdKfBjP4cz/gHN+6qKDBXhnZDPlbvizy/hl//Alp8wneaYToVrMLwS+WY647mTY8MAuOk0VbYwI9uTR2Dx9qPOvAKqjT92Hj3lTPJ1jSjd1Z9fNx/M4XhxKU2iQ7lmgApb+WXL4ZMcVTkurxXThFfmbfX5bPmuY/zofLgTSASkpbtfv3589dVXPPjgg/z73/+mVatWvPjii4wdO7ZG7ZZPpAaGYZ2l2x3T7Xx2YjMcNCZTfVhPspfX94lPp76CXv3Vqa8gSrfgi8jWC8OAv0xXSaX8YKkX2foS16QZPztUXLBhlhJZkkVCfDLtElVMb8vGEew6ks+vWw6zcLO6CW4WG8b+rAJmrN5Pq4RI9mUedz4L8cjzy5X7wQjlxmGtiQ0PZsq4Plz0yq/8suUwd05byQ9rD+AwIbVROHef1Z4f1x1g9oYMpv++h0cvrIL19hT57I89PPzNn3yxYh9f3joYm83wGf8cZ9xxQlQoV/RN4YqeCRAcxt0j2nHTh8tZuTuTlbszOZB1nPwipQz2aemldG+bo0rMRSWrGumgQk5aDDi1jrpyMGTvgzz1AISkLhiG4Ylzrk7YQ+fRsOVnOPvfFWezt9lh+L3qryqkDlBu60e2qCRsQOGwByla+CIxRj6l+cfcu2YfL4EQ5V4O0KtFPH87uz1//9Jzb/rZH3shJYHuqXHMBw7nFHLD+3/w1vi+hIfUbZhHdkExXy7fy6JtR3CcINZ83VL18GnZzqO8uXAbN51WC+E89Yzqzq9bDuYC0CEpmqHtmvD+4l38trX6Svf+TM/Dte9W7+eeEe1JS1APnbcdyq12uzoTkEo3wAUXXMAFF1zg1zbLJ1Lzjumu/o1FZa547pJhNo97eazDFdNdPyzd9d11UKe+gl7u8LrdWIvSLXgjsi1D895+a0pk60uT6FBKCOKIGU1jI4cmRha92nVzy+aMDom8t2gnczce5JctSgn8+/mdOPbZXVx87Beu/OIZoDE3ndaaR15yp28lr6iU9qlRDG+vHsK3T4rmofM68fA3f/L9mgMAXNYnhScu6UpokJ1GUSHM3pDBVyv3cf+ojioxmR/Z7LypX7Unk29W7+OSXim+yb4KldKdHBsK392lymldP4uzO3diyjV92H00j6d+3MiibUewO2+AfCzdaz9X/7uOqVkugIhGynCRd0i5cEclQXxauURjp0z3y2veN28MQ1m7Zz2s3g+9h7Az72PxL7M4g98pyfZYEjOdSndTp6Ub4I6z2vFDuybM3KLe5xWVkhwezCW9UpgP2A2TX7cepts/f/J4VtYRJaUqZPNkZDu9NA5mF/HkDxsZ0zvFHUoRKFQ3fGdLhgpPbZsYxYDWjbDbDHYeyWfP0XzfEA6UvrP7aD6p8REVnhuFJaUcdpZA7NY8lrX7spix9gATz1APtLY7Xc8DjYBVuq2gfJ1uLHUvd2eYdFq648khCLVoiaW7aujUV9Crvzr1FUTpFnwR2VqHyNaX+IgQ7DaDQ2acU+nOpGvzGPVhaTGXxO8g3P4da1e0IsPRjbBgGyM6NsEM/o1wx3FOK17EjuCLuX5IK54MtuNtQ7ppWGsfGV8zsCXzNh1i7sYMRnVJ5qkx3Qiyq3uI09o1cVvQf/oznYt7NuftX7ZzOLeI+0d1qPFY7TmW73799I+bGNkl2VfpPl4CNkiOCYdt81Sd9qVTMC56iVHtYyAokWU7jjJ7Q4Yz3juU1EbhKi77yDbY9INqqOtlNeonoGrab5wBaUOVa7rNXnOlG/yfGLDPBNj7u4phH3AzGAZbo/pwRu7v2HIz3LtlFRRDLCTFhPkcHmz3epBvM7j19DbYNu0E1AON7MgQjuYVUSWN12LaJ0UxpncKceGVV074/vhiPp4HIc4HRnuPHQ84pbvalu4MNXO0S4oiJiyYHimxrNidyYLNh7hmoG/Oq7kbM7j+/T+4sm8qT1/WvVxbrrCFkCAbV/ZLZe2+LGZvOMjEM9pimibbMsTSLdQQH0u3oxSHaXocvWowSVfuXu7aqCbNRMPpShQWC0F1N8nodEOlU19Br/7q1FcQpVvwRWRrHSJbX+w2g8aRIRwqiKUje2hCJp2axkBBNrx/IT0OrKJHMOSbofQqfIMzO7YgLGs7OJS16KbUvVx6yVASY8IIDwlyK91TrunLub1TPF+Udxhj6RTeOPtCVg4fRO8WcW6F29WPK/ql8uLsLby5cDttmkTx+IwNAFzQvSldm8fW6HfuOaqU7mC7ylL+1cp9vkp3QQlEQLOYYNi1X21c94UqmTVtLLQeztiBLzB7g1Im+7SMw8hJh9cGQakzNjm+lX+8Mnpdo/688IvS7W/CYuDKD302HU0cCLlTCC084t6WeVwZZJrGhqsNDgd8cgXG3mXufd45O4jzBzfnrc3q98WGBzHzgTOV0l3HBNkMmkSHnlT2exfG8TEQHaoU873H8umZGmd9B+sRNXUvb+9MeHZL/B/0Pvgcj828iX5pd9Mh2ZMIbekOlYF/+h97uLJ/qq/HCbDfmSOiaWwYF5b8TFrwu0zacxuHc/tS6jDJKyp1e6sEEqJ0+xFX9nKXpRuLLd2eOt0upTtTfVCHmctBrxsqnfoKevVXp76CKN2CLyJb6xDZlqdJdCiHCuIASLRl0TExHD4fCwdWQWgMOEqJKM7jq4uCSendHTZ86j628ZEVNG6sHrR7y3N4x0TP+/yj8MFoOLiW4JUf0X/iMrCXvy8ZPyiNNxdu58/92dzy0XL39lV7MmukdDscJnuOqRvxC3s048sV+/h9x1H3fQx4lO5W4fngcHrtFeXCx5cpN++N3zN82L2kNgpnz9Hj6kZ/+zylcIfGqnJ8wybVyMhxIuql0l0BIcmdydgWh50D7m05BSVE4Emkxv4VsHUWRr7H+yB0weOQsA/D6AkoQ1JYsJ1mceG12Pua4TJ+RYQGcRxl6Q40qjO/5haWuJMptm0SBXuWcfaWxzCMYq4s/YEJ7w7mm9uHkBitzp/thzzu4f/6bj1fOXM0uHBVIWgaG0bsqrcYat/EneaXzNs4nObO86lZXDg1K9SsH/U/iFUjXBe7zTDANJ2W7tpLpJbksnTXYTw36BV3rFNfQa8YdN3KAknJMMEbka11iGzL0yQ6lEOmUmrbhOcRsfQl2DobgiNg/DfQ8XwAOhesIiYsWLkUuyjOU0oUlci2pEgprgfXqvc5B2DeE+U7UVxAox3f80HTL0g1DvooLKsrqe9dVQ7lFlJU4sBmKKUb4I9dx3z6m12okqOlBR/zPbjUY2m1LZvCM5f24JJezbmqXwvY+av6oN9f4aZ50OnCGvXzRLj6Wt/P2bQmkcwu7Y23IdHEINiuPCoA5ToPPvvYDGDNdIxCFdtrniBhWX3F1efIUGVT3CdKd5WOcbl7J0SFEm/kwvRrMBwq4/0A+0bysg4zbdke9/47j3iU7tV7Mvnnd3+6Kw+AR+luFhMGWXsBuMI+nxVr1rLNGc/dopE+D3P8Rf2eOTTDN5Faqd/qdFemGLpeui3d9SBzOeilGOrUV9DLQqRTX0Es3YIvIlvrENmWp0lUKIfMOADSwnKVwg1wzmPKXbrVaer9joXq/74/1P9Qp/V5xy9AJbJd9wXsWw5hcTDqabVt2Zuwf6V6XZgDC5+F5zvC59fR9+CnPBPyDuBJvLWqhkq3y7W8WVw4fVvGE2KUsPeYJws5QJazrFVTQ7mu0qgNBIWp+6dzHlfb1n7OoKDNvHB6CLERwe7fTdrQGvWvKuhi6W7ZOJKfHX3w6aVhkBgd5rFGOuPfvX+KAVBahG3Xb4DHe1MnXH2OCvO4lwca1TEmueK52ydFwZZZqrZ9fCto3A47Dk63rXbPAaUOk91HlFxvGa6yw3+weBdnPT+fwZPnMGn6Kg5kqYcdaZFF6qEgEGKU0mPXVNbvz1KfJWhYPrOG1H8tQyPKJlIz8Y/SXXkiNef3Out01xdLt043VDr1FfTqr059BVG6BV9EttYhsi1PYkwoGU6lu6ktCw5tUh+kOstdtXKWwNq3HHIPwcH16n3/G9T/HQuACmRrmrDkNbVh8B0w8BaVaMx0wPf3QNY+eG0wzH0cjh+DmBSwBTHIWMflSQd4dayKj956KJecguJq/77dTqU7NT6C6BVTWBN6IxfYFpOR46kTne0sGdbE4SxTlNwNJvwA1/2o+p46ABzF8O4oeH0QLHgGsnaDLQhSB1a7b1VFF6U7rXEkix1dKDE8SdsMDI9r+ZFtcGgj2IJ8z5dQVaLO2Kke7Ohs6W5cks7jQe9wyf7nYdfiOu5V7VKd+XXLQeXd0C4xCg6pPA60OdPtOTLCvoI1ezMxTZP9mccpKnUQYrdx78gOvD2+LzFhQew5epz9WQV8uXIfy5wx361DlV5i2pTnwRjmMn/5OgBaxPtmRA8EROn2IxWXDLM+ptulfTepJzHd3tT3xUm3mz+d+qtTX0GUbsEXka11iGzL0yQqlEPEAZBYsF3Vm8bw1IyOT4O4FirWeenrYJZCdDPofpX6fPcSWPpGednu+g3S10BQOPT9q/pg5JPKQr5/JbxxmlJcY1vAmLfh7jXQ/UoAnk2eS+8W8TSPC8c0Ye1eZaH637Ld9Pz3zyzZ7knUdTL2HFWWr9RG4bBpJmEU8lzwFHKPeOKOi0rUPVRM0UG1ITYFUvpAC6dCfcZDqt+hzszuLhf5Zr3BqTBaiS5Kd3xEMAlxsaSbjT0bDejoSoTlyvLecghGhGcfo88EiGmOUaS/e3lcxjKuCZrDRcU/Yn53Vx33qup8s2ofk3/cQGkNssVXS+l2WrrbJkVDxka1MbETdDgPgNNtq8nKzWdf5nF2uNzDG0dgtxmM6JzE7EnDeW1sb7qnKM+bjenqHEq1qTnCSO7GwdgehBilXMpcAFomiNIt1AAfS7ejFLO2YrqdTzNjDGfsSlTdupeDPouTbjd/OvVXp76CKN2CLyJb6xDZlqdJdJg7pjuk0Om1FtcCgr3iHtOcLuZLXlf/U/pAQjtoP0pZgH+8D6Mg07278Uo/mD5Ovelxlao/DRCdBCMeUa/zD6uKJ+O/VnWkbXYYcjdgwKYZcHA9A1JCeDf4aYJ/mUxJqYP/zt5CZn4xT8/cWGXFzFUurEWjCDi2E4Awo5j2x+Z7djIM4iKCCc51Zi6Pae7bSOvh8OBuuHeb52EE1IprueqePvc1n90yiMikNu5tt53elofO76TebHQq3R3Px4hp6jmu1TDocK7Hi1JjpdtmernGH90ODhXG4HCY3PvZasZPXcaPaw9QUlrehb7UYTLp01Wc+Z/5jHh+AR8u2VUrfS8pdfD3L9fyxoLtLNxyiMO5hVz37jK+X7P/lNo51fk1p6CYP3Yqy3SHpGiPpbtJR2jeByKbEGPkM8j2J2v2ZrnjudMaR8LO32D6OBILd3Net6aM6prs03aiw1m2LjaV2NNuAeDqoLnYcNCysfUPyuobonT7kXKWbkftxHSXmmUuqnpg6Xb1s77HSUtMt3Xo1FcQpVvwRWRrHSLb8iTFeGK63TTp4Pu+7Znqf7EzTrXdSHUjcNUn7lhto8STOMrI3AHHj4ItGAbe5ttWn+ugxSD12WVTVeZv9/e2h84Xqde/vsBVjpmcYV9Nr13v8cv6PaRnqyRJK3dnut1IT4YrprtlbBBk7wPgqBlFpOkdc2uQHBPm/pzYMkq3i6AQT2w6iNJdAc3iwmmS4hnTXvGFRIQEQd4R2LNEbexwLkZcqnsfIzwWWp/uTq6mc0y36zcUmXaVECxbKa5zNmbw2fK9LNx8iFs/XsHd01cB8Nkfe7j09UVszcjh8+V7+HLFPrYfymNrRi7/+XlThcq5v9lwIIc8Z46DhZsP8eHiXczbdIgXZ285pXaqEtNdVOLg09/3sOdoPh8s3kV2QQltmkTSp2mI+6EYiZ3AZnO7mP876D027djttnS3SoiA3/4LG76Fj8ZATjqnt/fVP+KLPEp3WPcx5AfF0tw4woXha4iPqLzeekOl/msZGuFzsZsOTLzdy6s/SZ/M0p1vK/O0qI5jusFz0df3xUmyl1uHbhmKJXu54I3I1jpEtuXp1SKeiwd2xmF4VXJNaO+7U+dL4JI3lBv4rYugt9OKbbPDgJshNtXzoB+wGXa4ejrcvFAp0t7Y7DDua5i0AdqOKN+hoZPU/3Vf0HufqgMdRAk/z/oegIgQ5WE3ZcG2Kv0+l9LdKvgoYGIGR7IgaKhP9mwMg27NY1WcOaj48spoN0JZ5DuPhrRhVepDTdEle7kLW4jHS8J2QGW3Z8tPKp4/uRvEtcAW4nHxtdlskDYUw2kkMosLarW//sDb+HXcCGev6fT8dCqSby5U52vvFnHYbQbfrznA8l1HeXzGBpbvOsYtH63g2Z82AypJWGx4MJn5xaysYSLBqvD7Ts8DrAWbDzFzXToAWzNyycyveq30qjzU/GrlXu77Yg0XvPwrby7cDsAdZ7bDfkT9diISIDJBvT7zYXLDm5NmO8jwP//BzkPKFT0tIRIOO/fP2gOfXEGnxgZJMap8YViwjdB857UclwrBYdDzGgDuiv21yr+nIaHHzKEJZROp+St7eeVKt/r/R4vrKDS9Fuo6zl4O+jwR1s3iolN/deoriKVb8EVkax0i2/LYbQb/Ht0NW7SXe2ZZpdtmU27i3S+HpC6+nxkGtD7d5/m+kdoXOoyCpM4Vf2lwWOXhaM16KmXcLCWoMNPTpSNKeXvpql7YDJi36RAb07PJyClg3DtL+d8y38q7pmlSWFLKAad1PAVl+TLi0+g69AKf/jaLC+cf57aHXKVsVGrpdnH2v+CK95XluxbQ5b7Ghc915sp27ywVRofzy+9jGBAejxGfBoCZXzUvhvqE9314iS2UPabTCHVsB8t3HeP3nccIthu8fk0fzu+mXOtv/nC5O3P+1oxcDucWktY4gklnt+f0Dur6mLMhw/K+/7HLI+/th/LY5ExuBsqrpKpUZX5dul19V9bxYrKOF9MqIZILujdVCfZAWbldRDTi8PnvUGgG0btwGWbGnwC0jg+CTKfrfVgsHFiN8fn1nNEuniSOkhITjOEsF0aseoAWMeh6dWzWUuV1EWCI0u1HyrqX11ZM9zZba54pucpzQD2wdOuyOOl286dTf3XqK4jSLfgisrUOke0J8F6/y7qXn4w2Z/iUiTIqsmCfCi5rN1Ca3AOAfrZN9G0Zz4jOSZzbVSktbyzYzguztvDLlsM8+u2f7Dmaz7p9WdwzfRX9nphDx4dnYpoQHmwntsB5Ex6fRrv+5/r095bT2xBbclhZYm3B9SJUzhtd7mtc+FxnhzdD5h7YppJY0eHc8vu4fp9T4TKPl6mXrgHe9+EOexi73Ur3Tt5ftBOA0T2bkxQT5i53dThXWZEv6N7Ubcx66PzOhATZOKtTEgBzNx60vN9/7FTyjg4LKvf58l1VH4uqzK8rdqv2ujWPJSTIxgPndiTIboMMr3huL1I7D+QPoysArXPUg7e29gx1rYbGwNgvVLLGLT/x+OaLWBp2O4/b33LX6HYp3TRuA816qUSQW36q8m9qKJQfWaHalCsZZuL37OUVuUMfzS/io9JzaREfyrVndPdNvFJH6LI46Xbzp5M7vG6yFaVb8EZkax0i2xMQleR5XdbSfTJane6Up/Nhf9uzataXloOh7/WQvR/7affC22cyMGQbzccoy/ldPUyO/7mCb1eD4VSfi0ocTPp0Fev3Z7vjU130a9UI45izrnZ8GkQmYEQ1AZQLalxEqCeeO6apsuzXI3S5r3Hhc53hgG/vUPkAYlKgaY/y+7h+X1Jn4Asc+cdUyTlNfi94wjwNgGCP0u04upMFmw8BcFV/FcfeuVkMw9s3YcHmQ0SE2Hnikm6M6d2cwzlFjOikjhvergl2m8Hmg7nsOZpPaqOaZ9yevymDnIISLuje1C3zPUePk5FTSIjdxrWD0nhl3lYAeqTGsXpPpo8V/GScLAzxSG4hO511tj+6fgBhITZCg5zl5VyW7jIP/Ow2g+a9R8KKVQy2reMT2/k0LnB6tTRuC6n94NK3Yfo1BBUr9/MBeXOh1OkWH9vC01jXS1XlhE0/Vvk3NRTq14ymOT6Wbkeps2SYhYnUnP+P5RUDBgsSroY+E6r9Pf5EEqlZg043qzr1FUTpFnwR2VqHyPYEuCzdEQmebONVJbIxhtPV2jBQFqWaYBhwwfPwl2mqrbBYQkrzaVOyHYqP037mX5ga8hxjjPmUOEw6JkdjGPD7zmPkFZUyoFUj/nfjQJb+/SxmTxrOO9f29SRpcrowG3EtvL7O8FjGThTPXUforXQD2+epNx3OdSvSFV2LNmf8v1lSBAf/rJW++gt39nJDxbTvNtVDrOMHt5J1vJjosCB6pMS59/+/czrQKDKEiWe0JTY8mDM7JnFFv1S3LGIjgunbMh6Ai1/9jXP/+wvbnDHNlbHjcB4XvPwLrR+cQceHf+S5nza5+/Xp73uY8O7v3PG/lTzyzZ/u0mCueO5uKbGc0yWJtsZeUo1DPDBKWZxX78miuIrJ3E5mnHG5qrdNjCI2ItijcINvubAypPVV5cMG2TcyunsitiPOBG+uh4OdLoAb5sD4byGhA4ZL4Q4K953Lulyi/u9bXqXf05AQS7cfqTCm27DQvdzZ5NE8dWLHhdefTIC6LE463/zV9/7qJltRugVvRLbWIbI9AS5L96m6ljsxQiKAAmV5ttlPun+VsdkgdaByCd29GPYsdcdePxL0IYtKu/DUpUOYtmw3037fQ5+W8Uyd0I/IUHWbmeQsrc0xZwyoS+mObwksVq8dJZDtynZ8knjuOkCX+xoXPtdZ+1HQxlClswbdVvE+rt8XrBJhmaBqeid3rZX++gOP8csgIiKag3aVI6H06E4AhrRJUG7UeYfh54fplrmbFS3skHIn0BYObYK8Qyojfu4hmPUILxduZkNwEc/mX8G6vNbcNW0lX946hJCg8vf1y3cd44b3f+dYvooRLyh28Mq8rew9lk+jyFDeW7TDve+HS3ZR4nAweUx3Fm1T8c19W8bTLTqPH8MfpiQ4mtDUK4kJCyK7oIRDs16kmXkQRk6u0AvENE0Wbzvidpd3yaEsLtfy3i3iVDm11dNUNYPjxyDLab1uUl7pJrkbhMURWZDJU4NM+F1Z40nwKt+X0kf97zoG5k9Wr+NSfb0lYlNU5YR1v5X/jgaOKN1+xO3WYuBUur1iuqn+JH2ymO5jzqyGcRG1k0ykKuiyOOl482cYhsoXUM/7q5tsRekWvBHZWofI9gSk9FX/Ww2v1uFGeDxwFMMKz62Wg5XS/dt/ldsxYIbFEV2QyefNPqZp6gS6Novh3G5NGdCqEWHBZZR+04RjTqXDrXSnefq++BXIUOWIiGvp//7XEF3ua1z4XGcDb4aRI0+8T5nfZ5qoxGvD77O2o37E2/hlDwmna5fusBFiHFlEkc+w9glQkKVKXB1Y7TkwcxfcNB/eOVt93uc6ZYlNX0MikGiHninRDD94N+v2ZfPi7M3cN8o37nnN3kzGv7OUvKJSuqfE8uKVPVmy/SgPfb2Wr1d5am1f3T+VQW0SuGvaSv63bA/ndWvKd85a3Gd1SsLY8g3BjkKCCwthy0x6t2xO0tZPabbkLdVA9ytU/WwvNqXn8Pev1rJ81zGy1h5wbz+R0t2nZTzM/Res+xxWfgx2p0rYdgRENi4vXJsdWg2DDd/B9vmezOWN25Xft8slHqU7tgKvlS5jAlLprv/+tBrh7daCqWKZ/B3TXdFrt6W7HtW8k5Jh1qGL675uZYGkZJjgjcjWOkS2J6D9SPi/rdVWdGzOG2dL5Np7PCR2htyDkJcBMSkY1/8M9lCaHl0G+1cRtG8ZwxdeTdi+xeWPzz8CRU7XXKdbuS3Uq2TVwbWq9FCj1tDvev/3v4ZoVzKsCtdZRfu4lW6AA6s8Lv8a4BPmGRzO6IEdOWJGA5BqHOK0tgkwfZxSuCMSVAm+0BgV9vDlTUrhBlj+LqSvUdWALngBgNgDv/HfkcrV/PUF23xq1G87lMuEd38nr6iUQa0bM+2mgbRuEsVfBrTg9bF9GNYugWsGtuC/V/Xk8dHduKhHM85zZk+/6YPlFJU46JkaR7+0eNjys+cHrZ7GBbG7eCxoqmfbUY+1HKDUYXLjB3+4k615l1cve19bUupg9R71G3u3iIc9y9QH2XuVDGJbwCVvVi5g18PAbXPhsMvSXUHuiSYdINFZYaFCpfsSFbpyirz66qukpaURFhbGgAEDWLZsWaX7vvXWWwwbNoz4+Hji4+MZMWLECfevDfSYOTShvHu56bFvWxDT7cqyWB+Vbl0UQ91iukGfp+26WbPE0i14I7K1DpHtSYhq4uuOeQpYuj5ENILrfvTUxT7zH+rmutMF6v2K92HG32Dv7/DFDXA80/d4Vzx3dDNVrqxMP43gcOhwHtw4D2Ka+b//NUSXtddFVa6zCmO6nfdCjhClrLLxB4t66H9cHqc2AwgKo0/LeDKcLub9YrNIdeyDHQvAHgrjvlQl+Lpdrg52Kbs9x0JwJITFwbivoO9f3crm8PxZXN4nBdOEe6avIqdAuZE/8s06juYV0T0llreu7UtEiMeReFTXZD68fgCPj+7GxT2bY3fevN95prIQHy9WRrpbhrfBKClUVmQXW2Zx0eb7CTFKKXGpbEe3+/zmhZsPsftoPjFhQdw3qoPP3GEYBrmFJazYfQzTNPlq5T6OF5cSGx5Mm/A8pzu5AS2HqmoBV31UsZXbRdsRSp/Z9RsUZqljG7WueF9XGEObChI6RjWB9udW/j0VMH36dCZNmsSjjz7KihUr6NGjByNHjiQjo+JybvPnz+fqq69m3rx5LF68mNTUVM455xz27dt3St/rT8S93I/4JlJz4HBAsNvSbZ17eYkzEYO4l586Ot78iWytQZRuwRuRrXWIbK3D8vUhPA6u/Q5yDngU417jYN0XsPw9VUII1OffTITweI+y7ar73KhVuf4CGJe8DpdfYU2//YAua6+L6irdbkt3WBxwFDbNgAE3WdZPf+Jj/AoOxzAMwhPbQPoWzkzKg53O7PktBrgzuNN7PPzxjnodlaws2yOfUO/D4z377FgAKz/mn1dfyt5tWezPSuflz48zul8bftt6hCCbwWtjexMV6qVa5R1W1vOwWIhM8GwvKaTD7mnc09rkhe3NaZ0QyTmdk2D7HJVhPrqpur72LSek4AgbHC2Y6+jJxKBv4egO9mceZ9Knqzi/W1Pmb1JZ2S/vm8p1g1vxqFditCN5RVz48q/sOJzH2Z2TWOKMHb95eGts+52JzBI7wXUzVLz/yfJANGoFQ+6CX5X1n/iW7gdo5eh1DXS9rPLPT5Hnn3+eG2+8keuuuw6AKVOmMGPGDKZOncoDDzxQbv+PP/7Y5/3bb7/NF198wZw5cxg/frxf+nSqiNLtRzwXu6FKhuGf7OWVK92++yVEidJ9quh48yeytQZRugVvRLbWIbK1jlpZHwzD1xLdarhyF890JmFqezZsnQUbv6/4+OTuXk15nQv2+n1Lqsva68JvSvfOX1WSLZcCWo/xMX4FKWWvZdsukD6T02IyYKfTyuny1gBo1hOa9lSu9ANvgaBQ9edNxwuU5Tt7L5Fv9Od/AKHAVvX3F/v1FPQYT0q8V0mxX/4Dcx93PogyYOg9cObDcGgDfHkzHFzLXUCvpAuJuOBpbDYDNjut7e3OgabdVVx5eCOeDX2UiIxV6rOj23l13laWbD/Kku0eF/exA1oQHmKnXVI0h53b/v3deo473bxnrVe1xnumxnHTsNYw7yO1kyuPRFUTL57+IGyZBQfXVRzP7U0VFO6cnByys7Pd70NDQwkN9ZV/UVERy5cv58EHH3Rvs9lsjBgxgsWLKwhlqYD8/HyKi4tp1OgUq0L4kfo9w2mGz8XuzF5uWKh0l51Em0SVmSTqEF0WJx1v/kS21iBKt+CNyNY6RLbWUSfrg80GPa+B+U9CSBSMeRN+fxtWfaxcS1sM8mRbtodCmzPK9bfW+1wNdFl7XdRY6baHqizWhzYoJat7/fVCcOFr6VYKsNH2LPj1P9g2fOPeRtpQ3wMvfQe2zVGu5BURHAbD74cFT7u9OQpLSiktKSHCKGSM/Rdihv/Ls//cJ2DhM+p1SDQU5cCvz8P6b+DoNrU9NBYKszgt6ztYEw5pL6skZaCU7vYjoaQQWp9B6hIHyw+qBGmOo9v4YqdvnP3Qtgm0bhIFQJfmsbjU0KP5xXRpEsm953TgkW//pLC4lOcu76EyuO/9Q+3UvG+VZOsmKBQuexd+fggG3nby/U9C586dfd4/+uij/POf//TZdvjwYUpLS0lKSvLZnpSUxMaNG6v0Pffffz/NmjVjxIgRNepvTRCl24/4xnSXYpqmXxKpnSym20VCPVS663uctMR0W4dON1MgSrfgi8jWOkS21lFn60P/G+HgWuh0kYr9Hn5flZLB6XQu6LL2uqhxTLfDAR0vVEr3xu+1ULp9YrpdVtaWg5V3RfoaKClQFvAy2b9JaOtb+qoiBt3mU27NVurg1le+YeqxCfS2bcUWreK7+eNdj8J99r+VO/aq/8G3dzgVbkPlQTjvP6pPH18Oaz9VynzOfpXzoO1ZYA+GQRMB6JO2n68WK4XTlncIW3E+nZsmcs3Alkz9bQf3nO20OB9YQ6Itx93HG09rzb0ThhAbEcxZnZLc8dw4SmH/SrVTSr9TkjEATdrD2M9O/bgKWL9+Pc2be0oElrVy+4OnnnqKadOmMX/+fMLC/OPuXh1E6fYjPhe7q063lZZurzJkdpuhLqR6gmQvtw5dHmjolqFYspcL3ohsrUNkax11lmE7ohFc+dEpH6bTuRBw2ctNEzqeB788B1vnQHGB3+JzrcLXvTwc95uBt8HXt6j3qf3Lu49Xg2C7jScmjCJjShsSj29TGb2jkuCH/1M7nPGQUrgBel6tFNXtC1Tmbldeg+izocfVsPoTVboL4KyHITjc57v6tIwnm0iOmlE0MnJpaRzkr0MHcFmfFP4yQFUCIHs/vHM2xuo893FndUom1plkOcQGIcFOneTQRlVJICRKJUOsQ6Kjo4mJiTnhPgkJCdjtdg4ePOiz/eDBgyQnJ5/w2Oeee46nnnqK2bNn07179xPuazV6zByaUNa9XFm6LYzp9mqycWSIigepJ+jyRFinp+wuRLbWIJZuwRuRrXWIbK1Dl/XBhU7nQkOU7Qndy00TmvZSlteiXNix0MLe+gdf93KvBwRdxyiFGHzjuWtI09hwEnudr96s+xK+uB4cJSoj+mn3+u7cvA8Mm+STSBBQVQBcDwiSu0H3q8p9T7PYMNo0iWSXqRTMnpFHubCHKjnmrhG27C0oKcBwFLuPc4/v4S3w+iB4oQvsXgK/OMt1Ne9d9VjuOiQkJIQ+ffowZ84c9zaHw8GcOXMYNGhQpcc988wzPPbYY8ycOZO+fU/Rjd4CROn2Iz4Xu6PU8phum9fr+uRaDvosTjot+C5EttYgSrfgjcjWOkS21qHL+uBCp3OhIcr2pEq3zaas3QAbvrWop/6jokRqgLJsn/88tDlTZSL3J+3OVv83zVA17Bu3g4tednaiCsQ2V27o0U1VHyvwSjAMg//dNJCktE4APDgwhNAgu3JlfzoNFj6raovj+7XGhu/hxwfgrbOUdTsvA6aOUlZ1WxAMnVSTX16rTJo0ibfeeov333+fDRs2cOutt5KXl+fOZj5+/HifRGtPP/00Dz/8MFOnTiUtLY309HTS09PJzc2tq58g7uX+xNfSbfotprtS93JvpTtalO7qoNOC70Jkaw2idAveiGytQ2RrHbqsDy50OhcaomxPqnQDdB6tEuP9+TWMegpCoyzprz+oKJGam04XeGrK+5PUgZ5kaQAXPF/OPfykDLjppGXZEqPDoFUX2P0dMfl7IWsf/PR3VWJs7uNqp+imGOx0H2Msex0OOlW91AEQGg1bZ6v3o1/3SWpY37nyyis5dOgQjzzyCOnp6fTs2ZOZM2e6k6vt3r3bJ1zi9ddfp6ioiMsuu8ynnYoStdUWonT7Ed9Eag5MvGO6qz9JVxaX4+1NXp/KhYE+ccc6JlLTJV5ep5spqB2l29/oJmOdENlah8jTOhqiYlhfaIiyPWkiNVCZvhu1hqPb4c8v/W8p9iMVJlKzmqAQaHumykze42podZp13+VyTT+yDWY94qnpnaMymzPkLmy/vABsAcBms8Og2yGhPfS4ShkAl06BRm08Hgwacfvtt3P77bdX+Nn8+fN93u/cudP6Dp0ionT7EV9LdykOq2O6xb28xui04LsQ2VqD1Uq3FTLQTcY6IbK1Dl3mMB3RTbY6XWcNUbZVsnQbBvSZoJS85e/Va6W7wkRqtcHIJ5Ulufe11n5Po9bq/65fnRsM+Mt0OLQZ9i6DPhMwkn/GpXQbPf8CI5/wbWPwHdb2UagUPUx7mlDW0u1wgGGhe7kOlu76vjjptOC7ENlagyjdgjciW+vQZQ7TEd1kq9N11hBlWyWlG6DHX8AWDPuWw4E1FvTWP1SaSM1qYlNUeS+rXe+TunoUb4DBt0PTHtD9cjjvWQgOx0jy1L02ev7F2v4Ip4RYuv1I+ZJh1lq6fWK665mlWxcXaCkZZh06lYIB60uGWSED3WSsEyJb69Ct9JJO6CZbna6zhijbk5YMcxHVBDpdqNzLf38bLnrJgh7XnDqzdNcWIRFw+3LlVm7Y1PsyGF7x5EZQ/SklLIil26/4XOyOUsA/dborUwx9Ld31S+nWRTHUMaZbl6ftOlkwQCzdgi8iW+vQZQ7TEd1kq9N11hBlW6WYbhf9nYm+1kyH/KN+7Kn/8Bi/jFNPZqYLNpuyqFegcKuP9TMmBQp6aBmaUM693DSxGRZmL6f+Wrp1WZx0WvBdiGytQZRuwRuRrXXoMofpiG6y1ek6a4iyrbJ7OUCLgZDcHUoKVGx3PcTXvbyBKt0nQadrKtAQpduPlC0Z5lunu/onfmUXkIlnQpSY7uqh4+QksrUGUboFb0S21qHLHKYjuslWp+usIcr2lJRuw4CBt6rXv78NpcV+7K1/qLROdwCh0zUVaIjS7Ud8Ld2lzjrd1sV0Z+Z7JrxGkaJ0VwcdJyeRrTWI0i14I7K1Dl3mMB3RTbY6XWcNUbanpHQDdL0UIptA9j5Vt7ueYTrdy8XSXf61UPeI0u1HfC3dDkwfS7f/le6jeUXu10H2+jWUuixOOk5OIlvrEaVbENlahy5zmI7oJludrrOGKNsTKd3lYroBgkKh/83q9a8vQEWKeR3icFrfxdJd/rVQ99QvTU1zHO4nbEYF2curf+JXlhThWH5RRbvXC1z9rO/JybzlWd/76kKXhV/Hid8K2YrSrSciW+vQZQ7TEd1kq9N11hBlW9E+rnuhCi3dAP1vgJAoyPgTtszyU2/9g1laAjgTDQeopVsSqdVf9NAyNME1QdkMwFFaJqbb/5buI3n1V+mWkmHWocsDDZ1KwbiwoiSMlAzTE5GtdehWekkndJOtTtdZQ5RtlUuGeRMeD32vU69/fb5eWbtNh1K6DcMG9sAsl6XTg6xAQ4+ZQxPKuZdjYsO67OX1aJ4rhy5PhHWcnES21iGWbsGFyNY6dJnDdEQ32ep0nTVE2Z5yTLeLgRPBHgK7F8OOBX7orX9wWbqNAFW4Qa9rKtAQpduPlCsZ5vBPnW4dLyBdFieRrXWIbK1rs2zbVrUfyIhsrUOXOUxHdJOtTtdZQ5RttZXumKbQx2ntnvt4vbECeZTuoDruSd2h0zUVaIjS7UfKJ1LzT/byylyEeqTGAXBxz2bVbtsqdHGB1jGmWxfXfR0nflG6BRciW+vQTXnRCd1kq9N11hBle6KY7goTqXkz7G8QFA57f4fNP9Wwt/7BlUjNFlS/KvrUJjqFbAQaMhp+pKyl2wQMC93L3xrfh8dHd+XJS7pVu22r0GVx0mnBdyGytQ5RugUXIk/r0GUO0xHdZKvTHNYQZVttSzdAdBIMuEm9nv0olBTWoLf+QSzdel1TgYYo3X7Ex9LtKC2Tvdz/SndidBjXDGxJZGj9m1x0WZx0nJxEttahs9It+BddrjMdEdlah26y1WmdaIiyrZHSDTDkblW3+9BGWPhc9TvrJ9yJ1OyBa+nW6ZoKNETp9iMuVxyb073cYQZuTLcuLtCSvdw6dHRx0jV7uS7y1QmRrXWIbK1DN9nqtE40RNlWK3u5NxGN4Lxn1etfn4f0tdXsrX+QRGp66gyBgh4zhyaUcy+3uE53fUYXxVDHmG5dnrbrOPHraunWRb46IbK1DpGtdegmW53WiYYo2xrFdLvoPBo6XgCOEvjh3jpNquZw1ekOClylW0edIVDQQ8vQBN9EaqWYprUx3fUZXRYnka11iGyta7M22g50RLbWIbK1Dt1kq9M60RBlW2P3cnUAnPuMSqq2ezGs/6aaPa45HvfywFW6dbqmAg1Ruv1IuZJhFsd012d0WZxEttYhsrWuzdpoO9AR2VqHyNY6dJOtTutEQ5StX5RugNjmMOQu9XrWw1BccIq99Q9maSkARgBbunW6pgINUbr9iK+l2wzomG5dFieRrXWIbK1rszbaDnREttYhsrUO3WSr0zrREGXrN6UbYMidEN0UMner+O46wHQ4lW57aJ18f31Ap2sq0BCl24/4WLodpSqm2xD38vqMyNY6RLbWtVkbbQc6IlvrENlah26y1WmdaIiy9avSHRIJI59Ur395Hg7+eWrH+wFxL9frmgo0ROn2I76Wboczprvmlm4dkyK4+lnfk5N5y7O+99WFLgu/jhO/KN2CC5GtdYhsrUM32eq0TjRE2Va0zyknUvOmyyXQ4XxwFMPXt0FB1qm3UQPcidSCA7dkmI46Q6Cgh5ZhMU899RSGYXD33XfXqJ3yJcO8Yrqp/omv06LkQkqGWYcuDzR0KgXjQkqGCS5EttYhsrUO3WSr0zrREGVb45JhZTEMOP8/EBoLB1bBW2dCxsZTb6eaeNzLxdJd9rVQ9+gxc1jI77//zhtvvEH37t1r3JbHvdwAs9QZ0y3u5fUZka11iGyta7M22g50RLbWIbK1Dt1kq9M60RBl61f3chcxTWH8VxCTAke2wruj4MCa6rV1irgTqYnSXe61UPcEtNKdm5vL2LFjeeutt4iPjz/hvoWFhWRnZ7v/cnJyyu1Tzr0c02PfFqW7XiKytQ6RrXVt1kbbgY7I1jpEttahm2x1WicaomwtUboBmveBmxdC875w/Bi8f2GtKN5uS3eQJFIr+1qoewJa6Z44cSLnn38+I0aMOOm+kydPJjY21v3XuXPncvuULxnmnb28+ie+Tu5XLnRxgdYxplsX130dJ35RugUXIlvrENlah26y1WmdaIiy9XtMtzeRjWHcV5DSDwoyYfo1UJBdszZPgsOZSM0WJDHdZV8LdU/Ajsa0adNYsWIFkydPrtL+Dz74IFlZWe6/9evXl9vHx9LtcGCaJoa4l9dxT06MyNY6RLbWtVkbbQc6IlvrENlah26y1WmdaIiytczS7SIsBsZ+DrEtIHMXzHyw5m2eAE+d7sBVunW6pgKNgFS69+zZw1133cXHH39MWFhYlY4JDQ0lJibG/RcdHV1un7KWblPqdNf7/opsrUNka12btdF2oCOytQ6RrXXoJlud1omGKFvLlW6A8DgY8wZgwKqPYMGzUFMreiWYpmrXsIvSXfa1UPcEpNK9fPlyMjIy6N27N0FBQQQFBbFgwQJeeuklgoKCKHU+KTtVTpi9PMCUbl1coCV7uXXo6OIk2csFFyJb6xDZWodustVpnWiIsvV79vLKaDkYht+vXs97HD6+FHIP+a99J56YblG6y74W6p6guu5AXXDWWWexdu1an23XXXcdHTt25P7778dut1erXd9EapK93Pt/fUVkax0iW+varI22Ax2RrXWIbK1DN9nqtE40RNmeyNJd45juspz+AMSmwA/3wra5MGUoXPo2tBrmt69wiHu5VtdUoBGQSnd0dDRdu3b12RYZGUnjxo3LbT8VvN3L3S4ufrB0izXWOrzlWd/76kKXhV/HiV+UbsGFyNY6RLbWoZtsdVonGqJsK9rHdS/kV0u3+gLoPU5lNv9sAhzeBB9cDOc9A/1uUPs4SqEoV70OCoOKspCXloCjBILLh4e6LN22Cj4LFHTUGQKFgFS6raJsyTCQmO763l+RrXWIbK1rszbaDnREttYhsrUO3WSr0zrREGVbKzHdZUnqDDfNg+/uhrWfwoy/wfyn1H3z8WPu+2cAwmKV8u2itEjtAxASDSERPk2bxQXqN4ilu9xroe4RpdvJ/Pnza9yGTyI119M2cS+v456cGJGtdYhsrWuzNtoOdES21iGytQ7dZKvTOtEQZVsnSjdASCSMeRMSO8Gcf0NeJfHdBVlAVsWfFeWoPy9Mp6HLiEzwY2f1QqdrKtAQpduPiKXbgy6Lk8jWOkS21rVZG20HOiJb6xDZWodustVpnWiIsq0zpVt9EQybBD3HKqXbsEFEI2XdNmxQlKe2lxZ7jrHZISIB7MGQdxhKCnyaNOfeANuXYoRGWdv3eoxO11SgIUq3HylbMky9dind1T/xdYzPkJhu69Bl4ddx4helW3AhsrUOka116CZbndaJhijbE8V0+z2RWmVEJ6m/sgSFKiW8MsLjym1yBCl3c13u56xAR50hUAjcs9ICfEqGOV8H4Sw/FmCWbikZZh26PNDQqRSMCykZJrgQ2VqHyNY6dJOtTutEQ5RtRfdqtWbptgCPx6ke93NWoKPOECjoMXNogo97eclxbrN/w2j7b+rDqMRqt6vjBaTLE2GRrXWIbK1rszbaDnREttYhsrUO3WSr0zrREGVb0W8SpVtvdLqmAg1xL/cj3u7lRkEm9wVPB6C08xjs3S6vdrs6XkC6LE4iW+sQ2VrXZm20HeiIbK1DZGsduslWp3WiIcpWlO6Gh07XVKAhlm4/4r7YW59OaVI3Vjna8FDxXyke/VbFtQariLhAW4fEdFuHjhO/KN2CC5GtdYhsrUM32eq0TjRE2Vb0myyr010LuMM8NbmfswIddYZAIXDPSgtwK91D7iB3wjxGFz3Gx6Ujanzxi2JoHTot+C50iZfXUbaidAsuRLbWIbK1Dt1kq9M60RBleyJLN+ineIulW0+dIVCQ0fAj3he790Rlq+G1r9Oi5EKXxUlkax0iW+varI22Ax2RrXWIbK1DN9nqtE40RNmK0t3w0OmaCjRE6fYjLrcWpXR7tttqeNLreAHpsjiJbK1DZGtdm7XRdqAjsrUOka116CZbndaJhihbUbobHjpdU4GGKN1+xHWx22w2HF4TVU3PeR0vIF1coHWMfXH1s767DelUCsaFlAwTXIhsrUNkax26yVandaIhyrai3yRKt97oqDMECnrMHJrgfbE7vOapmp70ohhah46xL7o8bddx4hdLt+BCZGsdIlvr0E22Oq0TDVG2Ff0m73shlwenLkgiNT11hkAhcM9KC6goprum8dyu9ip6XZ/RZXES2VqHyNa6Nmuj7UBHZGsdIlvr0E22Oq0TDVG24l7e8NDpmgo0ROn2Iz5Kt3ObP054HS8gXRYnka11iGyta7M22g50RLbWIbK1Dt1kq9M60RBlK0p3w0OnayrQEKXbj/i6l4ul2/t/fUVkax0iW+varI22Ax2RrXWIbK1DN9nqtE40RNmK0t3w0OmaCjRE6fYjFcV0++uE13Wyr+9xNZXFMdVndDkXdJz4RekWXIhsrUNkax26yVandaIhyrai3yQx3XojMd31l8A9Ky3A+2J3OPxn6Qb9JnvJXm4dujzQ0CkrrQvJXi64ENlah8jWOnSTrU7rREOUrWQvb3jo9CAr0NBj5tCEii52g8C2dNf3/uo4OYlsrUMs3YILka11iGytQzfZ6rRONETZint5w0OnayrQEKXbj1gV0+1q0/t/fUeX/uo4OYlsrUOUbsGFyNY6RLbWoZtsdVondJbtyfYRpbvhoNM1FWiI0u1HKorptgV4THd976+Ok5PI1jpE6RZciGytQ2RrHbrJVqd1QlfZnqi/J1O6dY3p1mWMrECnayrQEKXbj1RUp9tP3uXaxRLpEndcWfKQ+owu8fI6TvyidAsuRLbWIbK1Dt1kq9M6oatsT1Xp9r4X0tXSrcv9nBXolCch0JDR8CNi6fagS391WvBdiGytQ5RuwYXI1jpEttahm2x1Wid0lW1NLN26Kt26jJEV6HRNBRqidPuRiizdEtNdv/ur4+QksrUOUboFFyJb6xDZWodustVpndBVtqJ0BxY6XVOBhijdfsSnZFiAW7p1cYGWkmHWoaOLk5QME1yIbK1DZGsduslWp3VCV9meqL9SMqzhIUp3/UWPmUMTfCzduF77p21dlFgXuiiGOsZ06/IARseJXyzdgguRrXWIbK1DN9nqtE7oKtuaWLp1TaSmy/2cFQTyb6/vyMj4EZ+Ybuc85a/JuSFO9vUBnRZ8FyJb6xClW3AhsrUOka116CZbndYJXWUr7uWBhW7naSAhSrcfkTrdHnTpr04LvguRrXWI0i24ENlah8jWOnSTrU7rhK6yPVWl2/u9KN36odt5GkiI0u1HKrrYDT/VDNPtItKlvzot+C5EttYhSrfgQmRrHSJb69BNtjqtE7rKVpTuwEK38zSQEKXbj1hp6ZaYbmvwlmd976sLXSZUnW6mXIjSLbgQ2VqHyNY6dJOtTuuErrKtjtLtuh+SmG790E1fCCQC96y0ANfFrpRu3K/9QUOc7OsDOi34LkS21iFKt+BCZGsdIlvr0E22Oq0TuspWLN2BhW7naSAhSrcfcdfmttk8lm4/SVi3i0iXJ21SMsw6dCoF40JKhgkuRLbWIbK1Dt1kq9M6oatsT7VkGIjSrTO66QuBhB4zhyb4lAxzWbolpruOe3JidHrK7kJkax1i6RZciGytQ2RrHbrJVqd1QlfZiqU7sNDtPA0kROn2I75Kt2Qv9/5fX9FpwXchsrUOUboFFyJb6xDZWodustWln6CvbGuidOsa063LGFmBbudpICFKtx/xTaSmttn8dNLr5takiwu096RU3/vqQhfXfVG6rWuzNtoOdES21iGytQ7dZKtTf3XqK9RM6XbdZ+hq6dblfs4KdNMXAgkZET9SkaXbT97lDXKyrw+IYmgdIlvr2qyNtgMdka11iGytQzfZ6tRfnfoK4l4eqOh2ngYSonT7ESst3bpdRLr0VxRD6xDZWtdmbbQd6IhsrUNkax26yVan/urUVxClO1DR7TwNJETp9iPe9QEDPaZbFxdoyV5uHTplpXUh2csFFyJb6xDZWodustWpvzr1FSR7eaCim75wKrz66qukpaURFhbGgAEDWLZs2Qn3/+yzz+jYsSNhYWF069aNH374oZZ6WjF6zByaUBsx3bpcRLoohjrGdOsyoYql27o2a6PtQEdkax0iW+vQTbY69VenvoJ/Yrp1TaSmy/2cFeimL1SV6dOnM2nSJB599FFWrFhBjx49GDlyJBkZGRXuv2jRIq6++mquv/56Vq5cyejRoxk9ejTr1q2r5Z57CKqzb24g/Pe//yUmJgaAgoICwBnTjX+fDjbEyb4+IIqhdYhsrWuzNtoOdES21iGytQ7dZKtTf3XqK/jHvfzVV18lISHBoh76n9zcXECfMbIC3c7TqvL8889z4403ct111wEwZcoUZsyYwdSpU3nggQfK7f/f//6XUaNGce+99wLw2GOPMWvWLF555RWmTJlSq313IUp3DXnuuefKbYuIiMCRp177y9IdGRnp87++o0t/DcMgPDycwsJCwsLC6ro7VUIX2YaGhmK3291/OhAVFQX4V7ZWtFkbbQc6Ilvr0GUO0xHdZBsREQHo0V/dZFuV/lY2z0VGRpKZmclLL71kXQctRJcxsgLdztOcnByys7Pd70NDQwkNDfXZp6ioiOXLl/Pggw+6t9lsNkaMGMHixYsrbHfx4sVMmjTJZ9vIkSP5+uuv/df5U0SU7hoyduxY96QF0LNnT5o3b87mTcrdwV8eLq+99hqrV6+mc+fO/mnQYu644w6io6PdT6TqMx999BE5OTnExsbWdVeqxP3330+LFi248sor67orJyQiIoL333+f4OBggoL0mGoeffRRevTowQUXXOC3NseMGcOePXu44oor/Nami5EjR/LEE09w/vnn+73tQGfUqFE88cQTfj0XBIWV10SgM27cOLKzsxk/fnxdd6VKJCYm8tZbb5GUlFTXXTkpN998MzabjRtuuKGuu1Il2rVrx0svvUSnTp0q3adv3748++yzDBo0yGf722+/XafKSU3o2rUrrVq1qutu1Bmpqam89tprpKWl1XVXqkRZvebRRx/ln//8p8+2w4cPU1paWm6eSEpKYuPGjRW2m56eXuH+6enpNe90NTFM3bIk1BP27t1Lamoqe/bsISUlpdzn8zZmcN17v9OteSzf3TG0DnooCIIgCIIgCIJQv3DpUevXr6d58+bu7RVZuvfv30/z5s1ZtGiRzwOi++67jwULFrB06dJy7YeEhPD+++9z9dVXu7e99tpr/Otf/+LgwYMW/KKTo4f5SUMcfs5eLgiCIAiCIAiC0FCIjo5258aqjISEBOx2ezll+eDBgyQnJ1d4THJy8intXxsEbno/i3FlL29oiQwEQRAEQRAEQRBqg5CQEPr06cOcOXPc2xwOB3PmzCkXGuFi0KBBPvsDzJo1q9L9awOxdFuEWLoFQRAEQRAEQRBqxqRJk7j22mvp27cv/fv358UXXyQvL8+dO2r8+PE0b96cyZMnA3DXXXcxfPhw/vOf/3D++eczbdo0/vjjD9588806+w2idFuEKZZuQRAEQRAEQRCEGnHllVdy6NAhHnnkEdLT0+nZsyczZ850J0vbvXu3T332wYMH88knn/CPf/yDv//977Rr146vv/6arl271tVPEKXbKkyxdAuCIAiCIAiCINSY22+/ndtvv73Cz+bPn19u2+WXX87ll19uca+qjsR0W4TEdAuCIAiCIAiCIAiidFuEidK6ReUWBEEQBEEQBEEIXAJS6Z48eTL9+vUjOjqaxMRERo8ezaZNm/z6HS5Lt00s3YIgCIIgCIIgCAFLQCrdCxYsYOLEiSxZsoRZs2ZRXFzMOeecQ15ent++wx3THZASFgRBEARBEARBECBAE6nNnDnT5/17771HYmIiy5cv57TTTvPLd3hKhomlWxAEQRAEQRAEIVAJSKW7LFlZWQA0atSo0n0KCwspLCx0v8/JyTlhm66SYYIgCIIgCIIgCELgEvDOzw6Hg7vvvpshQ4acsHbb5MmTiY2Ndf917tz5xO1KTLcgCIIgCIIgCELAE/BK98SJE1m3bh3Tpk074X4PPvggWVlZ7r/169efcH+H1OkWBEEQBEEQBEEIeALavfz222/n+++/Z+HChaSkpJxw39DQUEJDQ93vs7OzT9y41OkWBEEQBEEQBEEIeAJS6TZNkzvuuIOvvvqK+fPn06pVK79/R1GpAxBLtyAIgiAIgiAIQiATkEr3xIkT+eSTT/jmm2+Ijo4mPT0dgNjYWMLDw2vcfkZ2Aa/O2wpASnxEjdsTBEEQBEEQBEEQ9CQgle7XX38dgNNPP91n+7vvvsuECROq1IbDoSzZ416eSXhsgs9n6dkFZGQX0qJxBFd0imTv3r017rMgCIIgCIIgCILuHDhwAPDoU4GAYZpS3Ko6/P777/Tv37+uuyEIgiAIgiAIgqAdy5Yto1+/fnXdjVpBlO5qUlJSwsqVK0lKSsJmC/gk8PWCnJwcOnfuzPr164mOjq7r7giVIOOkDzJWeiDjpA8yVnog46QPMlZ6UHacHA4HBw8epFevXgQFBYbjtSjdQoMhOzub2NhYsrKyiImJqevuCJUg46QPMlZ6IOOkDzJWeiDjpA8yVnog4yR1ugVBEARBEARBEATBMkTpFgRBEARBEARBEASLEKVbaDCEhoby6KOPEhoaWtddEU6AjJM+yFjpgYyTPshY6YGMkz7IWOmBjJPEdAuCIAiCIAiCIAiCZYilWxAEQRAEQRAEQRAsQpRuQRAEQRAEQRAEQbAIUboFQRAEQRAEQRAEwSJE6RYEQRAEQRAEQRAEixClWxAEQRAEQRAEQRAsQpRuwRImT55Mv379iI6OJjExkdGjR7Np0yaffdLT0xk3bhzJyclERkbSu3dvvvjiixO2u3DhQi688EKaNWuGYRh8/fXXJ9z/lltuwTAMXnzxxZP2ef78+fTu3ZvQ0FDatm3Le++9V26fV199lbS0NMLCwhgwYADLli07abv1Hd3G6tdff2XIkCE0btyY8PBwOnbsyAsvvFBuv4Y2VnU5ThMmTMAwDJ+/UaNGnbTPgXhN6TZOgXo9Qd3PfRs2bOCiiy4iNjaWyMhI+vXrx+7du0/Y9po1axg2bBhhYWGkpqbyzDPPlNvns88+o2PHjoSFhdGtWzd++OGHEwuinqPbOG3atIkzzjiDpKQkwsLCaN26Nf/4xz8oLi722a+hjRPU7ViVnftcf88+++wJ25Z1qv6PU4NZp0xBsICRI0ea7777rrlu3Tpz1apV5nnnnWe2aNHCzM3Nde9z9tlnm/369TOXLl1qbtu2zXzsscdMm81mrlixotJ2f/jhB/Ohhx4yv/zySxMwv/rqq0r3/fLLL80ePXqYzZo1M1944YUT9nf79u1mRESEOWnSJHP9+vXmyy+/bNrtdnPmzJnufaZNm2aGhISYU6dONf/880/zxhtvNOPi4syDBw9WWS71Ed3GasWKFeYnn3xirlu3ztyxY4f54YcfmhEREeYbb7zh3qchjlVdjtO1115rjho1yjxw4ID77+jRoyfsb6BeU7qNU6BeT6ZZt2O1detWs1GjRua9995rrlixwty6dav5zTffnFCmWVlZZlJSkjl27Fhz3bp15v/+9z8zPDzcZ6x+++030263m88884y5fv168x//+IcZHBxsrl27tnpCqgfoNk7btm0zp06daq5atcrcuXOn+c0335iJiYnmgw8+6N6nIY6TadbtWHnPewcOHDCnTp1qGoZhbtu2rdJ2ZZ3SY5wayjolSrdQK2RkZJiAuWDBAve2yMhI84MPPvDZr1GjRuZbb71VpTZPpMjt3bvXbN68ublu3TqzZcuWJ1Xk7rvvPrNLly4+26688kpz5MiR7vf9+/c3J06c6H5fWlpqNmvWzJw8eXKV+qsL9X2sKuKSSy4xr7nmGvf7QBir2hyna6+91rz44otPqX9yTSnq+zhVRCBeT6ZZu2N15ZVX+si4Krz22mtmfHy8WVhY6N52//33mx06dHC/v+KKK8zzzz/f57gBAwaYN9988yl9V32mvo9TRdxzzz3m0KFD3e8DYZxMs/bvJ7y5+OKLzTPPPPOE+8g6pajv41QROq5T4l4u1ApZWVkANGrUyL1t8ODBTJ8+naNHj+JwOJg2bRoFBQWcfvrpNfouh8PBuHHjuPfee+nSpUuF+5x++ulMmDDB/X7x4sWMGDHCZ5+RI0eyePFiAIqKili+fLnPPjabjREjRrj3aSjU97Eqy8qVK1m0aBHDhw8HAmesanOcQLngJSYm0qFDB2699VaOHDni87lcUxVT38epLIF6PUHtjZXD4WDGjBm0b9+ekSNHkpiYyIABA8q5Yk6YMMHnexYvXsxpp51GSEiIe9vIkSPZtGkTx44dc+9zouuuIVDfx6ksW7duZebMme5rCgJjnKD25z8XBw8eZMaMGVx//fU+22Wdqpj6Pk5l0XWdEqVbsByHw8Hdd9/NkCFD6Nq1q3v7p59+SnFxMY0bNyY0NJSbb76Zr776irZt29bo+55++mmCgoK48847K92nRYsWNG3a1P0+PT2dpKQkn32SkpLIzs7m+PHjHD58mNLS0gr3SU9Pr1F/6xM6jJWLlJQUQkND6du3LxMnTuSGG24ACIixqu1xGjVqFB988AFz5szh6aefZsGCBZx77rmUlpa695Frqjw6jJOLQL6eoHbHKiMjg9zcXJ566ilGjRrFzz//zCWXXMKYMWNYsGCBe7+mTZvSokUL9/vKrinXZyfap6GMlQ7j5GLw4MGEhYXRrl07hg0bxr///W/3Zw19nKD25z9v3n//faKjoxkzZozPdlmnyqPDOLnQfZ0KqusOCA2fiRMnsm7dOn799Vef7Q8//DCZmZnMnj2bhIQEvv76a6644gp++eUXunXrVq3vWr58Of/9739ZsWIFhmFUut8HH3xQrfYbOjqN1S+//EJubi5LlizhgQceoG3btlx99dXV6otu1OY4AVx11VXu1926daN79+60adOG+fPnc9ZZZwFyTVWETuMUyNcT1O5YORwOAC6++GLuueceAHr27MmiRYuYMmWK23ozefLkGvyiholO4zR9+nRycnJYvXo19957L8899xz33XdftfqiI7U9/3kzdepUxo4dS1hYmM92WafKo9M4ab9O1bV/u9CwmThxopmSkmJu377dZ/vWrVtNwFy3bp3P9rPOOqvKMU1UEC/ywgsvmIZhmHa73f0HmDabzWzZsmWlbQ0bNsy86667fLZNnTrVjImJMU3TNAsLC0273V7u+8aPH29edNFFVepvfUeXsaqIxx57zGzfvr1pmg1/rGp7nCojISHBnDJlSqWfB/o1pcs4VUQgXU+mWftjVVhYaAYFBZmPPfaYz/b77rvPHDx4cKVtjRs3rlzM/ty5c03AnTAvNTW1XF6MRx55xOzevXuV+luf0WWcKuLDDz80w8PDzZKSEtM0G/Y4mWbdzn8LFy40AXPVqlUnbUvWKT3GqSJ0XKfEvVywBNM0uf322/nqq6+YO3curVq18vk8Pz8fUDEX3tjtdvfT5eowbtw41qxZw6pVq9x/zZo149577+Wnn36q9LhBgwYxZ84cn22zZs1i0KBBAISEhNCnTx+ffRwOB3PmzHHvoyu6jVVFOBwOCgsLgYY7VnU1ThWxd+9ejhw5UqH7l4tAvaZ0G6eKCITrCepurEJCQujXr1+5Ej2bN2+mZcuWlR43aNAgFi5c6FN6atasWXTo0IH4+Hj3Pie67nREt3GqCIfDQXFxsbs/DXGcoH7Mf++88w59+vShR48eJ91X1ik9xqkitFyn6k7fFxoyt956qxkbG2vOnz/fpzRAfn6+aZqmWVRUZLZt29YcNmyYuXTpUnPr1q3mc889ZxqGYc6YMaPSdnNycsyVK1eaK1euNAHz+eefN1euXGnu2rWr0mMqyog9btw484EHHnC/d5WNuPfee80NGzaYr776aoVlI0JDQ8333nvPXL9+vXnTTTeZcXFxZnp6ejWlVD/QbaxeeeUV89tvvzU3b95sbt682Xz77bfN6Oho86GHHnLv0xDHqq7GKScnx/y///s/c/HixeaOHTvM2bNnm7179zbbtWtnFhQUuNuRa0qh2zgF6vVkmnU793355ZdmcHCw+eabb5pbtmxxlyr65Zdf3Ps88MAD5rhx49zvMzMzzaSkJHPcuHHmunXrzGnTppUrm/Pbb7+ZQUFB5nPPPWdu2LDBfPTRR7UvRaXbOH300Ufm9OnTzfXr15vbtm0zp0+fbjZr1swcO3ase5+GOE6mWff3E1lZWWZERIT5+uuvV9iOrFMK3capoaxTonQLlgBU+Pfuu++699m8ebM5ZswYMzEx0YyIiDC7d+9erjxBWebNm1dhu9dee22lx1SkyA0fPrzcMfPmzTN79uxphoSEmK1bt/bpq4uXX37ZbNGihRkSEmL279/fXLJkyUkkUf/Rbaxeeukls0uXLmZERIQZExNj9urVy3zttdfM0tJSn+Ma2ljV1Tjl5+eb55xzjtmkSRMzODjYbNmypXnjjTeWW8jkmlLoNk6Bej2ZZt3Pfe+8847Ztm1bMywszOzRo4f59ddf+3x+7bXXmsOHD/fZtnr1anPo0KFmaGio2bx5c/Opp54q9/2ffvqp2b59ezMkJMTs0qXLCW+SdUC3cZo2bZrZu3dvMyoqyoyMjDQ7d+5sPvnkk+bx48d9jmto42SadT9Wb7zxhhkeHm5mZmZW2I6sUwrdxqmhrFOGaZrmye3hgiAIgiAIgiAIgiCcKhLTLQiCIAiCIAiCIAgWIUq3IAiCIAiCIAiCIFiEKN2CIAiCIAiCIAiCYBGidAuCIAiCIAiCIAiCRYjSLQiCIAiCIAiCIAgWIUq3IAiCIAiCIAiCIFiEKN2CIAiCIAiCIAiCYBGidAuCIAiCIAiCIAiCRYjSLQiCIAiCIAiCIAgWIUq3IAiCIAiCIAiCIFiEKN2CIAiCIAiCIAiCYBH/Dy8yna8X9Gx7AAAAAElFTkSuQmCC", - "text/plain": [ - "

" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "period_hours = 4\n", - "\n", - "plot_data({\n", - " 'NOx': await load_data(period_hours, '4', 'CI-W3W01A3'),\n", - " 'CO': await load_data(period_hours, '8', 'CI-W3W01A1'),\n", - " 'O2': await load_data(period_hours, '5', 'CI-W3W01A2')\n", - "})" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "6137c2f7", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/grezewave/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", - " self.metrics_controller.start()\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", - "/home/grezewave/Documents/projects/sientia/sientia-dataops-laborious_temporal/venv/lib/python3.11/site-packages/sientia_do/observability/sientia_monitoring.py:80: RuntimeWarning: coroutine 'AsyncMockMixin._execute_mock_call' was never awaited\n", - " self.metrics_controller.start()\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
variabletimestampcurrent_valueprevious_valueprevious_predictioncurrent_predictionsensor_inselected_value
02026-01-27 07:40:59+00:00529.374900524.902649523.069031508.5605471.0524.902649
12026-01-27 07:41:29+00:00585.815900529.374900508.560547513.4113161.0529.374900
22026-01-27 07:41:59+00:00624.189453585.815900513.411316560.6661991.0585.815900
32026-01-27 07:42:29+00:00617.834000624.189453560.666199599.8107301.0624.189453
42026-01-27 07:42:59+00:00575.486450617.834000599.810730591.3146361.0617.834000
........................
3072026-01-27 10:33:59+00:00710.051900793.385254688.647400744.0227051.0793.385254
3082026-01-27 10:35:29+00:00763.943000710.051900744.0227050.0000001.0NaN
3092026-01-27 10:35:59+00:00710.051900763.943000744.022705716.5969241.0763.943000
3102026-01-27 10:36:29+00:00705.572100710.051900716.596924672.2628781.0710.051900
3112026-01-27 10:37:59+00:00766.472168705.572100672.262878672.2576291.0705.572100
\n", - "

312 rows × 7 columns

\n", - "
" - ], - "text/plain": [ - "variable timestamp current_value previous_value \\\n", - "0 2026-01-27 07:40:59+00:00 529.374900 524.902649 \n", - "1 2026-01-27 07:41:29+00:00 585.815900 529.374900 \n", - "2 2026-01-27 07:41:59+00:00 624.189453 585.815900 \n", - "3 2026-01-27 07:42:29+00:00 617.834000 624.189453 \n", - "4 2026-01-27 07:42:59+00:00 575.486450 617.834000 \n", - ".. ... ... ... \n", - "307 2026-01-27 10:33:59+00:00 710.051900 793.385254 \n", - "308 2026-01-27 10:35:29+00:00 763.943000 710.051900 \n", - "309 2026-01-27 10:35:59+00:00 710.051900 763.943000 \n", - "310 2026-01-27 10:36:29+00:00 705.572100 710.051900 \n", - "311 2026-01-27 10:37:59+00:00 766.472168 705.572100 \n", - "\n", - "variable previous_prediction current_prediction sensor_in selected_value \n", - "0 523.069031 508.560547 1.0 524.902649 \n", - "1 508.560547 513.411316 1.0 529.374900 \n", - "2 513.411316 560.666199 1.0 585.815900 \n", - "3 560.666199 599.810730 1.0 624.189453 \n", - "4 599.810730 591.314636 1.0 617.834000 \n", - ".. ... ... ... ... \n", - "307 688.647400 744.022705 1.0 793.385254 \n", - "308 744.022705 0.000000 1.0 NaN \n", - "309 744.022705 716.596924 1.0 763.943000 \n", - "310 716.596924 672.262878 1.0 710.051900 \n", - "311 672.262878 672.257629 1.0 705.572100 \n", - "\n", - "[312 rows x 7 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlsAAAGdCAYAAAAlnLZFAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsfXe8HGW9/vPOzNZTc9IT0ggtIITQYiiCEAzIRUC4VzFXpCsSiiggNXQ0ShFQUDpKUUG5eOEHYqRcIJTQIZCQkJ6ck3Jy6tYpvz/eMu/MzrZzdnf2wDyfT7K7s1PenTPlmef7fL9fYlmWhQABAgQIECBAgABVgeL3AAIECBAgQIAAAb7ICMhWgAABAgQIECBAFRGQrQABAgQIECBAgCoiIFsBAgQIECBAgABVREC2AgQIECBAgAABqoiAbAUIECBAgAABAlQRAdkKECBAgAABAgSoIgKyFSBAgAABAgQIUEVofg9gINB1He+++y5Gjx4NRQn4YoAAAQIECDAUYJomOjo6MGPGDGjakKQgA8KQ/KXvvvsu9ttvP7+HESBAgAABAgQYAN58803su+++fg+jZhiSZGv06NEA6B9r7NixPo8mQIAAX1Qc+pfZIMTCNfvegQMmT/N7OAECDHls3LgR++23n7iPf1kwJMkWDx2OHTsW2223nc+jCRAgwBcRumEgMjwEAGgcOSy41gQIUEF82SxAX65fGyBAgAAlImPq4n1Kz/g4kgABAgx1BGQrQIAAATyQ0QOyFSBAgMogIFsBAgQI4IGsYYj3qWzax5EECBBgqGNIerYCBKg1LMuCruswpBtwgC82+pMJjA2zBBzdRCqV8ndAVYKqqtA0DYQQv4cSIMAXFgHZChCgCDKZDDZu3IhEIuH3UALUELpp4OIdLgYANGqtWLlypc8jqh7i8TjGjh2LcDjs91ACBPhCIiBbAQIUgGmaWLlyJVRVxbhx4xAOhwMF4EuCdDYD9FkAgJbwSIxqaPV3QFWAZVnIZDLYvHkzVq5ciR133PFLlyUWIEAtEJCtAAEKIJPJwDRNTJgwAfF43O/hBKghLJVASVPioYVCiEajPo+oOojFYgiFQli9ejUymcwX9ncGCOAngkeYAAFKQPC0/+WDZdnvTcv0byA1QHB8BwhQXQRnWIAAAQJ4wJTYlgWrwJwBAgQIUBgB2QoQIEAAD8gEy7ICshUggB94+eWXcfTRR2PcuHEghODJJ58susyLL76IvfbaC5FIBDvssAMeeOCBqo+zGAKyFSBAgIri5JNPxrHHHuv3MAYNTrZ+u+C3OOKgw3weTYAAX0709/dj+vTp+O1vf1vS/CtXrsRRRx2Fr3/963jvvfdw/vnn4/TTT8dzzz1X5ZEWRmCQDxAgQEXxm9/85guhBDl/wtD/PQECDEUceeSROPLII0ue/6677sKUKVNw0003AQCmTZuGV155BbfccgvmzJlTrWEWxZAmW7e+sBJNrd1V3YamEBy/93YY2RjBg6+tQm9aL75QFTCiMYJTD5iCD9Z14bmPO3z1kLTFwzj1wCn4tL0HT3/QXtd+FgKCObuNxszth/s9lLpHJpOpSJ2llpaWCozGGy98ugkvf7Z5wMtHQyq+/9VJ6Epk8bd31sEoQArTehb92R4sIbsis2szrv7HxwPerhcaIxp+sP9kjGiMIK0buO+VVdjUW/vCqduPaMAJe47Gy6v68Oiny6Bqas3HwDGxLY6T95+M5z5uxxsrO30ZAwHBUXuMwS5jmnHfKyvRmfCnVVNTNIRT9p+MYQ32OTnY43/AY2HH6ua+NJ54ex10c+DX/N4u+nft7e1FT0+PmB6JRBCJRAY91kWLFmH27NmOaXPmzMH5558/6HUPBkOabP3lnXYokZ7iMw4Syzr6sN+UNtz27+VV31YhTGyL4+bnl2Hlln5fxwEA41pj+MPLn2NpR6/fQymKF5Zuwgs/O6Ri67MsC8ls7SvJx0JqWTW+DjnkEHzlK18BAPzxj39EKBTCWWedhWuuuQaEEEyePBmnnXYaPvvsMzz55JP49re/jQceeACvvPIKLrnkEixevBgjRozAcccdhxtvvBENDQ249NJLsXDhQrzxxhuObU2fPh3HH388rrzySpx88sno6uoS3op0Oo0LL7wQjz32GHp6erDPPvvglltuwb777gsAeOCBB3D++eejq6tLrO/JJ5/EcccdJxSy999/H+effz5W7PljKJHBleD4+NNl+NebHyM8cY/SFiBTgClTcP+rqwa1XS9oioLzZu+IVz7bgl8++2nF118qdh4Zwy9f3oRB3EMrhukTWnHuo+8hY/iXAfraii04/aDtcdPzy3wbAwA0RlSc+bWp4vO5j77r2wN/JKTizZWdeGnZ4MiemabFoXfddVfH9Pnz5+Oqq64a1LoBoL29HaNHj3ZMGz16NHp6epBMJhGLxQa9jYFgSJOtk2aOR8uwtqqtf8Wmfjz7cTv60zr62AE+fbsWHLjjiKpt0wtPf7ARq7YmHOP4zj4TMKKp9tWen/2oHSs296M/Y4/l+L22w5iWwT+RVBpbejP48+K1YpyVQjJrYNcrax//X3LNHMTD5Z2yDz74IE477TS8+eabWLx4Mc4880xMnDgRZ5xxBgDg17/+Na688krMnz8fALBixQocccQRuO6663Dfffdh8+bNmDdvHubNm4f7778fc+fOxY033ogVK1Zg6lR6E/j444/xwQcf4IknnvAcw0UXXYQnnngCDz74ICZNmoQFCxZgzpw5WL58OdraSjt/586dixkzZmAlI1oHjspizIg2jB49quByf/vb36BpGr71rW9h8apteGNlJ5YsXYHhY8ahF8A3dx+D9Z++h8mTJ6OpsRGbNm/GM08/jf1mzsSe++yFvmwP3n79baxesRo/PONMAMCf/vQnjB49GocffrjYzuN/fRyRaBRHH/0fAIBnnn4Gm7dswaFf/zoam5qwdOlSvPTSSzjjjDPQ1jYMry7fivfWdqE/Q49NfoxuNyyGY/YcV9I+qQQeWrQavSkd2xJZQbTOOmQqFB/q9j78xhp0JbLoTmQF0frh17aHptZuMB09aTz+9jr0pXX0s7/J1JENOOIrY2o2BgB4edkWfLi+G31p50MdJ1qnHDAZ8XBtFMhXPtuC99d1O/bJEbuNwdRRDQNaX/e2TlwPYMmSJRg/fryYXglVq54xpMnWjw6ciO22265q63/2o4149uN2WLD9G/tMbsOFc3ap2ja9sLS9D6u2JhzjOPmAyZg2trmm4wCAVVsSWLG53+Fn+f6sSdhzQmvNx1IMn2zswZ8Xr8UXwD40YEyYMAG33HILCCHYeeed8eGHH+KWW24RZOvQQw/FT3/6UzH/6aefjrlz5wrJfccdd8Rtt92Ggw8+GHfeeSd22203TJ8+HY888giuuOIKAMDDDz+MmTNnYocddsjZfn9/P+6880488MADwndx99134/nnn8e9996LCy+8sKTfsWbNGvzsZxfi/5j4c9sPv4m2huIPGzsmd8X3v/99PPerebg/rOGNlZ1Yu24tdv3qTujtBY6bsR0On7u3Y5lfpz7FY4/dgbsvegHtiTXoW7QUG5a+hAvn3AwAePrGV7Dr8D1x4ZxzxDKv3rkYra2tuHDOz7BmzRpcctRFWLNmDcaN48Tpq5g9+48gH/4DF95wA3TjE7y3tksod7zMxJQRDTW9vjz1/gb0pnSHonXhN3aG4gPbevajdnQlso6SGz85fCdEQ7ULa763tguPv70OlmVnoO4ytrnm1/z+9Mf4cH23w/sovz/n0B1LOv4rgVR2Cd5f1033CZt27IzxAyag69atw/UAmpqa0Nxc+XvYmDFj0NHR4ZjW0dGB5uZm31QtYIiTreqDXnAsyxK+JD8atfDIET3XLMe02g+Gj8USJ3+9Nq+x91Fl2VYspGLJNbU3WsYGcNP56le/6gg9zpo1CzfddJNoqL3PPvs45n///ffxwQcf4OGHHxbTLMsSbYumTZuGuXPn4r777sMVV1wBy7Lw6KOP4oILLvDc/ooVK5DNZnHAAQeIaaFQCPvttx8++eSTkn/HBRdcgDPOPBPjL/gbAGD1qlVo222nost985vfRCgUwlNPPQWM3odtP4zmlhagtxsEwJ///GfcdtttWLFiBfr6+qDrOpqbmwds8v/www9hGAZ22sk5vnQ6jeHDmXfQcU4DJouYKTU+sYl0jRPTfDqh+XHqZyhT/umWx7SajYNtVCae8n6pJRfmm7IgXfPr9aIPeo175plnHNOef/55zJo1y6cRUQRkqwAEyRH/+XOQOQ92Ps2fo90ei3QxqtMTz76RVHi9hJQdzqtXNDQ4QwF9fX344Q9/iHPPPTdn3okTJwIATjzxRFx88cV45513kEwmsXbtWnznO98Z8BgURckhNtls1vH5qquuwn9990R88wHqm5w1ayYeffA+HHfccQXXHQ6HccIJJ+CRRx7BERdQsjVl++3F90uXforz5s7F1VdfjTlz5qClpQWPPfYYy2SSx2S/Lzbevr4+qKqKt99+G6rqJMiNjY0ApGOTTec31VoLSvZNXZ7m77XF9JH4EcfDJJ9W+/2heBBP536p3ZiIdNH3g4D29fVh+XLbL71y5Uq89957aGtrw8SJE3HJJZdg/fr1eOihhwAAP/rRj3DHHXfgoosuwqmnnop///vf+Mtf/oKnn366hqPOxRfjjlEliGNMPsh8OPFkZctvgsN/P5XZ2bQ61bYcZPlLCreR/fXXX8eOO+6YQwI49tprLyxZssQzJMix3Xbb4eCDD8bDDz+MZDKJww8/HKNGeXunpk6dinA4jFdffRWTJk0CQInJW2+9JUKVI0eORG9vL/r7+wX5e++993LXtcMOAOhF9+j/OBr3339/UbIFUL/X4Ycfjr3m/hwAsP3222MbU5I++eQTTJo0CZdddpmYf/Xq1QDyHzcjR47Exo0bxWfDMPDRRx/h61//OgBgxowZMAwDmzZtwkEHHeS5Dqdabb/WXtmiMCv9RDIAyERHTKvxtUUmwX4qW4ogwbKyZeV8XwuIaz7gCwFdvHixOLcACBX9Bz/4AR544AFs3LgRa9asEd9PmTIFTz/9NH7yk5/gN7/5Dbbbbjvcc889vpZ9AAKyVRDOg8zHMCLqYxzydunFqL4lZZss+38j8Qtr1qzBBRdcgB/+8Id45513cPvtt4v6M164+OKL8dWvfhXz5s3D6aefjoaGBixZsgTPP/887rjjDjHf3LlzMX/+fGQyGdxyyy1519fQ0ICzzjoLF154oXgSXbBgARKJBE477TQAwMyZMxGPx3HppZfi3HPPxRtvvOGo+JxMJnHhhRfiW8cdL6a98/ZinHDs0SXtg6997WsYM2YMHnvsUWDnIzB8xAh0dvQBAMaPH4c1a9bgsccew7777ounn34af//73wG4jhvp7aGHHooLLrgATz/9NKZOnYqbb77ZkUm50047Ye7cuTjppJNw0003YcaMGdi8eTMWLlyIPfbYA0cddZRDrQbsG2mtH+bs0J3/lgB+nXOqbDUeg/xg62PITJEeajnk97Uk5fJ11A8CesghhxS8hntVhz/kkEPw7rvvVnFU5SOoIF8AtuXHlpT9uBrZMq5VB8oWH4pV8fBcKdiW6MOZT/0K/7dySdF5A2ULOOmkk5BMJrHffvvh7LPPxnnnnYczzzwz7/x77LEHXnrpJSxbtgwHHXQQZsyYgSuvvFIyelOccMIJ2Lp1KxKJRNFq8b/4xS9w/PHH4/vf/z722msvLF++HM899xyGDRsGAGhra8Of/vQnPPPMM9h9993x6KOPOlLAVVXF1q1bcfoZ9ri/8Y3DcfXVV5e0DwghOPHEE7FxwwY6wbJJzle/+lX85Cc/wbx587DnnnvitddeE8b/fPXjTj31VPzgBz/ASSedhIMPPhjbb7+948kbAO6//36cdNJJ+OlPf4qdd94Zxx57LN566y0RinUrW7zeV83DiOzVFIpFbbcvg2/bkNiWX8OR//b+eLboVuV9Ib+vqQIqH6tDwLNVrwiUrQKQ/QziYuTDqecYh+kj64P8lOPPBfrk/5mPzzP/xKIX/ooPp7xZZG72pFwPxYN8QigUwq233oo777wz57tVq1Z5LrPvvvvin//8Z8H1tra2IpXyLsDpftKMRqO47bbbcNttt+Vd37HHHptD2njGZDgcxqOPPoreVBa7X0XHdcvNN5eVpfbLX/4SU19egRue+RQWbEM6AbBgwQIsWLDAMf/555+P9l5afPHsi87GORf9THwXCoXwu9/9Dr/73e/ybi8UCuHqq6/OSwjdxnR+iNY6jAgXwakHS4Bf3iS6PT4G/9RGoHgYsbZcy1Yc64GUD1UEylYBCCkXdjaiH/VnbJ+U/bzlxzjodu19ArFPajeYz5Ov0zdqsui8fB99eanWFwvObKzyjzmZ4NjnUf71WAU+DRbuY5OTLqXGV2SFOEmfX9cVeSx+eZPkMTg8qXUSRhzs8T/wsdBX+T7oVxLFUEZAtgrBEb9nk3zNRoS4Ovt2sPu8Tyyl9Ir1Yh8FbOsLAdm3obI7wJFHHonGxkbPfzfccINjeflwKMWPI4eSKt6SynUz5eqrfwZ51wQfIFQlU57mj7Il/8X9UPt4nTOZeHod/7UA8brm12zrXxwEYcQCkENm9jQ/wojS05YYhz/wztap5Ylf+k3PQVK/hHjxxRf9HkJF4VVn6J577kEy6a1y5qtOX/L5bOX9MGjkGuTp55qTLVe4ys8woldtqZqPQSoX46+yRV/ro86WvTE/y2EMdQRkqwA8sxF9Vrb8LipXL9k6pcArlTzA0IWXl0du91EM3ins+ed3qFkVPobcBnnf6my5atH5apCvg7E41E8fC1nXY50tv7IRvygIwogFYCtb/h5k3ge7zwZ5+HziGY1FZ3EXjgwwtDFYQuI8n4vfSB2tUiqubDmPTd/qbOUoW/6hHsbiOEbqQNmyHMqWvxmr9fCwP5QRkK0CkA8ou/SDD2FE6b3fT6Ce8fsajWVrwvZraVZL0fnd6kGAoY3BtrRxerb4xPzzV/OwyVf6wa/wTD1kmbn9Y74WkJan+Wgdkf1rfrV0cvsLgfrIWh1qCMhWATji9772Rswdh19wxu9rq20t6Vgr3muk9A7xfu+zAJWB/WQ/QLLF3zi8jwW1LWnh6ni2+Db8a9fjzEb09SZaFwVWZc+WfyoOP8YNT2XLnyQKR7u4gGuVjYBsFYD9lOPvQVZPB7tnSLNGY1neuV68t2AWmJMiULa+WDAHefOzPVulNdR1ev2q69nyv12Pa4IPkEN4gM+erToJI3rV2fK1qr6PosNQR0C2CkDORvTVK+U42Nkkn9iW4yZR4zTg1d12P7rSyFbg2foigd93Bpr2TrzOo0LbK/BpsCCu0Iwo/VBjaasefFK5Y2Gf/QjfsVc529qPvcKPca92PbUs+wC4MtDrgJQPVQRkqxA8PB7+KFsSaagxwfEaDR9GrYnf+t4O8d6ySiBbYubqjCeAN04++eSiLXwGgkqFER0PCgXXVf6Bc9VVV2HPPfcUn4vti9zSD7nzVGt/Ah4Epw6ULaHm+KJs2Rd9f7MinSFV+b1fSRS+iw5DHEHphwJwttTw07MFNg5JxvU9jCj7PGqDzclN4n1ZYcSAbdUUv/nNb6pSbmPQhMDREaL4upzHzcB+T759ka/0w113/g7/OeWHDsJWrf0JuK9x/t5EbYLBPvsxBvZKr/j+XfMVFwmW3/vU0QkoMfwewBsB2SoAr+yluvFs1VPphxoNpSu9WfpUirKVK8UHyI9MJoNwODzo9bS0FM8UHQgqqWwVelDg+8F93JiWVfa28+2L3NIPluO1lHVUAoNVtkzTxKF/PBX9RhcOGjsHN8w+DdHQwI6hwLNlw6t1UX0pWwHKRRBGLABH9lJJYYcqjcPTs1XzYeSOpcbEr8/oFO/LU7YqDMsCMv21/1cmazzkkEMwb948zJs3Dy0tLRgxYgSuuOIKcTObPHkyrr32Wpx00klobm7GmWeeCQB45ZVXcNBBByEWi2HChAk499xz0d/fDwC49NJLMXPmzJxtTZ8+Hddccw2A3LBXOp3Gueeei1GjRiEajeLAAw/EW2+9Jb5/4IEH0Nra6ljfk08+6TjX3n//fZx00g8AANs6O7H33ntj8eLFRfcBX/eTTz6JK6+8EgDw2muvIZvVAdBjhIf+7rnnHkyZMgXRaBQA0NPdhSvPvxIH7XIQZk6ZicMOOxTvv/++Y/2/+MUvMHr0aDQ1NeG0007Lac7t3hemaWLBggX41YJfAgD++te/4vrrr7cVDMvCjBkzQAjBIYccMqD9+eKLL4IQgoULF2KfffZBPB7H/vvvj6VLl+bsH3fortwzeV3PVmzF20ipK/D8pt/h+pceKXMN0lhy/GN+eLY8bBI+jMOrzpZf/Ssd/kIf74NDHYGyVQCOitM+hqK8yi34dag7L0a1fQLNWF3ifSl/D/eTcsWQTQA3jKvsOkvBpRuAcENZizz44IM47bTT8Oabb2Lx4sU488wzMXHiRJxxxhkAgF//+te48sorMX/+fADAihUrcMQRR+C6667Dfffdh82bNwvCdv/992Pu3Lm48cYbsWLFCkydOhUA8PHHH+ODDz7AE0884TmGiy66CE888QQefPBBTJo0CQsWLMCcOXOwfPnyvC113Jg7dy522vcQrAfQNqwVP//5zxEKhUpaNpFI4Prrr8epFyzAPe8nkM3qaO/oAGLDwI+S5cuX44knnsDf/vY3qKoKADj75B8iFFVx52N3oqmpCc//+V847LDDsGzZMrS1teEvf/kLrrrqKvz2t7/FgQceiD/+8Y+47bbbsP322+cdyyWXXIK7774bx19+F57fBBz0ta9hdHQNtoq0RBP/+te/sNtuu+VVGUvdn5dddhluuukmjBw5Ej/60Y9w6qmn4tVXX3WuzK2glHkuJ7Jpx+f2/s155iwOfm0xTMfQagpnApCfChvfF/a1y/CrzhaDH9GMLxICZasA5PRbv3qXAYDC/kqmafmeoi0/cdXSQ2CaJkypCbUFo/hCHr6HLxsmTJiAW265BTvvvDPmzp2Lc845B7fccov4/tBDD8VPf/pTTJ06FVOnTsWNN96IuXPn4vzzz8eOO+6I/fffH7fddhseeughpFIp7Lbbbpg+fToeecRWMB5++GHMnDkTO+ywQ872+/v7ceedd+JXv/oVjjzySOy66664++67EYvFcO+995b8O9asWYOZX/0qAEDTNPznf/4npk+fXtKy2WwWd9xxhyCHe+41A+k0JQn8eM5kMnjooYcwY8YM7LHHHnjllVfw/jvv4eZ7b8ZX9vwKJk2dhBsX/BKtra14/PHHAQC33norTjvtNJx22mnYeeedcd1112HXXXfNO47e3l785je/wYIFC7DvPnsDAEaOHIXTTz/dJjuWheHDh2PMmDGeRLSc/Xn99dfj4IMPxq677oqf//zneO2113KUN/77B3oj7884yVZST5S1vAy3suUXqeBj8NM7Vqhdj1/NuU2rPrJWhyrKVrZ6e3txxRVX4O9//zs2bdqEGTNm4De/+Q323XdfAPQmPH/+fNx9993o6urCAQccgDvvvBM77rijWEdnZyfOOecc/OMf/4CiKDj++OPxm9/8Bo2NxVuw1BJ+Vkt3jYSOQ5ri14XIL0m5va8LRNGlKcXDiFXbR6E4VZlqjVC87EW++tWvOv4+s2bNwk033QTDoGR1n332ccz//vvv44MPPsDDDz8splmWBdM0sXLlSkybNg1z587FfffdJ0KSjz76KC644ALP7a9YsQLZbBYHHHCA/TNCIey333745JNPSv4dF1xwAa6++hqMOukWJPr7HMpaMWiahn333RcrF68DADQ2NEJh6hXfN5MmTcLIkSMd+yHR348DdrLHrRAFyWQSK1asAAB88skn+NGPfuTY1qxZs/DCCy94juOTTz5BOp3GYYcdhoXrnB4tm2sVPq7L2Z977LGHeD927FgAwKZNmzBx4kQxPccnVXDruUjpGcfnpO7dFLwUiJu66aMxXbGvtbZyXx9hRL8K3yrSNd9PO81QR9lk6/TTT8dHH32EP/7xjxg3bhz+9Kc/Yfbs2ViyZAnGjx+PBQsW4LbbbsODDz6IKVOm4IorrsCcOXOwZMkS4YWYO3cuNm7ciOeffx7ZbBannHIKzjzzTMfTcn1Azl6qj2xEMc2Hccio9T5ZsbXdOaGEit7yuCzLqtwFgpCyw3n1ioYG5+/o6+vDD3/4Q5x77rk58/Kb9IknnoiLL74Y77zzDpLJJNauXYvvfOc7Ax6Doig5od5sNuv4fNVVV2Hfb3wbZz+1Fpl0Crvuuisee+wxHHfccSVvx+nhI9L/3vth5OhRuP/J+8S08Y0TEQ9HcvxlpSIWi+VM47+aEwyUUNKkVMhhVrv9i3P97hID5Z4iyayTbKWNVJ45i4OHEf1U7719urUfByd9srLlX50ttn35mu/3DWgIoqwwYjKZxBNPPIEFCxbga1/7GnbYYQdcddVV2GGHHXDnnXfCsizceuutuPzyy3HMMcdgjz32wEMPPYQNGzbgySefBECf7p599lncc889mDlzJg488EDcfvvteOyxx7Bhgw9qQQF4FvD08QLgV9d3x1h8UvtWd3c4PpdT1BT48mYkvvHGG47Pr7/+OnbccUfhS3Jjr732wpIlS7DDDjvk/OMeou222w4HH3wwHn74YTz88MM4/PDDMWrUKM/1TZ06FeFw2OEVymazeOutt0TIbeTIkejt7RUmfAB47733ctY1gZG90aNG4dvf/jbuv//+kvaBrutYvHixuJn39fXBZKQm37G71157YcumzVA1FRO3n4iJ20/ElKnbY4cddsCIESMAANOmTfPcv/mw4447IhaLYeHChblFTSWDPFcdvVDK/iwH9rWFfy7vZE66PFuDIls5Bvnaw9E1hE/zxSBff9mIDgJa0xF8MVAW2dJ1HYZhCIWKIxaL4ZVXXsHKlSvR3t6O2bNni+9aWlowc+ZMLFq0CACwaNEitLa2OsIXs2fPhqIoORcujnQ6jZ6eHvGvt7fXc75Kw5bY/c1MkdORxbSaj4Jv159snXU9rMaWwdWBMoqa4stb13TNmjW44IILsHTpUjz66KO4/fbbcd555+Wd/+KLL8Zrr72GefPm4b333sNnn32G//mf/8G8efMc882dOxePPfYY/vrXv2Lu3Ll519fQ0ICzzjoLF154IZ599lksWbIEZ5xxBhKJBE477TQAwMyZMxGPx3HppZdixYoVeOSRR/DAAw+IdSSTScybNw+LF78NAMik03jrrbcwbdq0kvZBKBTCOeecg89Xfg4AeP/9DxAO02tYvmN39uzZ2HPfvXDuSefi1Rdexfo167Hotddw2WWXiSzI8847D/fddx/uv/9+LFu2DPPnz8fHH3+cdxzRaBQXX3wxLrroIixm2YNbtmzBvffeK26kmqri2WefRUdHB7q7uwe0P8uB+9pStrLlCiNmzMGHEe2x+JiN6Ley5eE39a/OlnTND8KIA0ZZZKupqQmzZs3Ctddeiw0bNsAwDPzpT3/CokWLsHHjRrS301DP6NGjHcuNHj1afNfe3p7zFKxpGtra2sQ8btx4441oaWkR/wbyBDcQOJURHzNT3PK6T+OQt+uoVVSDsbT3bwEAaFYrm1J66QegChmJQwQnnXQSkskk9ttvP5x99tk477zzRIkHL+yxxx546aWXsGzZMhx00EGYMWMGrrzySowb58y+POGEE7B161YkEomi1c1/8Ytf4Pjjj8f3v/997LXXXli+fDmee+45DBs2DADQ1taGP/3pT3jmmWew++6749FHH8VVV10llldVFVu3bsVVV9PSEps2deDII4/E1VdfXdI+iMfjuPjii/H7u+6i69M0DB9Ozef5jl1CCO5+9AHsPWtvXHHuFTjqq0fh1JNOxurVq8X17Tvf+Q6uuOIKXHTRRdh7772xevVqnHXWWQXHcsUVV+CnP/0p/t8zzwAAXn3tNWzatEkcn3PmfAO///3vMW7cOBxzzDGe6yi2P8uB17WlHKRdZCtrpvPMWf5YfM1GhN9FTTnpqx9lqx4e9ocyyvZs/fGPf8Spp56K8ePHQ1VV7LXXXjjxxBPx9ttvV2N8AGi6tGzAXb9+fU0Il2we9fNW7ZbXAX8UNrpdCqfMXn1sSVCyFVVa0YeNJXq2JLJctZHVN0KhEG699VbceeedOd+tWrXKc5l9990X//znPwuut7W1NSezjUNWpQCq6Nx222247bbb8q7v2GOPzSFtvDxFOBzGo48+isWrOnHCXYswZfIk3H7hyQXH58a3v/1tmJP2w8/++j722XdfLG3vAUATLq666ioHueNoaGrEpTdeiktvvBQAMDY+EW3xJsc8l156KS699FLHtF/+8pfivXtfKIqCyy67DOMPWYmr/rEE/3H00bjke3vh8ic/BADsvdde+NOaNY5lyt2fhxxySM7DxZ577un9wOEO3ZV5I0/pTm+dbg3xMCJ7taRwhp+kzxFG5P0zfWI6VNkKPFsDRdmlH6ZOnYqXXnoJfX19WLt2Ld58801ks1lsv/32GDNmDACgo8Ppr+no6BDfjRkzBps2bXJ8r+s6Ojs7xTxuRCIRNDc3i39NTU2e81Ua3hXk/ZC2KRxPnz4rW3L8vhZj6UpvAwA0hYazzZdgJHYoW9UYVYBaYrDlVxwPTyXdSJ0HjVnBg0juwUfXTV9rrlqw14F6cVK6k1zp1sCVLQ7Tx2utox8un+RLNiLdplxny7djRM5G5NMCbatsDLjOVkNDA8aOHYtt27bhueeewzHHHIMpU6ZgzJgxWLhwoZivp6cHb7zxBmbNmgWApkZ3dXU5lLB///vfME3TszK1n3DE78U0H8bhISn7F0a04/diWg32Sk+Wkq22yAg2pcww4pdW2/riwCtj7sgjj0RjY6PnvxtuuMGxvPN4YNPKOHbNCh5DshEb8LM6OH0daDaiW9kyMIgwous654+yJXu2/A8jetfZqu1YpOdrn0sgDW2UHUZ87rnnYFkWdt55ZyxfvhwXXnghdtllF5xyyikghOD888/Hddddhx133FGUfhg3bpwIEUybNg1HHHEEzjjjDNx1113IZrOYN28evvvd7+b4QvyGIzOlDuRTsw5i5g6ZnU+rwWCSRhegAqPio/BxAgMo/VCtkdUvXnzxRb+HUFHw419Ofb/nnnuQTHqbstva2tDW1oaTTz4ZQPmZtG6CXqwGVjlwK0q8IoNS87R+l0+qzOUzBvNsGVFATcEimcILFBwL2Fh89Md6qeE+jENV+BhyPVs1L/0gebaCh9aBo2yy1d3djUsuuQTr1q1DW1sbjj/+eFx//fWipstFF12E/v5+nHnmmejq6sKBBx6IZ5991pHB+PDDD2PevHk47LDDRFHTQn4Ov+G/skVfnQZ5v6Qtj7HUYLMZi2Zmbdc0GtgClKZsBY9fXyTw+44cRhk/fnzJy9tZVXaQqOAh4rqvVDTJIqf0g7/m54F7tii5IlYcFlKwyGCULT4WMWXA6xrwGKT3/mag5ypbXsd/TcbCt49A2RoMyiZb//Vf/4X/+q//yvs9IQTXXHONaErrhba2tjosYJoLh63CV88WP/HqQdnyGEsN9omh9IIAmNQ6lm2TVjVXlPyR8C+7svVFw2DblXgqWwXOJPdTvFkNZYttw/bjVGwTpY1D2icDQcagYcQQaUAGnbBIpuh5mXcs7FVWtpZuXYYfP3s1JreOx71HLRjYIMsZg3Rs+amwFaqzVft2PfaNMPBsDRxBb8QCcHq2/Je2HRXkfTbI17Ka/dZEL4hCn6B3GrGdmJ4x9HyLAAg8W180VIqQOJTqMgzyqt6fZ77y4T6PLL+ULdfDU7mbz5r0vAyTRra8if7swNQtd6FXAuDpj5djk/4BFq1fPKB1lj0G6b2fBTwL1dnyjZCj1MSSAF4IyFYB2AeU5euJ537io9P8Odo9x1LloSzfQuuvWWYIoxtbxPSsWbgZtaP0Q8C1hjwGG2qzkzusgZUXMAuT+7LGIhWKBADDL/Oz66ZefjYiJVtR1W53tC3ZN7CxgI/F3hfbkizb0fLueFBpyPu/HpQth2fL9IuQg40FKCn8HsATAdkqAK+wgx9HGd+knAbst7LlGEuVid+qLkq2FLMJIcWOfOsF2poA3tlnAYYuBltnSL5pDMQgX4pPsOSxSKZjwL+0fo6BhqiyjICGlRgsRog6kwNTAN3XFgICRWH7vFZkS7qWyeOoNbyus341opb9haWE3wN4IyBbBSDHzX070OEdv/cLXmMhVT6KNie6AFBfSEjq6ZcpomzJ+LJWkP8iYbD1l5x188onF5U8hnKyEX26vrjrOZW7fR5G1JQQiEl7Z3YNVNlyXVsUAigKO8ctFbpRObKbfxD223pQtrzb9fijbMn3wUDZKh8B2SoARxaGmOaHaYu+mNK1xq8nYHHimbnTqoXOJM1EDJEGh7KVLaJsyftooO1IAtQPjmHlYwaa+i7O3ZKzi52hxkr6/kSYiK/b97T+gS2fNalBXiMaCCjZ6k4NUNlir/zaQoitbFmWiv5M6Q9XA4W8++Vx1Br8OKiH0g/ysWqJaTUdwhcCAdkqgHLr8lRtHJ4ZgLUfh7zhWmYjdqVo4/GIEncoW3oZBvkgjjj0ce999wEYRBhRMvqWk10sOf8GtuECKxVhRJ9u7Lk+qTLDiCwbUVNCUKwIAKAnPbgwojMDj53jlor+dOU8c/nH4JGNWPWteo3DOQbA/8K3DjtNEEYsGwHZKgj56aIeTjxpmg/jkLdby7H0ZHoAUBOuJqWUFzfI2wiyEesbmUzxYpgtLa0ABhFGZK+l1s3jx4wiMgYHtNnCY2Gv/tXZcoaryt06V7ZCSggKGSTZchdYJXZpCaBGZEt672dD7HoMI6JOinsPVQRkqwCcHg/ntJqOg71aNVST8o7F9UQuT6sWejNU2WrQmqAoCiyLblAvRrakgVXyRmlZFhIZveb/yvUMPf7449h9990Ri8UwfPhwzJ49G/399EZ4zz33YNq0aYhGo9hll13wu9/9Tiy3atUqEELwt7/9DV//+tcRj8cxffp0LFq0SMyzevVqHH300Rg2bBgaGhqw22674ZlnnhHfv/TSS9hvv/0QiUQwduxY/PznP4eu2zfLQw45BPPmzcP555+PESNGYM6cOUV/z3+y+n4KKW2Mbnh7torvx2qEEd1lDnxL62evA72J6hYnW2FooGSrN5MY2Fhc1xZCgAwjc5alorcmypb93m7XU/trbaE6W74qW3xabYfwhUDZRU2/TBAHlOMg8y8zpW6LmlZ5NP1ZarhtCDWyKQoAQ2RC5UO1oojJrIFdr3yugmssDUuumYN4uLRTduPGjTjxxBOxYMECHHfccejt7cX//d//wbIsPPzww7jyyitxxx13YMaMGXj33XdxxhlnoKGhAT/4wQ/EOi677DL8+te/xo477ojLLrsMJ554IpYvXw5N03D22Wcjk8ng5ZdfRkNDA5YsWYLGRvr3Wb9+Pb75zW/i5JNPxkMPPYRPP/0UZ5xxBqLRKK666iqx/gcffBBnnXUWXn311dJ2AMvEkNWfQmP0WAEA3naET/EnjOhWtnyrs+VSzcvdus7IUFgNQVNol5C+QZItUyI5QtmqVRhR2gP+GuTpq/x85V+zcslf6JO69kVAQLYKQG66LFQFXz1b0jSfjnXPkGaVx5I0qBrTFGZky1IAYiCjl1HU9EuWjbhx40bouo5vf/vbmDRpEgBg9913BwDMnz8fN910E7797W8DAKZMmYIlS5bg97//vYNs/exnP8NRRx0FALj66qux2267Yfny5dhll12wZs0aHH/88WKd22+/vVjud7/7HSZMmIA77rgDhBDssssu2LBhAy6++GJceeWVorr4jjvuiAULSq8KTjzIVqEx5i5PX0tWqi0LILb8X1lli2+CrtOvOltwPTyVexPV2QNPSA0hTCjZSmQHSLY8wojZWpMt2SDv3yU/JzMT8K/OluwvDJStgSMgWwUgS+x+HmSeypbf2Yg1JC8pox8gQGu0mU2ht7+ywogVHE8spGLJNcXDXpVGLFR6raHp06fjsMMOw+677445c+bgG9/4Bk444QSEw2GsWLECp512Gs444wwxv67raGlpcaxjjz32EO/HjqVtkjZt2oRddtkF5557Ls466yz885//xOzZs3H88ceL+T/55BPMmjXLsf8POOAA9PX1Yd26dZg4cSIAYO+99y5vB7D1yYd+oTHmLM5e5Y4QhSAyryy6cCWPIffp65tq4Q7dlbm8wcKIYSWMsBoFDKA/690YvPhg6ItsTBfqtaWiL139bEQZ9aBsedXZ8olrlR1+D+BEQLYKwPtJ2IcwInutB3HGK1W82rskYyYAFRjGyBZhnq1ipR9kVLaHMCk5nOcXVFXF888/j9deew3//Oc/cfvtt+Oyyy7DP/7xDwDA3XffjZkzZ+YsI4M3lwekJ22WNnf66adjzpw5ePrpp/HPf/4TN954I2666Sacc845JY+xoaGh+Ewy2BhkQlJojLmLy0q1Y5V5wJQEMLZV0TCi07Mlwog1dtHmJLwM0LMVVkOIMLKVGCDZcl/nCCGijpdlqehLZb0XrCCcarg9jlrDriBvT/OPkNtj8dNOM9QRGOQLwNkbkU/zYyBuqd+PQfCh1N6zpVs0jDg8xpUXpmxZxcmWI93/SwZCCA444ABcffXVePfddxEOh/Hqq69i3Lhx+Pzzz7HDDjs4/k2ZMqWs9U+YMAE/+tGP8Le//Q0//elPcffddwMApk2bhkWLFjlCt6+++iqampqw3Xbb5VtdCT+I/t0HXmeLQVaqSziZ7DBi5eA+Lv3LRoRj++WeywYrzRBWQ4iqMQBAyhgg2XJf52CHKVGjOlteni0/wI9xL4N8zWuxsVe/E8WGOur78dxnyBdEP+VTd+iuHo7zWtb8Mgi9eA+Pu8hWCcqW0CO+ZFzrjTfewMKFC/GNb3wDo0aNwhtvvIHNmzdj2rRpuPrqq3HuueeipaUFRxxxBNLpNBYvXoxt27bhggsuKGn9559/Po488kjstNNO2LZtG1544QVMmzYNAPDjH/8Yt956K8455xzMmzcPS5cuxfz583HBBRcIv9ZAQISyNdDl6avD6FtwCef5Vo1DSGQjMjHON/PzAG+ipqUDBIhqkcGTLbFO+4/DS0vQMGKtPVs+XvM9rCP+19myvpQPrZVCQLZKQC1DZl7IyRjy8bHCj5pfJkmBABjZwMkWCyOW0K6HEOJQMr4saG5uxssvv4xbb70VPT09mDRpEm666SYceeSRAIB4PI5f/epXuPDCC9HQ0IDdd98d559/fsnrNwwDZ599NtatW4fm5mYcccQRuOWWWwAA48ePxzPPPIMLL7wQ06dPR1tbG0477TRcfvnlg/tRTNkadLseybNVcFXsO8WqQhgxp/RDvShb5UFWtuIhSrbSRmqQY4EYC1e2rJplI9qwx+FfGLEu6mx5hlZrOoQvBAKyVQCeni0/Sj84iqv6q2y5xwJU9+RPZTMgShoAMKZpGBsDddEYpZAt9loPfrdaYtq0aXj22Wfzfv+9730P3/ve9zy/mzx5ck72Zmtrq2Pa7bffXnD7Bx98MN58882837/44osFl/fCXb//PS77+0dQSGljdMNOYbdKPJ9lzxZXxKyK3GlstYytW/hxBr3q8saR8yBX3vImqPIUUcNoYGQrM0hlS+5byUtLALVStqSkmjrojejVrqf2tdg87DQB2yobgWerAIjEtkp6Eq7aOOirnxWN840FqC7529TfLd6PaWqlbyx62Brp3qLLf5k9W180DNogLClb5YSIhGeLAJVSt9yJJoZEMGoJd928ch8meTudaCiCeCgOAMhYafzpvRewx30H4roXHyl5XS+vfhcA0M+6CRAAutSupy/ll7JVeyge11m/Sj84rqE+7pOhjoBsFYAiHWT+ZqbQV54G7GcmiFdKcjV3SUcfJVuWGUI8RCtUh0EVrQn//hGgF27z4vakBKhPPPzww2hsbPT8t9tuuwEY/M1GVjntJ/TiyymWS9mqANzHpd/VwQf6IGeBkS01hMYwJVu6mcIv3z8XltqNv3x+Z8nr6tU3AQA2sXNeIQSGwyBfW8+WuN768oDtZZCnrzWvs8VQcvg9gCeCMGIByEX2/G1K6jrx/FS2vCrIV/HMa+/dRrdh0RAFtq7AKKsTa6FBSXcByU6gaUz+Fcim6AB1i29961s5pSg4eHmHwRqWPUs/lBBGtEN+BLBMAKXXO8s/Fr5Ono1IP/uV1j9QYmEysjVy81vYbtmf8XALkCCrxT6LkwLnpgsWoevKGAYABYTYnjCrRnW2vBtR+/+A7RhPzQm5fc3308c21BGQrQJwZGHUQQivHrIRcwy1VR7M1gR9ylU52fr8Bai86jYhQInNqE0zoFv1jKamJjQ1NRWch/8JB1v6ody+no7SDxVTtvhYnGNy/7b+TArdqT6MaWwbVCZn8XEMjFhwZWvk+pcwtXsNYk3jkFTselgxtfDflCORTQOEnstZw4Rd3qW2FeRl+JmNyI8DrzpbvpV+sIKipoNBEEYsAPuJ1t9iboLg+JQe7gU+lmqPZGuSki2N0BAFsilo7K9hAECRWluV2ldftnY/9YjB9g/0WqzwmrhBXvpkeRdMLX8stsoG5L+xr+3ZiG2ZDmxJ9FRku7njKLz9YrAYQQorIYwwTPxxQwe2lyL7Zgm18ABgfXeneM87QxBCYPDla+TZAiS/FL/G+VjU1Lv0Q23Ho8iKMJtWB7egIYeAbBWCbGL1UdnKMbH6apB3j6W6g9mWojeZsMKqjetJ2j4FgIESlC2XEblc8BBWIjGwfm8BKodBhxEHGALnxxsL+A1s4+6xSKo5kP9BiofpREPmCoNvjdesU8pWTVhvRFaWY+dsFv/TD8wZfTYAwCxxf23s7QRhe1g3TDE2kylbtSr9AHhc42qyVfcY4BiDYzw++fpg+ZuVP9QRhBELQD6gLPh/4tWBZSu3pUaVt9fNMg6jClO29LRwzJgEBZWG/nQayWwKQGjA2YiqqqK1tRWbNlHzbjweD9KefUKGGaQt00QqVX4tp2yWSi6GYR8z6XQaKTX3GLIsC2aWTs9kTZgwYVgWUskUYAz+GVXPUhJhsN9iMLalZ7OO32ZmdFjERMZMD+g3FwMvn5JO0/GUe2xbRAcBEJXPr2wKUY0ms5SqbG3stZUtuSm3YRn0IsMM8pZlVf38y7nG+XC6F6qzVfvCtxTyNTS4BJaPgGwVQL0UcxO+I5/Swx1jqbFnq4eRrZjWSCdkk6LukQkUVLYe+fhpWMQCrNCgrDZjxlCTLydcAfzB1s4uAEB/Xx9WrlxZ9vLt7bT+Uzpjq0Rr16xGVyTX8G5ZFjb107+3aRjYoqo0DBA1AC1a9rbd2NRBW1ClUmmsXLkSqTStJdfR0Y6VSreYr6OvHYCJiNKPdLx4qZNykeij41BYX8xCp/PnnR34y0cv4pyZx6IhQskU91lF5Bn1FEIKvbVYJZKtzYkugDB7ACOehNjKHiwVpgUks0bV+5IOttBrJeBVZ8sQ2bi1HYtXdCDgWuUjIFsFIB9Qtpzro2erHgzy7LVWmTp9WXqDaeBkS09D5WFEQgp6tmg4wumNGQgIIRg7dixGjRqFbLb6zXADeKNl7UoAnWhpbiq7jyMAdKATwEaomgawYpyTJk1CSyyUM29vOonznzkfAHDrps345aiRiFgm/rr3pcCk/Qf+IxhWZjYD6EAkEsGUKVOgae0Ashg3diymTBkm5jvviYsBJYWd4t/Ar+fMG/R23Wh8LwGgDxoLlxd6kDvu7yfC1DajM9mNBXPOhG4YIIQSo7CsMGeT0BRK3kyURra2JLaBn6WGdG0xpTpbANCX0qtPtli3AD8fbr3qbPnl2cIAw+8BnAjIVgE404D5NB/GIZWgYBN8A3HL21UeS0KnT96NYU62ksJoWEzZIlDBbPQVMbirqgpVHXzaf4CBQVHp5UrTNESj5atLkTBXY+yDNhqNIhrNJVv9ZhYbMxsBAPHeDdjYqkOzLEStJDCAbeeOJcyGQhCNRmGxMUUjYcdv25BZC6LoaCSbB/Sbi0Fj+7RYAc/uVAKmthkA8PamtwCcif5sWnwflh9nzCxCLHPSLDGhoDNpq3mmQ9li5y8zD/SldYwqaY2DgIvo+Flny1n6wfld7cbi3D4QKFsDQWCQLwD5gPKzVY7bTOurslXjsaQNakxvCHGDfFqEEYtlIxKpHpJeQmufAPWNwRb+9DId57tvpXVbwYyy+XVCYGYqkyghtwIDpHY90o+jraqospM2B9YCp+RxFLEFPPTu8+L9+IZJAICkRLairoeZkMVJUmnnXVe6GyIFQbq2uJWt/lrU2mKvfl5v5WNcJFH41q7HOQ4g8GwNBAHZKgD5gKqHJtD1MAY7jMjHUt3t8XYdEZUqAcgmRRjRLFJni5ItOnPGKFxpPkD9o1J1tkwzd5obGd3OfItJN5l0ukJkK28FeXtEnck+e7sDbO5cfBx8+85xufHsyn+L9xmDkqxE1j6nQq6HHq50lUq2ejN2aQuxtwkRdbxEGLEm/RHpq63e1/56Kx/jpuV8rXmdLQ+zfpCNWD4CslUA8gFVTxXkfX2qyEmLru5gDMtudAuAKVvsO6BgNiKBKky3KT2dd74AQwODr7OVazrO9+CSlUh8WJo/kenzmn0Ag6EvFvKrFp0Je1sZs3yy9XlnB2befzzmL3ww/zDcap/H7tANA2tSi8XnFCN+iQw9pyxLgeY6D3lLrVJLP/Rne8BplqwoWcIGQMlWxqhM6Y1CyO0XWXt4VbL3y7Pl9uk6JgYoGQHZKgRPZcu/YfjZGJWj1sqWIFsaJ1spUUHeJCiobOmWKcabyAZka6hj8O166Kshk60882YMqqAQizjIVibdP7CNu8fCXoWy5VFnqzNpZx9mB0C2znn2eiSUZfjbul8XGIdTtfDaH+u6twKqrTxlTXouJXlfUkvNOQ8jjHxZJXq2kkZfzsYJsSvUq8xenM7WomUPffXzmq847j2u8KpP2YgOZcuHffLb3/4WkydPRjQaxcyZM/Hmm28WnP/WW2/FzjvvjFgshgkTJuAnP/lJVcqnlIqAbBWAs/SDf6pSrq/CxzBijT1bvKhhjNXtgZ4STiwdxbIRs+BPy4lA2RryGGydIfeDApD/fObKFrFoMJo3o05nK+XZcoYRvVSL7pRN7HSr/OO3K7O5hHE4t++1P3pcPjUeRuSeLWJpOedhSIQPSyNHKbMPduVo+4VXqI+F6MNWWq+FskUx0BZGlYB8HNih5tzvagH3MQLU/oH/z3/+My644ALMnz8f77zzDqZPn445c+bkLcfzyCOP4Oc//znmz5+PTz75BPfeey/+/Oc/49JLL63xyG0EZKsA5ANKNGr1o/RDzhj8A//9duPaKocR4fZsScoWUFjZksw5qUDZGvIwB1lnyPumkSeMyJUt9jnElslUyiDPXvlIDA+y05Wyw4gDIVulLCPUvgLXt760Uw3IUbagOo1wAKIii7A0cpS1JMXQ4U2l64mz7M1MLcgWcV/jqr7JHMiESihbftXZcl3zgdo/8N98880444wzcMopp2DXXXfFXXfdhXg8jvvuu89z/tdeew0HHHAAvve972Hy5Mn4xje+gRNPPLGoGlZNBGSrAOQD3utiWLNxsLPLzzGIsbhCMdUeC1e2oiE7jCg8W8XqbJk6+JU7GShbQx5eJvLy4GH0zbMq3h5HYcuE2TJpvTJZgTntejxUi24pZGmi/PBHaWTLeW3x6nXdn3GRLculbMFD2SozG9Eg/ZA70AL0WsOVrYYQVbZrqWwZvnq27Pe2Qd6nOltlhN/LQW9vL3p6esS/dNr7eM1kMnj77bcxe/ZsMU1RFMyePRuLFi3yXGb//ffH22+/LcjV559/jmeeeQbf/OY3KzDygSEgWwXgXUHeP2XLPtbrIYxYm5Fwz4ZXGJEqW/mzk7LSd8lskI041DHYbNxySj9kDTuMCNCGTwCQzVbG8+HORrQ8yE5P2la2TFL+8WuUQrbAt+8cl4x+12/WLTqWFFO2iKXlKMxhlkVcirJlmiYskoC79DAdi1PZSus1KOHivsb5rGxxRcm3Olvs1SrhIaUc7LrrrmhpaRH/brzxRs/5tmzZAsMwMHr0aMf00aNHo7293XOZ733ve7jmmmtw4IEHIhQKYerUqTjkkEN8DSMGRU0LoF6yEXMyAH1UtnKbYld3MKYH2VKYzGAQ5IQvZNDaWpSaJYMw4pAHP+bUAT4iCs+WHA4pEkbkxF7jYUS9QmSL39DhvJGq0vnUJ5WZsAZAtkwUX6aU9lt9Gaeax0lcij3AEGj2Qw9RAMtE2GR1ykhxsrUt1Q+iGB4XVwu8HVBjJAogU1Nlq1YZ116Qyzu462wN9PgfKNzZ8EBl9smSJUswfvx48TkSiRSYuzy8+OKLuOGGG/C73/0OM2fOxPLly3Heeefh2muvxRVXXFGx7ZSDgGwVgFPK9dEg7xpDrWP2DpRwca4kLMLIFq/+nU1BAX3KNYsY5A3ToBd/AKmgztaQhyj8OdjSD45p3vNyg7zClS22UKZC9a7cCrHXw0tvVvIxkQxM04TiFefLA4uUomzl3kjdSLiULZ4hnGTnlEKkMGK4CUh3I1xGGHF9z1Y+Yuf4ARBWuqUpHAGQqYlni9s2/LzmO7MR6etgj//BjqUURbgcNDU1obm5ueh8I0aMgKqq6OjocEzv6OgQfWvduOKKK/D9738fp59+OgBg9913R39/P84880xcdtllZZ1HlUIQRiwRXEDxxSDPD3YfxyDGAvdYqg1KtuKhfGHEQp4t+7t04Nka8hismuoZDskzL1e2+AVSY8voemVIuwgjss9e5udE1laUCLHQky7PL2aR4n08c64tHvvWXTbFBP2cZvtCgWavIEw7PQhlq4Qw4oaeTveoANiqHwA0RmoXRsy5xhU43tZ0bcZnWzZWfgxeBnmfstEFIZeLAddwCOFwGHvvvTcWLlwoppmmiYULF2LWrFmeyyQSiRxCxVutVaJ120AQKFsFUDelH4S/ow7CiMQ9luoOxiI6CIC4FqXEysyKNHyDoIhB3gC/naUDZWvIozrtekoLI6rsHMwalSHtboO8l2qR1J01vTqTfWiNNZS+DaUEsgXnOLz2hgjBG1FATcFkJC4jky2hbDGyxdvslBBG7OjbxkfhmC73VaRhRCCdrV02YrHyNrph4Ogn/guG2olnj3sB27W0VXQcCqGqlpts1Twb0XWsArV/4L/gggvwgx/8APvssw/2228/3Hrrrejv78cpp5wCADjppJMwfvx44fs6+uijcfPNN2PGjBkijHjFFVfg6KOP9q2/bUC2CsDp2eLTfBiHuEn4NwYOO6Tp/Fy9DdKLdkM4AjC/jK1skYLKliERsaCC/NDH4OtseXlPvKFbPIxIP6sWfUo2KkW22Ku79IP82xKuzMdtyT4ATpPwoMfhasXitWu5wqZYDSwrkhnk5TCi6SRbGvuulDDipn5KthQ4G4LLN/fmSO2zEYsVNV3XvRWmtgUEwEPvPYdLDz6xouNQCIFpWYKIGx6Fb2sB9/4Aav/A/53vfAebN2/GlVdeifb2duy555549tlnhWl+zZo1DiXr8ssvByEEl19+OdavX4+RI0fi6KOPxvXXX1/bgUsIyFYBeHm2/GA6OYbNOihqWgs/g24YIOzJOBYKA8w7wm+AVNkqYJC3ZGWr+j3VAlQXlaqzVVJRU5eypUAFYCJjFVeLShuLcMizMeWeT2nDSbbkIqfFYBZIHPGcv4CKk2IEUyNxZLBVhCd5GFGVPVuRJgDlhRG3JbvpesAN0k5SbFkKYmFKxGpTZwuO7ec7RlZ12QU1X1v/BoDKky3A8mjXU9HNFIeXIlzjIQDAvHnzMG/ePM/vXnzxRcdnTdMwf/58zJ8/vwYjKw2BZ6sA5AOqWKPWqo7D4ybhF3LHUr390SfV96HKVpJtkW7TKKZsmXaGU6ZCXpsA/qFSDxul9Ebkfj/+NKow2qWbFfJsObmWrdpJd1I32eoqg2x1p5zFV/ORr9zWNLn7g6vCYdJI51EMZHQdGUaoVBKysxGFZ4vtJ1L8otWZ7gIAaISGCsGSEnjLJFgqIhq9VVXbs6UbhiC1xa75a7ttsrU2+V7ZBLcY3AVnffdsOR5S/IyvDE0EZKsA5AMq8GzVfiyyMbchHAXYRV9hhy1tRF2EbLHbWSbwbA15DDqMWMZDC6/Rxv2BnGwZBeq6lTUW9moVUC3czad7yiBbG3q3OT6nDG9FLud89pgnzZStmNokpnWnEqLwq0pyDfIhdq4SYhYlIT1pqmxFlKgYFQAkM6zWGWSyVV1l64/vLIJuUZJb7Bq3sXeLeG9qW/H62mUVHQs/zoN2PV8MBGSrAJzKlp91tlxj8NUg7xpLFbfVn5HIVigC8OwsQm98xRpRG3IYsUKKRAD/MPg6W87wVKHzyK1sEfbOqFgYkb66lS25zlbWcpKt3kzpZMs2nVOks3nIVgnXFh4ujGmNYlpXql88wGgkJBnk6Twh095epkgIvydDm1xH1BgfFQAgpcvKlsrGUl2ytSXR6xk280J7/1bH5yc+eaGiY1FdJSh8q7PFXitd+uHLhoBsFYBXywQ/K8j7Gcp0o5h5tBLoZ8qWZarU/MiflrnKUKTOllz6IRt4toY8Bl9ny7meQmvRWQV5FVxxop4hvUJkS5Q3KFBnS88hW6X3ZdzU3+X4nMwTRs81P+fulTSrLRZRorBMSjp70raypSkhySBPyZYmJRIU80smdFopP8rKu1gsjJjM8uVUREL0VlVtz1ZEA+AqNJvvmr816SxZ0Znc5jnfQOFWYv2qs+VOopCnBSgdAdkqAK9aJ76EEeupgrx7LFUkfgnh2WL6gl6esmVKylY2ULaGPAZdZ4u4P+dfDw8jaiK8R+s86QXI/UDGYsGCJWWcyWFEg5Ety6QkpK8EsrW2i6otWxLdjun52lWVomxxBSushkEsuh960glkTSmM6Cr9EJKyfzN6YbKVMnoBADHNGUZMZXnLJA1htTaeLU2zGQVPyHDvEt0wsK67E11pJ9nSC1yLBgLFda31zbNVotIXoDCCbMQS4WMyokf/Mv+QM5YaeLaIxclWim1TBWAwz1axdj0U2TyelQBDB6YHISkH7geDQqsxeBhR3OAoyTAqRbbYq2U5FQNZtTBZBXjVbIKppNGfzV/UdG3XVpz01M+wxVqMo8eeL1QijpSeL4zoVNi89knGZAZ5NQLCukT2pZPI8jCirGxFyle20hYNj8ZDTrKV1nWEwTxbodp4tizpAc3IQ0CPfORstJuvQtPHOe6glTo2OPhxzr1SxiCzcQeKWl7zv8gIlK0i4Ac2P9D9kE95+RA/xyDGwrZtn/jVGwsvpsj9Msg6la6S6mwRrmwFZGuowxzkMederNB6uLLFw4hEoeTFKKFuVGljsUmOrBjIY7LAswBbAACJrLey9dgHL+OoJ47DFmsxAODjrUvQlXIrW971wdwZb177hKvCUS1iK1uZBLIspBpWQjnKlib1kMwUedDRLRpGbAjzbER6wVNCXfQVkmerykVNs9L1RFxvXRS03XwVAKBrG9iMNHHArDjZcobv/Asj0lcjj9IXoDQEZKsI+EUx31NOTcYA1xhqPwR7LPzEq4GkzH0mOcqWwjxbBAU9W/LFLyBbQx+DriBfdIINgymmGrews0w5HRZQAf+fnI3oMB6zK7JpmrAUevzHVUa29Fyy9fqapbjunXNgaduE10m3suhO9zrmSxfJRix0feNkK6JGRF/S/kwSOiOkIUWSd5hnS9VTYjzFyJZJ6O9qDDsN8iTUzT5pIhsxY1SZbBk6iilbboTQyuavLNlyWzb8643rPkYCujUQBGSrCNwp2r6EEd2pt356tthrLUo/5ChbnGwJZQtFlS3CLpx6QLaGPCru2SowL8/AC/FtMrKVJRDH4WAgZyPKzy1ctejPpkVB36bQMABAymO7/1zxFggxoeljsGfT8XSMZha9GRfZyhtGZOMocD7rlq1sqSyc2pdOinMqRHLJFrJJoVAVykbUDQOWQn9Xc4SRLUtcZQAAClER5qUfslWusyXV5iv1mh9X6N/HqLhnC2y99eHZ8vMe+EVAQLaKoJSif7VCXbTrydkf1dsWL6bIM8FsZYuRLVI4G1HurZat8IUwQO0hyiMM+NHe5dkqsJp+lowRZ8eQysoSZAgRWbGDgV3fyhlG5KUftvbbZGlYhPbcSxm5ni1+jjSoI9EWpfPpZgb9upNslZqN6JXwwjMwo2oEGiNbCT0lFK+wIvWaY2FEmszCMwjzk62NvV0gLNTfEok7RsVHokCrWekHqoC7sxELL9McovvdsCqb8cyPc3edrYEf/wNDzjESsK0BISBbRZBTm8ePMbjk5FrH7F2jcYylmtmISbnRLWB7ttiTtAGUXGercin7AfzCYNuV5Cpb+VfE/VEN7PhShLJFKqxsWZ7tg7YmKVmyTA2NTC3KGLnbTem81lUYEY1nTGZFOQWOYspWoXZkBlO24qEYNEK9a4lMUpCLiCL1NBTKVgqCbBVQttb30MKglhlGWLOrmlHwv7dWs6KmciFkq8QH7GHR4QAAs4TWROXAnY3oV7se9ziCMOLAEJCtYhASKvvoi2fL/zFw5NQqqoWyBaeyxZUuWmcr/wXOlAzylX7qDFB7DFZddi9VaDVusqWpjMhUmmzlMcjTptMAsSKIh6iqljZzt8uru6tKCFGVEiHdyiJtOP1d6bzKVvFsRF7INRaKIKQwZctIielh1UvZSoGUEEZsZ8VXFSsu/T2ctyUFWs3qbMmeLQ55n5imKbxoAAAjhgjb75U2yLujCH5FV8qpTxcgPwKyVQQ5F2gfeyP6OQZ724U/VxKORreAfZNjYUSDoARliy1aoTYrAfxDKcru0s0bcO7Tt2Ndd2fOd+6bVKFjN8nCiA3sGFJVpmwBlQ0jwvm8wFWL7jQth0CsCBpDNLymFyBbIRJGVOPlKTLCZyXmK6Js2Z9z94qtbEUQYgpfMpuELpQtybMVkTxbop9k/nN0Uz8lW6rV4JHAwCqmE7vOVsYwRVZqNZD1eCiT90kymxVhTwBQrSZs1/MBAKApvbGiY8lXZ6vm2YhlhN8D5EdAtoog92Lkwxjq6GDPuWFVcTA22XJ5tthnEyjBs8WVrSCMONRRSp2tK1/8HV7Y8gdc9/L9Od/lKlv5V5Qy3MpWlcKIljOzl99Iu5mypSKKxjAlW1krl+TxTL+QGhZFQU1LhwEn2crkSRAp5eHJohQT8VAUYVYCI6WnhVoclT1bIaZsmVmQEsKImxnZCisNea9rKtEQCdnbqGZGomHqcDfPloflruIfJs2YuuX/AACjUssrOpa6qbNVRw/7QxkB2SqCejiw6ulJoqbKlulStrLcIM/CiISUUEGeolINhAP4h1LqbPVkugEA3a46U0B52YjcjB5n2wxr1SFbgKv0g0vZUkkEjRFKYAx4kS1J2QoxZQtZmK6Hi3xhRPdO8dq1JiNbjeEYIox0poyk2IYIIxIFCMXEcoSF2wq1yupkf6eo0uhBfnM9W0B1a215JdLIw+pNO//2w9UmcRNNqjFUEvVSZ8uNerofDSUEZKsI6kPZco/BxzBiDdkWv0FozCdiZyOWr2xZMB3ZiQGGHkprIE0JQNZDycx5cCqwngwL2cVNesyEGNnKVIpseWQjEmKf27zpdIhE0cyy9AxPZctupRPTeG/BrCBI9nzehKckZYvQdTWEoogw71rakJQtwsmWCoiWO4DCbi+FyBYvvhrXmr22DADQiAZNIULRSRvVyyw2TLtcDId8vPVl7L/910eciV9P2BMq+/tV+nFOeLZySj9UeEMljkN8ru3mvzAIyFYR1Kdnyz/UciwZk2dauUs/8Cd4FFa2XNW+08bgvTYB/EMpnhXuVfLy6JVz7GZNXvqBbjMklC1UxrMlshG9FQtBtpQompnpnLfvcYyT/d6QEhbtbkxkYRGmZJl0Wtoo3BvR/py7VwTZCkcR1VgJDCMFk9GLGM8iVFTa7oIZxhW2h7MFVOWeTA8AoDHUlD+MqGgghEi1tqr30KR7ebakI6UvzcpvGDHcdtQ52G3zu6JjD6lwEo7o1lF3nq2Abg0EAdkqglyPki+jqIMxsG3X8MTLyL3XPn8R2PIZ3SZXtkixbEQ5s4ggXYGbZAD/UEqdIV7io5QitoWOXd2SlC1FQ5ipRllUStmikCvIq9J4+pk3KKLE0BplPiiSS5h4f8KIFkac9UO0iC58VsRi486rbJWQNEDoso2RKKLcu2Zm2PkFRGRlCwAY6Wuy6G8o5Nnqy1Ky1Rxu8XiQtQ3y9DdWv9YWJekuA740rH65ZVjXWmDFv4WyZVVY26qbOlt19LA/lBGQrSLIDeH5MIY6CGXaGy/4saLg5t8WMwk8dAyw8T26TZX3qQNQ4KnZtEyH2TXlUacowNBBKXWGOAHwKvVRznmks5Bd3LIANSzCZ9lKFTWVlC2vYpG8NU9YjaItzjL8SBrm8hcc6+HKVkSNIMbJFrKwGEFSLFY2okRly31CZ3QdhFCFuMVIYVLvx/R9+nNE0ce2zUgWN8qz0GCbRYnUyI//4LltAOjT6bzDoi15x6IpIeCdh/Bn6yL8PXwl1DX/l3d9g4VX5qQ8LE62FEsD/vJ9INMnbqKVVrbc9RX9qrNVS+vIFxkB2SqGOsjEqIdQpr1t1+cqDoVXqG50pbzbyhaKhBHlJ2AilLIAQxOl1BkyhLJVOIUfKHzPMMGVLQtQQ6KsQqXa9fCtW5a38Z+Tragaw7AYJ1sWsg9/G+jbJOYzmIIXUSNoZI2cLWIALPSnEUq28oXyil1b+iWP0rC37sAuG54BAIzIrkSLtZWOUZEM8gAwYme6bfb3atnwIpDxbqKdMighG90wwuPvQVeQMVLAa7djF6zEDGU5WpY84rmuSsCwPLIRpb8LDyM2Qgc2vAvE2rB68nf50hUdCydVvtfZKkX9DFAUAdkqgvpQtuoojJhzw6reYHjzaEejW9jKlglS0CBvSRXkYQXK1lBHKZ4VQyhbXgZ51+cC67GYPypmmYBq17DKEGJ3MhgE5H5zXg2GU7xdkBZHGydbAFIEQP8W8dnuWxhGPBxhK8+IWlDhYmSriNrXnbJJUrR/C6IsYSChaJR4Aoip3LPFXudcDxz0U6xWtwfAKEgegpph6te45hF5r2tJo8+5fEXIrje8+hvKw0owVTPM+dgOhyEbHw8AsCpOtpzKllFCNm41UIqvL0BxBGSrCBSXZlv7juu52/TzUM8ZSxUHw303EeI8TFV2UdeBIqUfnMpW4Nka2jBLqDPEs/A8jc6u5QqthxvMqbIVsVvhVNyzJdcPswckSk+E4rSkg0nVowQhgKTQioxALYLGMCVWctHNsEIzGTP5KsiTwtc3nn1nWQrClo4ou/H3kjCtpg8gprjCiDseDhx2JUyW2KIXCL2aCg1FTmwe5UEi6LZSRj8gjd/KU6C1EpBbfHHIw0qy/REWMc4oSKiRjdZ0dhUfJNx1tnwLI7o/B1xrQAjIVhHkHlc+hBHr6GivpUFeNLp1TVeYSdckKGiQt2CXfgBIoGwNcXiRktx5dPZaQumHPOdyfzoNolASH7dMGkasuGeLhRHhndKfYWSrgRUJJRYlLknFTbbo+5gWQUPYLrvAEWbnSjyxMs84XJ9d+6Qvw7LvrBBgZATZShFCVT4AEVbdXRjkxbro53zKVm86CSh0+uRhoz1Gx8KIVsLxmyux//PBW9my90mS/Y6IJZMt+jeyiAWwLNLB4vZF/4OPtnwKAGBiYknHfzUQGOQrg4BsFUE9hPDqKWZeyxOPZ5ZF+UbUMHDoFUi27gKA9UYso/RDpq+9KuMMUBuUUmeIK1tefepKNch3Ju0mzjGTGeQr7NmylS3LViwkySLDDPoNrEioyskWURxkwwDvWxhFY8hJtixTQ4j5G3fc+HdgS26F82LdKbiyRRjZirCxpglEJa+4W9kS66K3F8OlxnGs3NbB9oGC8c3Dch/c2EfdTbbM6nkvDSs3G9GhbPEwIp8QikLh7ZRAgHRPRcbxjxXPCnXVXfqh9veg2j1gf5ERkK0iqItEjBqG7oqhliIbN/9G+VZ3OBz42s+gsTAiVbYKebbsbEQLBKlNS6o32ABVRymp7zz93vBIwy/1XN6a7KXfm6wFuhoWGYGdqoqXtqwpY9Te8MpGlEs/6KzOVzOrHh9i1ZwSLmWLK3kxLYxYKORokkyskCiboIMAnSvyjiPf535BtjTAyCAmky0RRuTKlruBdGFla/W2zXQxsxGKouT8PVrV7QAAp0w710m2jGqHEZ2Qx5ViZCvC+ZgWQ0jlHS0ApCpDtvr1bhCW4GMJskW/C0o/DE0EZKsI6sEc6N6ir+0aaunZ4soWn8DUBZVd3EtTtrhBniBbRWNtgOrDKsEgz0semF5p+CUeu9sY2VItZvjWwpg1YWfsmGmDTgguTLyNtzcsLW/wLvDf4KwgL5Et1prHJluUuCRdKpEp9S1UFAWwZHUphBAjQDoBwGqFyShmk+jLJtnUEKBnEGF3/JRi0Tp3AKL85u9SthQWVjTyhF7X9VKypVlN7Pc7vz948gw8fcy/cf6sYxy/mVQxq9iwDIA4rQny34WTLXFNCkURUjipJKLsxWCRMrvFg6Ldrscfg7x7e4GyNTAEZKso/A/hlZOyXm3khjSrNxpDhBH5Ix29WWjs4lasXQ/LVxTIBAb5IY3SwiiMbHkqW6Udu9tYGDHEyZYahqIouG/qkdg9lUZSNfDEJy+WNfZ8sGAJT44sWBisqGpLlCpqEcubbMlNogGmQDEoVggR9rCRJcTRSocjV7VwhtASGXrOKAg7PFsZxZ4vzivIuzxbxZStjj6aVRlVmtm23WMjmNg6MkfJqibZKhZ+5vXK+H6FFhPXI4OgYmHErNUL/qBo+N2ux/054FoDQkC2iqAeCorW08Fey/3Bb5jiFsHkesfFrWidLdsgnwkM8kMapRiEbWVr4J6t7pSbbNHjrjXegt92bMZ32odhj6Yjyxh5LuzSD94lLXhrnlZGtmJMoUoqijMzj/1eXj0eCNnbgE22dBDPTDk34XSnmyRZmQuVhAAjLciWDD42uEu0MPKlE3gqW5v6aZ2uuNbC5neTYbYtV5stUkJ3gIHC9MpGlN6nGGmM8sScUFTYGiqpbNEsTTqOvgw9Huum9ENNt/7FQVlkyzAMXHHFFZgyZQpisRimTp2Ka6+9VsibAJU6r7zySowdOxaxWAyzZ8/GZ5995lhPZ2cn5s6di+bmZrS2tuK0005DX1+fe3N1gVyfhw9hxJyD3b/DvZYeNpHWzm8BLAyiEq5skcLZiK7vMkFvxCGNknrDsWrnXq1T3Et19mfQm8q9cXenaUZZhCkzYJmI0CIYZprYty+EvvTgqoV7ZSPKyhY3R/NWPTF2qc5RtqS+hYBL2SIhRAhXtgB4kBT3ruTlNTgSuky2sp5kK18YUSVSuN9D2epMdQIAmkPDPMdCeG0wl7KlVNUg70XSpcQFk4URLVvZCrM6Y5VStrYmekGUrAgjbuij4VavHpq1QLEkigCloSyy9ctf/hJ33nkn7rjjDnzyySf45S9/iQULFuD2228X8yxYsAC33XYb7rrrLrzxxhtoaGjAnDlzkErZJ9vcuXPx8ccf4/nnn8f//u//4uWXX8aZZ55ZuV9VQdSHslU/jxa5GUPVGwxP349wssXDiKoURiygbAEG7GrQRPSRCzA0UazOFm0tw0zFXmTLdaymdRN/e2d9znw9adaGhrcYZscdWBPmCLLoTw+ugKUYiVRni49PNwzwPojDYtTPFGfz0NIP8kOD3bcQABRJ2VIRRpSRB52QkozlLq5lK1sIA3paZCNyKBagcCXIbZAnEgnxULa6M9vob4wysuX6XpAt17JKrZUtaWC8C0Xcsh8AQ4qUhFABg/zKzg72jo5jXfdmNrb6aNfj58P+UIZWfBYbr732Go455hgcddRRAIDJkyfj0UcfxZtvvgmAqlq33norLr/8chxzzDEAgIceegijR4/Gk08+ie9+97v45JNP8Oyzz+Ktt97CPvvsAwC4/fbb8c1vfhO//vWvMW7cuEr+vkGjHg6sepJxazkWEUbkF3ivMGKhbEQEytYXCcXalcitZSxSOKvMXiaXlPWx1jJRoWwxAsOU1QiySHgsVw7sbESpgjzjKt3phChMOqKBkS22HC39QG/4umGIemAN3LMlXdJVEhKqMC0AnDtmw8WuTBeZ4qUONCUMGFkQAMTUYCl0XZpF7HPQbZBn+48Wgs0993hfxOGxNudOYbCVLeeyahWVLXe5GMB53PDaf5zEIhSDZrCHvwopW2sYueJkq4OFW/1q1+NGoGwNDGUpW/vvvz8WLlyIZcuWAQDef/99vPLKKzjySOpfWLlyJdrb2zF79myxTEtLC2bOnIlFixYBABYtWoTW1lZBtABg9uzZUBQFb7zxhud20+k0enp6xL/e3t7yfuUgUB/Klv9jENt2f67iWLj51x1G1IiU/VNA2XIXNc1UMWU8QPXBiUC+1Pe+jH1T9mqd4nWsdnTnhrf6WGHKOHGHESmhiZLM4MOIUm9EbsPgpR+29Pey7whaIpRmNbLjWC5q2ieRy8YIVd0UYitbGgkhwkhBNo+ylUO2XJ959p1GwmK7cqhSs2CTOLdBXtTZgmcYMWlQsjWmYQRd3K0ocQXLHUascMNnGaZnb0T7PSdbMU62tChCPIwIVMSztaGXtWNiKu3WxDY2tsLHf7VQTw/7QxllKVs///nP0dPTg1122QWqqsIwDFx//fWYO3cuAKC9nRaNHD3aWQ149OjR4rv29naMGjXKOQhNQ1tbm5jHjRtvvBFXX311OUOtGLwyZPwehK+erRqeeNz8G+OSPbvphWSPRKnKlgWkzUDZGsqwPSve3/ezMgUUxbMRAWCjB9lKsr6E3Ccle7YArmwNMowolK1c4z8vPQErLELmDewcoJ4tehzL5JIXNFUhk60IooyY6IR4erb0HGXL+T3vuhBSwmK7xAoDrFE3JVveypYqPxR5KFtZ3hexiZEt94OTK4zYrRC8H4mgxQSm56ytMjA9PaD2cSOULT62UAxhk4URSWXCiO19vPcl/WN0Z7vopyLHf7WQW9g7oFsDQVnK1l/+8hc8/PDDeOSRR/DOO+/gwQcfxK9//Ws8+OCD1RofAOCSSy5Bd3e3+LdkSe2KU9ZD2YX6MijW7sSz4DLIs5ue07OV3yBPnwxlz1agbA1lFDPI98vKlkcY0evk9SJbCZ2GERsE2eJhREpoIqiEssXGaVlCTeI/qytFlTVKaiiamFKXJIpQeni407IIYiE6RlVStkJKGBFGtrIAYOSOWXedP26lqztNlZoGrVEkoxCJ0IUB+4EnR9mSsxFz97PB+yK20odvd0kHO4xIp9/b2oazx4zC402xwuf9IOCozcfHIX3mjb9jfGxuZasCYcTNia1sMHQ/92fpOktKEKkCAmpVGZSlbF144YX4+c9/ju9+97sAgN133x2rV6/GjTfeiB/84AcYM2YMAKCjowNjx44Vy3V0dGDPPfcEAIwZMwabNm1yrFfXdXR2dorl3YhEIohE7IJ8PT2VqWUyEPgibNVBKDPftqs6FHbDFE+RmrPOlkFIEWXLeeHMVNFYG6D6KFZnKCU3W/bybJVItlJM2YrzBXgxUE62SBb9gyRb/MTxUra6WOkJxbKveU3sHEgqtkokyKWl0YKmgKgYD1CfVdSihCybT9kyCnu2ejJdAIDhkSYxTYE9rpBlDUjZ6kr2gyh02qRWGgkh+Uo8MLK1h05/43uRCP0tSm6R1sHCoqN1IK3bxE63sgABonxsoRhCWfq9WSGD/LYUDRvCpMdbxkoga2T9q7NVR/efoYyylK1EIiFOag5VVWGyp4wpU6ZgzJgxWLhwofi+p6cHb7zxBmbNmgUAmDVrFrq6uvD222+Lef7973/DNE3MnDlzwD+kWnD9XF+qt+dU8PXxWcM9lmruD6Fs8Ys5DyMqpWYjWhIbJMh4GIQDDB0Uq7OVkJQtQgxxXRLTPJbZ0pdGWnceQyneBNqtbIW4spVFPw8jprqB1+8q26sje7bcigWv86USuwhpI1OzEpL3ytFKh0EjthoWVsKCFOTLRnSTK/dnbmIfEWkW0xRpGyHLssuv5K0gjxxl69PN6wAAlhnCuKZWOr/pLvHgJFt7KdSXtiocwgq2fKVBw4jOfdDf0wn0UYHAYD0rY3xfalGE2fGhV8ggz7M0w0pcTNvQvwGGKH5ba2WrniIrQxdlka2jjz4a119/PZ5++mmsWrUKf//733HzzTfjuOOOA0BDSueffz6uu+46PPXUU/jwww9x0kknYdy4cTj22GMBANOmTcMRRxyBM844A2+++SZeffVVzJs3D9/97nfrLhMR8DjQ/BhDHT1Z5Gy6SmMxTRNE4WHEPMoWCitbcCtbVTTWBqg+zCJFHRNZpzKSclcez2MJ6Oh2LpdmZKuRz+8yyNPSD+xY+tMJwLMXA8/PL/l30LHY7w1XGLGH1fnSJAWpkXmF5NIPtkFeykBUpBCfGkGUlU/JV2crR9lyhRG5iX1U1CZbMgmMyMqWK4yoKvmVraVbKFlSzVbxAE9cRYeFssUUy7ZIM6Zm6Pv/XfpKzm+pBLwSK4Y9ezbw25lANiW6WsS5CheKSWHEyihb/YzgxkPsuLMI1vWu861dT7HSDxv7NuKQh0/AsY+fWsNRDT2URbZuv/12nHDCCfjxj3+MadOm4Wc/+xl++MMf4tprrxXzXHTRRTjnnHNw5plnYt9990VfXx+effZZRKP2Cfrwww9jl112wWGHHYZvfvObOPDAA/GHP/yhcr+qgsg5rv33x/uKWoUR1/VsFe8jXJHini1FSrUupGw5sooIsoGyNaRRrM5Q0nVDT2ZdHiDX/Dyra2N30jE9w1rlNMJNtrhBPkPJlmkA62jZG3z8t1J/Rs5YDNdNVJAtpuQAQBMjIrT0A/2d/PcSS/JpycqWGkaEkQIdJI9ny0muXNyLtY0BxsdZGFENO9Uzy8xb+kGVlS1XiHBlF61vFlPapMG4PVtOZQtaFNNZEdo3N76X81sqActRm48ilNwMJDuBbasE2QqLoqYRROSEnQpYFVImJWy8dhqgYF3vOv/qbLk/uyZ8tnkrtupLsaLn45qNaSiiLM9WU1MTbr31Vtx666155yGE4JprrsE111yTd562tjY88sgj5WzaN+RyrdpTn3pohi22XWVJeV13Jy751x14r/t/ARWwLBVxrlC4shGL9Ua0XI2oMwVVsAD1jmJ1hpIuZSuRyWC4HYnJOVY1lUA3rRzflm4mARVoEj05XcoW0ZFIZ4C1Uqmalgll/Rb5NwjFjj369mcp2QoTW9lq0FMA4kzZYiFFpmwpkH1aIXBxJqpGETXTgJrfs5U1ChvkTUJDmuPjTNlSIwgpEXFahWHlLf3A/WNeyta63o0AgObQSDEtv2eLTdcimJ4x8DcAKxOf5vyWSsCzzRN/071WlKOJ8N2kxaAp1BdnEALTyA66Bx4nuC3RGGhWLcGKbWtgWsPoeHxujujeOs+KNS0VhmnVvDTFUEHQG7EIctNefRlFgU+1RbWrCZ/wtx/ivb6/AmoSqj4Gp+04HxF+0fUKIxbMSnLeONIB2RrSKFZnyK1spdxKietYDan08pdDtkDX08gnuJQtAAhlumF9+IS9UHd5HiKHsmU662zxEhZh1Va2GrP0hi6Xfkiw6u4OD5Viv4+oEUR4AVQCT89WbukH+3N/Og2odBuTGriyFaJkiyFsGUUN8l7ZiJsT1AM1IpqfbInxioetEPYy6D7qVTbmhI0rAWdtPjYu/rlrNUxBtiyAKIAaQkSzya5ZAfWcZ2k282iQRfB59xr/6mwVyUAX55mlera/CkARkK0iqIMoYg7BqacHh0qTz4RFn3gPG/UjLD75WfzkgOMcF1tAMsgTFPZsuUo/BMrW0EaxOkMpV9jQTb5ylC2FQGtZjJXbNjqmGyyM2ARvZQsAFod/CLL4HmnjXUByW/EfweBoOi2yzOi0BCNWUU629DTiLJQml35IspucrGyFVZtsRbUwoiz8qIN4V5AvoGyt6qKEyLIUjOE3fi2CsES2IpZlEykX2bIfipCjbG3L0CrpYxrsrPW8yhZfVo1gHM+2VAz0pBI5v2ewcGcwAzLZWit6UYYti7ZvIkT0RgQAY5BkS87SbIpwsk2wvnedb3W23Ntzn0dpiWz1JAOrRj4EZKsY6iCElxsz9zGMWGWzvkXoheb4Xb4u6mnJF1sA0FTe4BZ5PVumadI+eXJvxAJNqwPUP4rVGUq5btbJTOGn7Cx6ERv3OP7deZOtknz2LxBCb+JNor5bSLxarlAZxuwBhJnqs211ib8EjpPazjKjr7zOlyBb6T7ETKmCPPdssTHLtbXCiky2ogizMhbZPMpW1u3Zkj6vZmSLmA3Q+IOKGkJEtUlnyLIA1ksyr0GekBxlq9+ghTsnt9hJUaSoZysMVSKTGQ8P2uBRSNlaA7BCyxHLsrNTVXv/64NMwlm5jRNcVRjkLRBsTm3wr85WkVqTaZ3+nSxLRXcyULbyISBbRVAfylYdhRFzQpqVG41pmoJsjWiws59kzwYA0fjVIgRGnoubu1ijBSCDgGwNZRSrM+Rux1RM2RoWbYZlRpDWluOkv19Fb6YPH48QoZ6pJskELdYhqVvv7nMOnpl9DTBqGp2wbVXJv8WRjei6iWZYpwNBajK9oouCTgjSjLik2XmhwmmK54hqEUQY2cpbQd7liDekMOL6HkqINKvJJjxqOJdsZRjZylG2pGKfrr9FBlQFnDp8vJiWk41ouMiWSskWYWNM65UnW9Qg75y2KsSUq24PZQt2kWUAMAdJANcwgquYjXa40CJIm4mcrNVaodjDftqQlK0gjJgXAdkqgnrwbBXLBqklqqlsbUv12w144zLZcoYRw9KTpJFH2UqLG69skLc85w0wNGAWqTOUct3Q0+7SD64zqSEcwQ92uBgAsDT5v7jjlT/BBJBmszUKNccmMJx4rdE0nNrxLC76vwvwYogpW12lK1tOzxb9YYJsMRIV5cQu3Ye4pDj1u5UtqdxDRLWJYUyLIsJUsixIHs9W/jDixj4a6gsrjQ51OarJni2JbOUY5Llny6ls9aaTgEqXmTbSTiwgbnLsVrbUCCVc7HuuqJSKC/7fbzHvf39TcB6L5CpbTzQ10N/RtRaEF1rOo2yZMO149wCwvpfucw1N4nhQWSV5w3IeJ7VCsQx08XewNPQEylZeBGSrCOoxG9FPbaua5/mmvm4AtP3I8Hij/YUrjMiNzQBg5iFbukGny8PNICBbQxnFwigZV7sXt4fLa7ELD/pPTIt9CwBwz+a/Ymk4BIvN2ChKjtg3U+4RHKfrMJITQBQdPzM+x/82xNG3ZXnJv0V+iONhRD4pa3KyxUhNpg8hABr7/f2MuPCwqSaHETWbGDZoYYTZvDqBp2cr61a2JLK1OdEJAIirLY4HnqisbAFAmvVyLKhs2WRrSQcvaKphcmuBbET+9+ThRTVES09wZauM9lu96SSe33QXXtp6Dz7uWFtgztzrSSe73mT7O8Q0qmzR/aBJla91wHM/l4qOPlryJkKaxDHSwg4Qq0iduWqhWAY6V7aCMGJhBGSrCOqjzpb/6prYdhV7I27pp/VliBVxdiqQPBuAXfoBAPRSlS2QgGwNcRSrM+TOPszNRnSCH7sPHTcfMWMqDEXHn5uYSmUBcctZ342ulD4QaAB61p8EIzERadXAJaNG4AcdzkKb725Yie/89XJ80L4qZ6zyWNzFWrO8/55QtiiZ4b6tfnY+cHIp19aKSmNtVIAQO+SzeSrIGwWULd42pinUKp2DEcRCNtnSHJ4t5+1EI7Jny/5bLNtKyY5qDnOd5+6ipmwZ2Uag2cpWtoyQXW/arqX22db1BebMVba62Bgz0rVOJluKogAW85ES70SEUiEIrtYijvMIsz/YHRQGvPqBoUgGurjWWkoQRiyAgGwVQT0QnZr2IyyCao6lM0lvKkTqCQfA4dkA7CwngMn2HnDXDwKATB1lcQYoH+Jmk+dukzGLkK083sdoKIydWmYAADZq9NhSLQ2Kq76bjLSlAVYYo5PzsJ+6HwBgbch5o/nlq/dhSeJ/8ItXHshZ3suzxT06OlO2GkLcIE/Pi3O3deGyLZ1oZnWNuGdLk8KIcoivkZhCBaLZiF51tvIrW11pSrZawsNswqOGEdfskhThAp4tbpDXAYeytbJrAx2rMswxf04Y0cOzBTUMlf2mcgzycpPyDb1b8s5Hw4hOdKnUIp9mfzRiEZr/GbL3g8XIlgl4ktpS0ZmiZKs5NEwcD5wwW+yIzXf8VwvFrCN2NqIWZCMWQEC2iqCeiA5HPbXrqahni5EtxbKfnGFZTs8GgJBsSM2jbAmzNJGVrQBDFZbkg8kfRnTe5NJGMWXLft8cph7BTYxshU1FCp3lkq3PLVqy4Ctjx+Bn+/+Ybp8AJqv+DgB9WXo8d2c6c5aXH+LcxmedKVtxriAxMnNUv4Lv9vahiWVZcrLlrq3F0URMhJhKQ+tseYURXcqWtJ/7slTFGx5vc+yLuEQyaDYiDyM6a2Q7GsZLRKqdecEatTbH/G6yhZwwIiVbfCvlkC27tRGwvmdTgTlzSz+kFYIEIcLLpzBiJZcCIexWmi9cWyp4X8Rh0WG2Z8ty2sBqH0YsDHHeBWHEggjIVpnwpfRDERm3lqgm+dyWohdtTeq9BvmGycKIYemibuQp5yBa80gXqWw9MOUAA4LpuNl4z5N1kSv3zbjQE3pzhHoEO1iIOmpJBMFBtuhCTxtfBQBMbIuhqW0yAEoqUhs/ssfDlLaE3pczVnnbbi+aARfZYmQmHWoFAFhsXFl2kwvx8ekZjOxbKdbbBAMa228mIcjquY8bOcqW9DlpUrI1Kt7m2BeNYTkbEUCGEUyXQZ5nDZuAQ9niCqQc/gS86mxlXA9bA1e2eAFYAGjv35x/Rg9lCwA2h8IijKhxshWSHwp5OZrBhRF54+/hsTZxjGgWgXylrXm7npwkMednfq21gmzEggjIVhHUQ9mFeghlSlt3fqrgYHqY9yNEpB4r8g2UhxFVFbBYyCVPoVJxIebDswh0Qgadmh3AH8iVzfMdc+4woiEZmr2Wk8+rtmgLAKCXmaHjlpWTBQsA+OHLeGb0j3CnQU31E9viaInZx2v/hnfEe0620kbh4pu8/AK/iZpM2WoMs/Wy80KPUiWIe5n47xVFRt/8A3b96Bax3gYrK8KIAJB1V2gHoBcoapo2Kckb0zBcIjwhO7wJUOWsWBjRpWzxDGJVnn/tmyCbP3EsTywTWHiNTdS0sCMbsRzPVkIKI25NeYcRc2vzsXHAwvrmCbSgLCSyJYVT+a3UGKSyJRp/x4c7lC35Vl3rB/5ituWMXPohULbyIiBbReB+iqi1hAvYPdM4/CRbufujcuvuYRftkNSA19GcVgqREIs+NafyerYYCXOlYWcyvRUYaYBaw3SEEb3n0V2epImf/q7gOuX1DIu1OL5rtAyHKVxg7B5YvN0PYLBb/oRhcTSEIuL+nNz4oT0eRpoylh1a5HB4tlwGeZMpW428gjg/ZuPD6Xzsd3IyF+HKVucKGtZjaEZWhBEByVsjIbcRtf3ZBCU5bfFmx75ojNjkMiyHEV0G+RDzkolsRO4fY2SE907E5qXAvYdDcZ3LCizglZuBD/9KJ6hhapDnpv+ylC2bbHWlvcmWuwyGjLXxMXg7So+DESb7u8jKFiRlaxCeLd4XcVzzCHF8qiDi4RLwQ9kq/DlrBmHEUhCQrSKodsX0ksZQxUKi5SL3qapyY+lj4YioTLb407iiOVinYtEbTCJPhqG4EIvx0tdMJjekE6D+Id8H8z3w6JarrlY61yvlnMFez3C5rhuAZjObk5jBEQ/bisyEtjgURYFi0WmZrbY6ozMypFu5ypbDs+Vq18MLZzaG7QryABBqGgEA0Cwn2RKFTFM9tLI5HydMEUYE7LCjjEKNqC1WLb0xHHPsCzEu8DBivqKm3LMFALZSyAsRC7LFisHmXFl4sVhXnS2NnfPZMhQkucBtv+HdVikpyKjrmkIsrA614l8NlGQeEmYthiRli0D6rYNQtkzWF3FC80hxnCs5YcRaK1uFIyucbFmWhp5UEDnIh4BsFYGfxEaMoXr8pmxU0yDfn2VkS/MII7pueArLWEwQb7LFw4hE3HwCsjWUYZZgkM+6lC3LzBRsVC6vZXRjq+O7ZjPjCJ3J6EvbN5SxLbxZMD0+jW0rxDa598ogHmTLU9liq2Jkq4UrSLz0Q8soAEDI0pHJGoLMCVN8qpsqTQyNZgYqAMXi5MTLs+XcP3JFeZv0RR0G+ZaofX7KSlqOZ0vldbbYD2MPTro7jMiVMU6u+OoiDc7BsjpbXNkqx7OVlJStjNXlOU+GVaT3Orre0TN4jylb/9nMaoPJimcFPFu96SSgUDVx8rBRgnxTwlycbP329X/gqn8/NKBtF0Ixz3BWLv0QKFt5EZCtIqgPZavw51qimgZ5TrZiMtnSvckWb1GSyqNsifpbXIrnptpsbkgnQP3D6dnynscdRswSYqsuHpDXMyruCiOaOsAaQruPvW0Jm7TwPp0Wy6DNGkmgaxUAwGBhRIsk4Yb8E+Q6W6ZpAoQu18TJFvsNsZaRbD4L7dv6hJIX4YVMJbKlWBYiaVq3jqtbuqey5S79IJMvShri4YjDIG83SIaD3LmzEUPCs8VXx8kWXa/GlS02ThKKO5YnmqsEjMYryDvDkaVAVrYMpZvuZxfSgry5rykWloAWrN0zlcYk7gUMycrW4LMRV2+jxn3LUjCuqU2Qb8VyhhG9jv9tiT7ctfRSPLH2V/hsy8bcGSqIIIw4MARkqwhyiU4dZCP62Yi6imb9FDMSN4ak6vGuvogcPGMxSeDZHsP91CvSxTMB2RqKkK1Fah7TijuMmCUoTLak92PCTlUm7tig89jbfbyTmNF1UfUrpRBgEw0lGmw8RDHQnXKqW44K8lKdrWQ2K1pWNQtli/4G0jBcLLOhs1v83qikbDUzparFNKGkqdnaDrvl3gj7Qq86PnMPl24YIAp9YHGGEUNojdqKk+YgW97Kls5vM8zoLsKILmWLODxQcJRWoAuw0g9c2cpT9sULsrJFFB3re3JDie7SIQ4odMyz+xPAlmUe4xt8na3V3VLjb1WFyskWnGFE1cOn+vjH9t+xM1FZ9b7YA7YcRkzrJlLZ0v8uXyZoxWf5kqMOeiPmZAD6MQS+7SKS8mDAyVZDSAof5AkjaoTeYJIKASwzJ4TBDfI8yqhaBDqAjF44MyxAfaKUOluGlXWcHDqIHaJiIJB6CkjraTISiJgm0grPRpRuaK4w4vdnTYJCCA7e2W41o5AIDAApQoAUVWpM2Dfdjb2djvCbl7JFCNAt1elq5VmOTPnhBnkAaO/socoZcSpbI0wTN27aguGGAQzvosNnqogX2dJDawDsLT7zkKZcl6pJJltaxCaBYO16xI/yLv2giwJilPAYLINYtBniZCsccyyfo2xxgzxXtsogNe6+mZ9tXY8JrcMd0zjZsjyULY7DEwlAZ8dUKNezRRt+5ycb2xJ9uOLf9+K/9zgCX524s+O7dYxsaRbtYkBYE/Ecz1b3GmDEVMeyL6xeJN6768sNFjnXeNf5x8kz2D7oTemIhpzHQoBA2SqKegjhue8tNW/XUACVJJ8Zk15cmsIS2coTRgwxZStBFM+LGzfP8vGJDKZsQLaGIkqps8WVJMWkF/osIXYrGQ84VpPsRKO0Eaey5Tz2IpqKUw+cgqkjbQWWh7XTUuiSe54AoL23y7lt2bMl1dniCphlETSE7N6IAIBoK0w26k1dPbAYaRHZiKyV0H/0JzArlQaS29jY2HY8zhPL1QtQF2TLJicNkYjjoac5bxjRlY2ouskWV7YY2cpRtkogW6qcjVi6guLuJrByW26oLadcjBgI3eCETBTjdGmbHkVNDcCzUj/H/Bfux0tb78EZL5yQ810774uo0GQNhZFsi4RhydmIW5bmLPtZz3vifdIj63QwKJaBLsK5LEmkWqHE3/72t5g8eTKi0ShmzpyJN998s+D8XV1dOPvsszF27FhEIhHstNNOeOaZZ6oytlIQkK0iqIcQXq4p3c9sxMKfBwNOtloixcOIEZVemBMKEc2BZbh7JgpTbTbXPxOg/lFKnS2Tka2QxckWbFXIA47VJLehSfLxxKz8ZMsLGivQmSREeL0sSdna1N/l2rYURmSbVQjQk2LHp6XZfQM5YYw0wWDbSSQSMJmnKqqFqGrElBBBApJ0m4TdqA1PL5F3NmJfxiZ9UTXkeOhRFAWWSVWpQgZ5TbTrcZMt5tniHq9SyZaos1W+Z8ut9qzt6cALn3+Iq//9R+Hf4gb5XBsonbBvaJJzsodnq1hvxDW9a8T7dd3ObNmOflqSokFrBQAoGaaQkigcdbZcZGtrohdJxS5m627APlgUqzWpS54tAFUpbPrnP/8ZF1xwAebPn4933nkH06dPx5w5c7Bpk3c3gEwmg8MPPxyrVq3C448/jqVLl+Luu+/G+PHjKz62UhGQrSKoD2WrnsKI7rFUbjS6RW8WrbEme6JXYUkAYUa2ksWULXah5LeBIIw4NGG6Mva8YDDyEbYkZcvl2ZLvo45jN+EkW3H+3lVyJB80VliUKls0FCgrW1sSXTnL8FOJm9IVQtDLSA6xpOOd/4ZII0xGUCw9DZOpUhEtLEKXAIAGmrWIVJfjdxpeigtxnTtNi2GaJvrSLIzISZ8rnM/HFyrFs5UTRnSTLW6Qd5GtkFvZijg8W+WUfnCTrQ19Hbjwxcvw+NoF+NvHi9g8zGPnusaF2LF37MQDnSuVlS3edBso6NkaHR8t3t/3jlNl2ZykBvnWMKunxpUtJSYM8gpMYPMyx3JPfPwKLcbK4FbxBotiD/v872lxslUFZevmm2/GGWecgVNOOQW77ror7rrrLsTjcdx3332e8993333o7OzEk08+iQMOOACTJ0/GwQcfjOnTp1d8bKUiIFtFkNuqwIcxFJ1QO1RzLAYrotgalciWyIJyXnijKvWN5FW2uGeLfea+lXSgbA1JiCbUBU5A03KSLS/PlgMOZavTSbY4iShB1QKAEPMQphSbbIHYZGBbMldh45vnyhYhBD1pTrbYdg0pKzLcBIP1QbT0rPi9ETUkQoiINAO88jxXtsAzJj3Cbi6ypUY6sKG3S3i2ePHgHLLFwqYhB3t1ki3eVstWtui5bIowolvZcmcj5jPIc2Wr9DBixkW2tia3Ig1Kbj7ftp7N461sXd7Zie9OugQzps50fuEVRizi2ZLH8eLaFx3fbWPFVkfGqReQsAQHS42BHy2UbH3qWO6jTcsdnytOtooY5EViiiuM+K+V/4d5z/wSC1e94rne3t5e9PT0iH/pdG6HA4CqVG+//TZmz54tpimKgtmzZ2PRokWeyzz11FOYNWsWzj77bIwePRpf+cpXcMMNN8AoI/RcaQRkqwjqIYT3ZemNaBFWsdqhbHmHEaMhrmx5X9x0oWxRCJ+HHpCtoQh3/0Av8LBdDHIYsUTPVqITjV7KlktRzQfeDDrFlK1ENu1QG7alPMgWrxhv2apdf4aZohmZcShzkUaYvOm0nobFlDyqbDGyFW2RwojUs0VEayunEpTRdeT2ArTw2db1UsV19vuFQZ5uf7i2A1RTw+SspGK4Sz+oUm9ENmbA9myFXGQLxQzyGjfIU5SjbLnJVldmKyyFkuKtyW42j7dn68BkEpcd8j1gu32BvU4Cwk1AKA6M2V3Mo0BStgp4tuSWUl3GYqQW/U4co306DSuOZcqkwv+mDrJl0WxISVHcktzq2EYlDPKrtvTjoAX/xh8XrSoqOAiDvAgj0s/3vv3/8NLmP+GBd/7puY1dd90VLS0t4t+NN97oOd+WLVtgGAZGjx7tmD569Gi0t7d7LvP555/j8ccfh2EYeOaZZ3DFFVfgpptuwnXXXVfwd1cTQTZiEfjbh5CNoY56I+aOpXKDsUgaBK5q3lIxRRlxjfq6EopCsxFd4E+9gmyx17TUEDfA0IEgJAUeD7mHKcYKTFKDPCU5//jkLWxNdDvmdxy6/ZvRJFdfL1PZiqhRwABSRAGyCfSmncdZt4fCZitbdumHPtYwWSEusqWGAS0Ck7XAsYyMIFtRLSxChoi22F4iUVNOBaDDcj2U5DNSr+zciGZWUFSxXGSL7Y/n596D3mX/xLDV37UXdP1xwryoKf+h7NyzLAMgkkE+JYcR7TB/bhjR2Yi6PM8WJXqWGQFR0tiWXQfCnsC6mTdKeLZc4FYEqBrwrduBb95EDx6JiBPC62wV9mzJpC+rGNiy8HJsBwuYdTZSZhegABNbxtDNpbvomLUGOJStTB/QvxlopKSsK+0iWxVQti578kOs7Uziiv/5GN+fNdnxnfseYFg6QADC6AQPI2Z0lp2Y8X7AXbJkicNDFYlEPOcbCEzTxKhRo/CHP/wBqqpi7733xvr16/GrX/0K8+fPr9h2ykFAtopAPrD8IjnVVJPKRbXG0ptOiro+IxukOkZSMUUZDSxjMZFH2cq6yRa7Xi7f1InFqzqxz+S2Co08QC1glRBG5O1l4pBufJk+6IaBSxfNg6WkAFwPflQ4bhpdq9FkyMoWv8GWdgMIq07PVl/aeYPpzXgpW/RVbtfTx8KIPLvRVn3ow4VQtow0LI0e45RsbaDToy0554oIcbkeSvozKZtICFhY29OBya3j2LLsFuE6DzVVxbDGEa4f5F36QVSQ58oWuLLlLv0Qh0y2QBQaFuVJDjyMyL4uK4zIFKWQ2QZd2YiMslH89XtYuE74PN3deo6/1zlByyXgXNkqVmfLXX6jU1WxXddaALTYKgBMGUbbASkppkxqDZJniw1OIm29epdjnSl98J6pjV3OhwVCJDHNdQqajGw1hiPohE22LJ4V7NG5AACamprQ3Nzs+Z2MESNGQFVVdHQ4G8t3dHRgzJgxnsuMHTsWoVAIqmofk9OmTUN7ezsymQzC4dIeoiqJIIxYDMTzra+Q7zfXv/gI9r7vm3hjzWe12XaBsQwGm/vtm9GIuJyN6AxfcDQyX0qySDaim2y9vrId3/3D6+joCRSuoYRSwoi84nkD+6tnASDdi/fbVwFqgoX18lSi37YKjRIZifFLY4lhRN4yJ8k8Wz0ustWX9VK2WBhRMv8nmfqjcmUrbZvjAcBSZaVJJlseYUQBRgRcYcRknqy1DX0doi6VIsKIHgqzy9DuNsiH2Vg9lS3YYca82YgElGyJn+FUtsoJI/KWMnF1BFu3fc3oy/axeXi5GNcxNsVljPeAMMgTFPRsuYlHp6ICyU5sS/SJVj07jRgHGFkQkThgK1uCHEuELml0OdbpDpkOBHKXBMB53XefgVxRborS445nI5osYzVreHuxSkU4HMbee++NhQsX2ts0TSxcuBCzZs3yXOaAAw7A8uXLHZ0Cli1bhrFjx5ZMtEzTxLJly/DKK6/g5ZdfdvwbCAKyVQRy9lOtG4CK7bpSsOQn8sdW34iMuhbz/vWzmozFfSGq1D7ZmqAXXMsMIRqSTga5Ca2EJqFs5clGFBdONk4xXh26aeGlZZsrMu4AtQEXmgodblzZauD+PEIN8u9skA3EHsVRTQPoWuNQtqIxVvCyxDBilGXH5lO2koZH5wIu+EjtehIsgUMTYURG0hjpsJiyRYwMLEYYYlrESbbcldgF2XIqW5RsuULwxMKW5Bbh2VJ44VEv76SbbLkN8hpXtni/IGaQZzfnkBKi+56FOxW5vh7Y3ycqkS3RroetrhyyxUjOsPConO/6WZHSfBXkS/HIKlxNBSno2XK3lOpUFSCxFUu3UJO+ZYYwprEV6NlAQ4YAlFBc9F4UypZ0zctYLGvRjBT8HeVgW8LV1F068fJ5tloY2eIGeZMrW9bgyd8FF1yAu+++Gw8++CA++eQTnHXWWejv78cpp5wCADjppJNwySWXiPnPOussdHZ24rzzzsOyZcvw9NNP44YbbsDZZ59d0vZef/117LDDDpg2bRq+9rWv4ZBDDhH/vv71rw/oNwRhxCKoizCi+7PHOCzyObDod8CsH1d3LFUKI25NsKc4yxW2yRNG5H3jitXZEqUf2DWqKUrfvLxsM/5rnwkVGXuA6qOYsqUbhsj+a+QkhhU1/WTL557LiFX1bgSMDBqkYy/WMBroXl862WIkJMXqbLl9KikPsuX2bCkESDDPVoiVkhDKVpgrW9wgnwFX8mLhkJNspdz7iJu3nedJfybtcQJb6MpslRQ2t7IlKX1FlS1mkHcpWzwbMaxqjmxRwrMo+WeAGtHFzwg5shENr+zKPOAkZ2RsFFa7uAj/24gHNNeypVz3uWeLKlv5SWCOsqWqQGIrVrAiq6rZQktt9KwXxEpTVFiGyrbDyRYnNCZMpRcEQMgcBl1pr3gFecCtbDl3CPcONjPPVU+SfjYMDVByCeZA8J3vfAebN2/GlVdeifb2duy555549tlnhWl+zZo1dl06ABMmTMBzzz2Hn/zkJ9hjjz0wfvx4nHfeebj44otL2t6PfvQj7LPPPnj66acxduzYiniTA7JVBPI+9isLsJRComkFwHOXVJ9sFZ0wMGxlqfGK5bqA5wkj8sKn+bIR+YXYbZDfflQEWA+8snwLDNPK22cvQH2hWJ2t99tXgRATlqViHLus0TBiD1b3rBbzqUSHYblaiWxbBQCIhIeJSfGm8QDeKTmMGGfEQ2QjZpxh6ozpQba4Z8u0PVspw022uLLFQuuMbFlmWngcc5Qt183WYvvDsgxqvGEbps2Zc/uK9ulbhclahDO92mYVU7YksmUBIOzByRKeLYlsqWEQ174mxLUNNeLIRixH2eLlCWJaDDDigGp7w9ImCyOaechWCetXhWeLFPRs8XE0GSZ6VQVbVQVIdGJ1FyVbEdJKZ+xeL5QtQgCwkJwII7Kxbk70gijst6kj0Iv2nL6wlYDjPpjj2aJ/z2Fx+rfiYURBtqzBhRE55s2bh3nz5nl+9+KLL+ZMmzVrFl5//fUBbeuzzz7D448/jh122GFAy3shCCMWAXFSen/GUOD0t6za/gmrVdS0K0kvuipxhUDy1NlqjRXLRnReOBV2jYqETDRFNHQlsvhwfXdFxh6g+ihWZ4uHCjVjOGLseOBFTTuSa8V8GuwbkTiWGdmKRulTsmVqiLSyLCl3+YE8iDOfFA8j9medZEu1OnOWEcVGJdWOe6XCnGxlnMoWJzum1FjZmY3YmuvZIjycB8eDiXc9Jgsps4sRMal/odd56KqL5Va2Ipr9LE+N4846WyEtJJHJppzrKwFx7n+XZ8ur/VA+cHUlqoWhWU5Ttm5R4pXNs75SVA2F8NpuKOjZ4i2lxjBCRJWtTmzopZXQGzRG+LvXCmKlEMDiWaFCDqXrWbGVkjTLDCOm0mNksJ4tXQqnx1iPw0IRHq5stcQo2eJhRJ1lIxrW4JWtWmPmzJlYvnx58RnLQKBsFYHjIPNrDAWULdUYBlOjqb8mqs+eq2WQ72bhkpCbbIknaudTb2sRZYuHERVXBXndzGL/HYbjuY878PMnPsBtJ87ATqObcpYPUF8wpYw9L/BQYZM6FhF9HQDbs9WntktXOskgz98wstXcPAFId4BYUaCJZoSVqmw1cGVLoWHEfl481yIAsRCFB9niljFJtUsZdLmIys4DmYwAgmzphk3m4mGXQb7f7UfkJRiYn4gpTpRs5WYjGkqPIH3CO+YVRlTDNGOQP+zkCSMClOip3CDPFJuwojp+n2eJG5k4qhqgRqRzuQyyxZp2R9UoIkoLdNj1mQzCyFa+MGIJ61eICljSPs4D3jx8rG7gszDzbGV6sSVBM+1awyzDUwojEkIAkxeTdXq2VnfR5VSzCSoj1V4Nx8vBlj6brDVF2d+wQISHe/CGx2IAMiIbUTc4ya98WLPaOOecc/DTn/4U7e3t2H333REKOa8De+yxR9nrDMhWERSST2s2hgKfI6QVSVCytUlVMUYKE1RlLFXybPUIspUvjOhUGNpYxmJSUaDr6ZwDObeoKX2XMXWcc+iOeGvVNnza3oujb38Flx01DSe5askEqC9wsqXmeZrgocJRse0QTdE+cToBsuk+6E2ihjnNoHeb7Tvp/HtutyemrhuBnYftBIyaSL/jpKsI4mFnGDHJlC3NjEJXk+hT4AjhAbmeLVUhQpUQzaXdZIudB4Zpe8JiIRfZcoX3LMIruYOSJl4QOOsd3iFKGttY2QFNKWCQJ4SqW1x9c4URY6EI9P4pmEi20P0uwois0r/qVLY8L1vuUKUaEu16yvFsGVYWILQAbIM2DP2SGG4Rui/ta4YH6SsCbpCn6mH+MB43k49hNb06GUHtTVHyNyJGq8ejex1UnrxgQfSidHu21vVQRSxMWqBViGxt6rWJvAhxS9/n7A+WqDG8kZGtlA7LsqDrdJ+YQ1DZOv744wEAp556qphGCIFlWSCEDKgSfUC2yoBvldsLerbsD+tCGsZkk3a7jqoMpfwLUSnoZS1OIqpr7HnCiCPithrVnezHcNf6eE0hPjx+jcqYWXxlfAuePf8gXPjXD/DSss248n8+xq5jm4PaW3WMYnW2OpJrAQJMbpmE8NYXAQBZEGzQ+0CI7TNy9kZkYMqW0jYFTx54Hp1mmsB/PwGM26uk8TUKz5YCwEKSkYgGM45uNYm0oqC7dwtamkfa23dVkCeEIMMUK6FsucOIzLtoSun0UdVlkHeHPtnvN10FNz2VLXaidCRo3a6Q4la2XAkDWtQeo0vZagrHkFzzQ8xWn0Qk9LFd+oErW2oISDMVLtLsoZqT3JCo1IjasEr3JvF5I1oYzaFh2CTxTKLo6Er2C8+W2xdYUjaiXPqhgGfLZF0Oxuj0Zr2FqX8JfQugAmMaONlaDwJ6PbIAEF6Cw+XZ2thHW/zE1FZBjLMDzEbUDQNHP3Y+MskRAPYBAGQMyTeWB9yDNzzeAKAbhmmhP2Mgq/MOAkOPbK1cubL4TGUiIFtFUCjltWZjcJ3s8g3HklK312ka9kn3VJVsefoqKoA+RraibrKVpxF1SzQOYlmwCEFXqi+XbIkLp6sRNZs+qimKB07ZF6c9uBj//nQT3l69LSBbdYxi2Yh9Bg0V7jpie0SW079xlhCsIs4bsumos+X0bGHYZHtGRQF2sHuxFUNjWCr9ACDDSjY0kwh62HHa3rneRbboq5yNmDEpC4hykiHqbNGHC8I9WyyrzTJVmoXlIFtuZYvVuwIcRCBlZPNK013ZDkCVyJZ46HGFVWXflkvZUhQCTSFIw+n7EgZ5VQNSdmmLXD8ocsmWFhaZxeUoW/yGH9UiaIu2AS5Rb2PvNimM6CSgpARvhqqogMEKuBYIb5ouz1aXSmACSFldAIAJLawlTfdaKMwsb1qW3b7JVWdrc4KSraZQq/DXDVTZ+ufy97BOfxEIAUTdGZbRhKxhN0nnyD0HuWcripBKkDUs9CSzyGRZT04y9MjWpEmTKr7OgGwVQR344wuG7mSytV5j2T1N3lV1qzKWCu2UhE7JVkxzka1eXhnbaWpVFAVRC0gSoCuV2/9OZ8XsuDFeEWFE+8QnhGDGhFb8+9NNWNpeoGFxAN9RqM5WKpuBrm4BAbDXuJ3Qy/7GvYqCW9paAQBhYxIy6mpeiJuuC6DnC7thYdjAL7CNERaaY7JIlpGtGAiaTBM9qoqObRuw8+Q9ndsHYEiqXZaRrRgnTK5sRIW1sFHFDYxdwgvV2eJhRJefKJX1ykakn1OsSXNYCVNZMU+9O0eYz6OXUkhVkDY52WLKFgs7RbTCYUSajZirbGlc2SrDsyVaOWlhjIwPB9jusswwiJLBxt7OQWUjOpStAmSHj2M0U7ZMQtCjKEgTev3bftg4SrBTXcKzZVqwG5OLMCJdfhtr1TMs0iaq5A+UbMn1ubTmD5Hdtj+yRvEwokVoj4BYKISWWAhb+jLoSWWR1VXQqP3QIFtPPfUUjjzySIRCITz11FMF5/3Wt75V9voDslUETs+WP3TLU15ncChbIc1ubVGzsVRmvUmdmlTjmlTYMJsC1rDU3Yn75ywTNYGkAvSkc9PqdbeyxR5PM67Qw05jqGKwtCMgW/UMw8yvbL2z4XNa9sHUsPvoiXjbzAJQsT5EL2+q0YhrD7gaF79+sjOMSABso14vxNooURkgmlndtxQ/zvjDg2Wh2aJka0uvs2kuP48N/mBAbCN3jIcC+fkc5soWnT439E/8BeNALJWeJ7znp1vZUjRh3na3kilUj8lUekAAhNUwu7Hz9kVuZcu5LTfCmoJ0xpXRKAzymssg7wRVttyerYhQttwV8QuBK0pRLYJxrKcgjBhUKw5T2YpN/V3SNcM1jlKyEeWipgXCeJx4NJgWYoaCpGpio6Yiq9K/xU4jxgE9tMCpokWBLGBZll3vzBVG7M1Sb92I+AhsTlCCPNC6VnJ2aqj5fWS37Q/DtGCYljPC416Q8E4GETRHKdnqTlBlK4Kho2wde+yxaG9vx6hRo3DsscfmnW+gnq2g9EMR1IeylX/LlnT7WKdpoqlr7cZSmb2SMijZagxJZGvt6/Qm0jQWGLlzzjKsPqkn2TLh9GzxJ8+062l4F0a2PtvUJ27oAeoPlggj5n737kbaqipkjoKmqojIT+iWhfm7nINDt9+dr0lakniHEAcAm2zRzzp7eIjBQjMjU9v6tziW8aqzxWsSxbmik2BZjA00UG4OnwqA+tEoVOkBi9BK87JnS4tB4aUfCJyVx40s3MoWJw3ciB1RI7Y5Hsj1gxUIIwKMbFlusiUrW2zsnsoWAb5CjcoYOY393JDwbI3t/8hW9IpAFN7UuzG7azFgEcTJdtAIHf+WRHdOiy8xjhLWzzMBTYKCBnne5SBkWWhkFdaXhnntNA3bNQ8Humk2LYm10nValp0VKsgWPcaTBv39o+MjRK9JvQwSKkNOmFDjq0E0SuSyhulStuxPGV1nbbBookZTjI5hWyIrPFsguqNtTr2CN6/m7/P9GwjRAgKyVRQOclEf/njHRcmSakx1KYqjInOtxzIYpBnZauJGYABY8QJ93f4Qzw1xstWbSeR8J55SeRVm9mSYdF2IJgyLIxpSkNFNrN7q0VIlQF2gUJ2t1d201lCjSv1QUUmxuWTrNhw3YUdEQ2E0G6YzhZ2gYmSrJUofEkxCkAWQ5WTLMtDCbjRdia2OZfhQ+H1IIYSWBgDQwH2XvIwDM05np/8Ap2Z+RkOCAFW2OOGINNNQnqw2haJ5W8mkPepshRRnGD+shqlyxuE2yMthPsWDbKkK0nD6vizCDfKag2y5ry6EABi1C/CTJcCZL7LtxUGYA3N06jPgw8dztukFTnLGff4Udn3zNjw0+tv4+/H3IETo321rogu6xcvFOFHKNU5VeJV+UpBs8S4HEctCIwsNLg3Ta5NiNovq8QCgNFAPqWnaxWVFGJxtg5dVaI7Ehb9uwMqWS5HTmj8EwEzyeW6DSekYiofCaGFkq+3jB3CPdiudn1hI5OnDOdSx++67Y+3atcVnREC2iqI+lC3XZ8dIbLLVrdaAbBXwjw0GGYumXzdHJGXr8xfp6/Zf91wmyq48fdlkzneG6ylVU73JlqIQUWcr8G3VL4RB3kPa6mXZcFGVHjs7ZFL4j75+nNWVxH/29tH+gkYWbYYUDgM7NiqlbEVtgpMmBA0JWvcrbhmU5AHozXQ5lhFhRKmsBS8A2RCK0bss95MxshXSVHxgTkWW8N+gOf1agNNUrsWEqmu6MuVoGNGpbPF9yBHRIkCSqWuR5rIM8gBVtjLcrcJDnULZCgPJLjH2vNeWlvE2qdPCWDHmP+jqgJJtEzx8F2d9GGcoWYxrbkNUoQ9329I90PPW2Sp+lVN5UdNiyhYve2FZaAL9TcuYsmVXj6fKlhofBoAeHxpPVODXezZWXiA2qoVpRX6Ul6UpIyUpW5fvfTOynQcAALK6W9mSlpHIViwUQXNUA2Bhr6U34yDyifiuJ537QPxFwKpVq5DNlkZuA7JVBPXh2fJ44mOQPVvdigKdX7x8GMtgoDOy1RJlJR0y/cDG9+n7KV/zXIaTrf5s7onMSz+IoqbsYpXwqDYvyFbg26pbmAXCiH2ibEgMsCxoego3bt6KH5Nm2wSf6kGbK4TsULbapgxqfE3hKCx2PKYUgtYUrT49TO8WYcS+jJMYCIO8FEY0mVLRGI4CyW12wdA4DSOGNAUZqLRgKwAFqlQ9npEtWdnSIlB43z6XspXxyEaMq42Oz1E1DPDwZ9yd8+vaVl5lyxVGFB6fEMDVvoYRHl4g74tLXwMNpRqECNJRDFzZivLfzxTDuEZ/b3eqW2Q3DkTZEgb5Ap4t0zRBFDuM2MpUxI8j9NrUoFJyhW6nsmVZFkKEhm8tl2eLk7eIFkaYlQUZaMV2Tpzi5o74zlcOh8bIW9awXPc++30yI5EtLYzmWAit6INm0r86Yedtb9rZUeHLiIBsFYVkDPRN2ir00X4yNQlBl8sXUvGhFFTZBg7doidja5Rd7HvbAVhAqAFo9i4sKciWnnsi86c7foBrrP1JErlka2dGtj5aX12/W4CBo1CdLZ7JGtcanH0BG1g1bpbdNcww4VS2KufZUhQFsFhldkKQYeNszXQKspVwNaMWFeSlshYm4WQrbocQo61CUQqpBDo04dmiZMutbMktdaJ2KxkCBzmhWWtOZash5CJbWkRS10bk/nCZbHnUSAhpRPJspWCYhvCDhVXNESb1LP3gAU4CihUQda6Mky02P9tuQ4ie+73ZXpHFN5CbourIRvT29PRLylHEsjAlTMlrP8vibGGf0U3DUoSRW9OyS3BYrqKmvIxGVA0jPEjPFi+oqzAlMsQqCGcNM29xbx5GtCwFYU1DSyyEkYQej4T9TgDo/YIqW+UgIFtFID9J56vxU8sxAPmVLQDY0uf0hVQa7l1QqT7OJqGEqS3GSjzwzK2m0XmXiQmylRtG5IZM4dliXpOEx3hnTaUXtVeWb0ZfuvJNXAMMHoXa9fDkingoLoWqAMQ52eoF0j0Y5jK2KgRA12r6YZBkC7DT81NEoZXkAURN27OVMN03HDqPbsiqHWtUHInl+LUAIKQoyEKjxAmsAXIO2ZKVrZggAqaXsuUiW81hZ+uqqBaRlC0vsiWFEUvwbKWy9vkV0cKO3+i+luS73gqyVaQ1DoduGMLEHWHlEfh2m8P0etOf7RXWA57BzFHKZV8Tni3kHVO/1Jw8DAu7No9zfD88ZrfqAQCFJ0VYlt2Y3K1scQ+YFqJFYlFelqYMu/m4TewB6tnKZ6dJcALJevQ2R0MYRbaJ78PsvO3LBMpWQLaKoA788blPfHlKPwDANp69VK2x5IQRK7NXLEa2hnOy1cfIVmP+mmEjmFH21f6P0Jt2Ei47JMDJFvVHJD2Gu9u4ZkwZ0YBU1sS/lnQM+DcEqB5sg3zud7yfYEOoQcp4AxBnRWozNIw4zHQpW+luqoQpGtA8ftBjJEzBuXbEMLwZpcdb1LKEZytpOW84uUVNibh5NoSjOX4tgHrWLEUTYUSVSGSLZa85TOs5ypZMtjI5BTxbIs56dnFNHodHGFH2h3mUfgg5wogpRy2nCDHtscdHlGxR4MSGNn0uTizkG71InuijZKuJFYtN6v1SP1UnSvJscbLlqtIvoz9jH5shC9hrxGTH92MaRlEJl3m2bLLFEhUAmJwIGjyMaHu2ONkasGeL7RueTBTWZGXLO8LDQ4+EqWHNMQ2j0CW+jwplK/eB+MuGgGwVQaFu57UbQ6HPTrLVlemu7liqsA8yug6i0IvwiAb2ZN3LSE8BZes0tQXDDAPdoW04/anrHd+5/RcquynohCCbdodzCI6eTp8y//H+hsH8lABVglmgzhbPZG0JN9nKlhalhm6Aeba6Mac/4SwsuGYRfW2d6KnKlAte5fudaBRbNLq+sGWXfki6GvIKz5ZQ7SD8TPGQpCi5wnchVUWKHdkqlCLKVtQOcQEOIuCVtTYs5iRbsVDULj/h6dkqofQDJ1tGxpG9FuUeNqIAsWElJ99oMrEpwbPVJ5GcCCdbiS2AaSLOHsJ0KzMoz5ZjH+fxbPUy0qdZdBtjmkZA0e3abhPjzXRfs2NYYX93y7LsMKJL2eLV26NamPrrMHBli4cRVYUrW4xs6ZZL2ZI8WzzL0KK/n4YRuwAAK83RQtlKZANlKyBbRZCvD6F/Y3ANgz3pNLALenc2t5p6Rcfi/lyBXbIlYXulRjSUrmyNC0Vx5RZ6I/i471lb0oaHsiX5WBKp3FDrt6ZTX9hLyzajOzE0ivB9mVDIIJ9hTZmbIo0Av5mrEbt5c7oPSPdg50wWo4lNtwjPYh2xU0XGqCu5qnJUJluu4o7Cs8WIJG1ya9cs8gojAjS8k2I3d2+yJdfZijrDiBIRyFpZcf3gaOPqGAMdR6EwYmGDfMRRZyuFjG4TgWjaVrW8qs/nu7ZwBadUZYuH7yyLIMSVT1MHUl2Isor8hpXN8XmKcRTdguzZInk9WwmJbAEAQjE0qhPE91MjYaCHqlpoGAWi8bCgJRqTW646W7wafyxkG+R5lfpywSvQa8RJtjKFPFtZl7IVDWEUI1vLrfHCszWUyFY2m8Vhhx2Gzz77rOi8v//97zF6dH5BQEZAtoog30FW0zG45XXpMw8jcj9Kn15dI2KuiXXwO4WTLctS0BRmYYkSlC0QBYcmkggZIRAljWc+XSy+si+cTDVQwtDYiZ9M5t4UdxjVhB1GNUI3LbyyvLpJBgHKhwgjerAtO5O1QVK2ZLLVK4r9EsnETUbsCOx/LnD4tRUZI880kxG2LNuzRVzZkOzc4cqW3OsvXoBshTVFkC2NeJAtR52tmMhGpGUJJLLlUUF+ZNwVRgxFB2eQVxVkJGUrxWttWQQhfh6y31dq8k25nq2+DCPVlgpFri3Wv0VU6jdkZSvHs1X8Guc07XuPiT8MhgTZimJc3M6C3SFMRAgRLeOhsmPdNFlxWdg1ygTJJHYYMaJVRtnSFKdni/ZH9I7wiC4ETNlqlgzyMtnq96iFWK8IhUL44IMPSpr3e9/7HhoaGorPiIBsFYUjjOjXGAoY5LkHZRjzhfRZ1X2CqIqy1c9uhGaUZnUBJSlbUFQoACZalJA99/lrAIDrXnwEK1MvAgAa2AAtKIixi1zCg2wBwCE70Yv+S8s2DfCXBKgWCjWiNkCP+WHRJtuzpUVFP0EeRgQAIpE1MnYP4BvXAiMro2wdPfZ8TMw0oFUy4svKVkK1HJW03Z4tXVJpYqGIRLbcYUQFGXbpDkF11KoCQBUmdsOEFnUW3JSULZq15jLIR+OihAXAvGODMcjLYUQAaVamA5aSEybNIVcV8mwlWBiRWJozgaJ/s6jUbyCTUwi5yDBcY5IJoPeYeIV2HlpDKI6d23aky1sWJlhpUfYBLduJY920LKHmCWXL0B3G/1gojKiYJ3f7GV0v6pvi2ZgaKZKNKO2RtMEzOOkyLbGQ8GytsMYJspXh5UmGCP77v/8b9957b0XXGfRGLIY6ULbc8PJsDWMX8f4B1lgpedsl+irKQWeC1rcilmS27WOEh/cx8xwMvejOiE3Aiuw6fLztHXz/iWvxXt9fQBSgFXviK1oUr4OWxYhbQC+AZLrLc3UH7zwS97yyEi8t28xCOnXyBw9QsF0Pz2QdFmsCeGaqrGxl+kTxS4eyVeE/7w3fOA34348wf9WT+FsTJXphySCvE6A7lcCwOP2Ob94UZMsmaTEtXMCzpSBdSNkCqOKUzgKhGFTdu5WMl2dLVRQQMwKodJ/GtYhUC8vLsyWds16eLdVJtrKiBICSo5iVem0JycSmBM9WQiRNeJEtVvnfyooCoe5fUVoFeV6lH3k9W/0slMYJCLQoDp2yN/5nPTAhq0NLbAZ6aTcENG/nKA0S1aIAMg7PVtJRUDSMqAgj5oYxZz10DLJWD17973/STFcPZNm4Qyxk6TDIyzPKypYIIzJlK6rBZNmI660RiJt05vQQI1u6ruO+++7Dv/71L+y999456tXNN99c9joDZasI8hkDazqGginRXNmiJ1g/qW7pgtzMyMGvc1uK+sxUSBduUfqhsLIFAIcNp1J8L/kI7/X9BQCwS+xoLJx7P6IiJKAgxg73ZJ7+kftNaUMspKKjJ41Pg2rydQUeRvQiwBahN9Ph8WanQZ6XMUj3eIcRqzHQ/c/BZMtWeyKWhQbLgspusOt7bFXVXUGeky1es6iwZ4spW/nIFvdtaRHRhN3dJNnwULYIISDS+BvDUZtsFVW2PLIRNQU6VJjs3Muk6blOLDXn93n2RvRAqMw6W1xRKqRsmcjm+DyLjcMxJtEbMb9ny0vZOnTqHpiv7oGbN22han6Pl7IFQaTsMGIWKalyeTwUEWFEy0W2etNJZNQ1sLQuvPj5R3l/A/dshV0G+YxuOe458nuewcj7b8phxE1WKxQWXtSrnLhVaXz00UfYa6+90NTUhGXLluHdd98V/957770BrTNQtoogX8prTcdQqII8oU8dIoxIDJo+XKPBVkL96UpRYqMRRrb0jN0ipFAYkT1Jf7VtLLA+DqgJWJaC/xg3D7/4xhl0FlaB2yIK4kQBYCCR9j7xI5qK/acOx8JPN2HhJx2YNrbZc74AtUc+g3wqmxGZrMPjTUA/e9rXwk6DPHuyJoqsbFXhHGnbHqGD7gDePxcAEDEtEDWCZtPENlXFxt5OfGXMRLZ9ukhOGJHdoGxly022FKRNHkYkechWTLxy1cVweba8lC0CQEVMBKIaFdMmKAOsIA8Q6EoYYTOFrGitpeSESXO9qd7QVF7KojTPliBb0JxFb/u3oLGFjt8ksrIllQcp8RBRHaFN7zElmcIW4V0BGNE7YfK+wPL/pT5VybMlhxG5Z8sUypYh1gfQavw2IXMS0NXbNov3sZCrt6UEfjzw1kCyZytfCaS0nmXTWOjRTCNEqHq52WrFVHYsZ7ND6+H1hRdeqPg6A2WrCEie9zUdQ0F5nZ64rexpqltVaKubGo2nEvukhz3tiia4fcwcr4TsWkleUHgoxcQ+w46Bqo/Cz3a/SRAtQCJbIIixp69kxlvZAoA5X6Hk7q9vrxOhqwD+w8hT+mFLws6+HdngUrZkz5ZXGLFKYz10++n2BwKgeZzwbXX02QUf3WQry8gWsVT6wMFDLx4G+QwpQrZ4eC8UFX4iE3CE3Qx4KVtAWGpG3ciq81Ol0MMIXKT0Q0Tj5nx6A88IsqXmkMnC3lRpk0x5KVfZUt3aQv9mNIgwaBZZno0nj6Ho2tmYVB7azD8mPo6oIFts33H1vq9d8mxNEA8WlgVEQ25lSxeGe8siCKuaRKSc21/TbZOt/gLFRTnZ4v6wMCtf4g4jehnkuWeLX7tTVgg9iIsHB32IkS0Z69atw7p16wa9noBsFUF99EZ0T8gNI7YxZatHUUpuzlqR8VRgl/SwRsIRhV34ONlqHF340ZLfOE0D9x93Kd47bSFO3nu2cx4PspXI5D/xj9p9LBojGlZvTeD1z6tbIDZA6cjXrmdLPyUalqlSL4owyLs8Wzx0LJcYqNLpPK65DaOVWdghDWyfyQLN44Vva3OvnXwhshFzPFuaHbojKm3XIyGk2mQrAgAG+80OZYs3bo5BASc8TiXIsPScfUBAEFZsUtWkMzIbH+F9LjqULe9sRADQmVpiMLJFw4gusuVaNj/ZklrjlODZ4gqQ6nZj9W+mYVJQNShjsmzBAVzzVbn/ZJ4x8UzMsOTZAmD7Uns2SJ6t8WLbpmVRDx8AENZS28giycOIlgpFUcQ8livrdWOvnV3dl81vkteZ3zfM/lZhh7LlnSjGi9SqvKQKyyLfbLUCILBMRoz16goAlYZpmrjmmmvQ0tKCSZMmYdKkSWhtbcW1117rSHIpBwHZKoJ6sEgX7BlGnNmI3YpCn+RrNJ5K+NhEI2H+RF1Cqx4AdtjCo7k0hyOMyJ7Yktn8J35DRBMFTv+yeG2xoQeoEex2Pc7pW1nZEGIxj5KXZyubEGFpIqkv1fRg/uv7f8Df4ztRa3jzWMR52REpOcM2QLOhc2ULmjPE5iIxIdXuvRgDJ0/E/r2ARLYizrIEDs9WBjnKlp7CCMsOtcX5g5uXOR4oqagpAGSFspVgo5U8W/F8Bvk8ni2N/578mX8yuAKkEbeytQVNETp+omShm/TYCUmzlKxsKcWVLV5t3fZsMaLKrRJblgGWQRX9xtFC2aIGebtdT4plPKaFQZ7ud14zDC7P1kaphVtCKvDqBm9gzet12XW28nvY+Bi4Z4tnkW9CKx0tq7GmG0OLbF122WW444478Itf/EJ4tW644QbcfvvtuOKKKwa0zoBsFUF9eLZcnx0TeDYiCyMqKkxeo6oG46nEPkkw8hPVeBixhLIPgH1xz2NIBQDCLjwWFMR4M+ps4Zov392XFhp85sONQYHTOgEnJKrLtLU16cpkdShbUlPlHtoZwOnZqs5YBRrZw0JsGFRWTsHI2GFPvnmubPHefA7zuIcpPaQqyLJLd5wrVdFmJynjvz3caIcRXcqWaeWeN+TDv2DfnjcAAJapQUltyzsOAK52PQXIFqMwOsvII44wIl93ack3nNi464blAw91ae7bXf8mNEXs8Wcseh1ykK0SjxGb0Ob3kaUMSdlSNNFcPCcJqHksoCiOOltRjY/KREohjmxEwkJ1DQl23SSGQ33ZLLVw6y+obLE+i4qTbGX1/J4tHvoWyhbLIt9ktQIADJPVMTOGTlFTAHjwwQdxzz334KyzzsIee+yBPfbYAz/+8Y9x991344EHHhjQOgOyVQSVJhYDGkOeJz7TNEFcylZGIejtXFWz8VRilySYxNzA0rBFQdNCZR8ASdkqQLaEskUQZybTRJHCr3ts14JdxjQhrZv4n/fXFxl9gFogX50tnlwhMlk52VIjlHCxNHauNhBFVraqjO32pa9j9xShPFM69nKyES1Z2cpfSDSiKbayZXqEEAHggPOA6d8DdjzcVZbAVl1ML89WXwca2TlDLGkcXuZ4wCZ1RLFre0kQN2ymbOmsNIcCAnCFOU/ph3wIC39U/sw/GTx8F8ohW5vRpNnHQ5aTLamoaanqZ0iVQpt5lK20rGzJimBsmHPfNW8HAA6DPN+PIBbNRDV1u1UOVMCyMPLJH9BZiIWM9HfulOoKJgqQLV4Imnu28tXZkndJxnSHESnh28zIlslaWBnm0CJbnZ2d2GWXXXKm77LLLujsHJi9JCBbRVDpkNlgx0A/01ddenppMk1RIX3zluU1G08lCGiS9bYTZIt7tgqVfQBKU7aEZ0tBjD2FJ/XCJz4hBN9h6tajb64NjPJ1gHx1trpZ2RCNsJCMrGwBtm8LABQNRCpPUPWHp31OAX72GTBjrsjWkr0rfPOmKxvRGUZ0muMB7tmiS8fzka3tDwGOuxOItYqyBO6K66ZH6QfoSeEvA0K2/9O9fo5oC3Do5cDsq2gGqAtc2eJV5HXe90++9bDz3k2k85Z+UL3DovnAla0Qv15wNS7VjYYlfwdhx5ZJmJ3BMYiiqwcAV+FYb7IlSitYcCqC7qzKHQ4TkwFKtux9w8KIRtZuAm1pQKob4T47opFe8pR4353pEu8Ltc0xWPiYZzWGNcmzBe/7YIaNwVa26Bi4sgV2XhpW/vBlPWL69Om44447cqbfcccdmD59uscSxRGUfigC+Xz3KqhYy3Hwez4fhi4pOgosNJkWtqkEXV2rqzsW6b1XRe9ykWU3jBhPV5cN8oXAwyYlKFsAQZyFKRN6/qc7juNmjMeN/+9TfLKxBx+t78Hu2+W52QSoCfLV2eoWmaycbEmeLQAIN9pm8+bxIFJ19Eocu0XB1FkFKgAThqyqiocmJ9lSiCYV/PQmWym4lC3edNsDNhGAg5xYyOaQCSWbRCPhylaIls0AnCFZN752Yd6vItz3w5Qtk4XSbLJFhELt/mvku96GRRgxP7GRkWQEI8y32TiKhpVNHcq6NxGxGIFRk2w+WdkqDaVkSPJ2OFTZijq/nHoosOLfwO7/BRx4AQA46mzxERHJs8XN6QQa0LcJIemhMLPhHWD6CQCA3myXmJ7S85Me3uaH+8Nkz1a+tnW5YURmkGeeLcJ8uIbkAxwKWLBgAY466ij861//wqxZswAAixYtwtq1a/HMM88MaJ2BslUEzjCif2zLK5yZldqCqBYQZrV30v1V9mxVOIzIs2B4s9WSCpoCdhHFAhdc4dkiBDFmSE2axU/81ngYR+xGt//YW2uKzh+gurBLPzin94rkCh5GlHojAk4SMmySY9lans4Ka+5reShbHHa7GC1vqx6AebbY4CP8WNaiOfNxaIp3iMtd/BIAiJ5AI9/X0ACeuSsrhGUgxNQRt7Kl8l+vhsUfomSDvKxsleDZ2pamvrNWlRHyUNz2oLV/aBvWGcIDqLPlyJDMM6a07NmSw4gAcMzvgP/+G/DtP4iHSE62LMsSYXQQS3i2UnL19r52h3KSkXoRJg07O72Qqs8bWEdcBvmc0g/Se1Eug4dB2bWbK1uKSkm6UeXOJpXGwQcfjGXLluG4445DV1cXurq68O1vfxtLly7FQQcdNKB1BspWEVSaWFQCnPTJ7T0UAKqlAdCRTW72XrBS269wDyOTpaBzr0DJylZsGH3N0+sQAIglGeQ1Gq5IlhB6AIDv7DsBT72/AU+9twGXH7UrYuFcA3CA2sDK49nqE2VD2M2Le1I4+ZAVmdaJINukhWt4RptKHEAaihTScT+8cc+WQnLLIsgIawRZtmi0DLJluirI0+KXzudtkk1iJDtnNNJgZzaHB0a2wszLlGJZaYbByzCw7ap26LFQ8WbnOsvzbHWl6fVhuCj0GqH7ta8d2Pwp/j97bx4vR1Wmjz+n1l5u3y3bzZ5AgCQsCSQsYRMhssgiiiiKoIA4OqCjGXWG3ygyMIr6ZVxQBkdUwIVRcQFFDbKqSAAJJAQS9uzLzXJz19676vfHWepUdXV3Vd/u231DP5/P/fRWXXW6btWpp573fZ83Mm0iZLMc3aVsBczZClAhydvhGLCL/1/tU+mfBKcaEVIqg40Ur0aU3duHd4MA0G0bOUKQlUhVxhoS/+YgylbUS7byVsnUEW8/RW+CvKLT46bg06+x2TFt2jR8+ctfrtn6WspWBbiJRePG4YecpOgoABSWjJjP1NcfqtYElN/1RDSTTp68L2IlZYtXK3IlzA9ygjy7YCQDStrLDpqAmd1RDGXy+OO6nYG+00J9wMOIXrI14q1k5Q7c/NiRFZnOxilbFrvDV/POZb1I2WLnswq9orKVZ4OPcEd0n3wpsbyreq+SspXC0ZkM/nX/MP6/468LFkYsA56zxfsj5tl4HWXLSQwP2hvRCJmzxcNokwTZijj7tZAtUrbMKpQtjXBXewTI2fJRtnwg+2xZYkg2Mixni7u3K1DFHMhDidmcQ6ryxLECypSpCrSYjQi3kJB9ttI551iRr4mCbCk6nbtH6NzNE+R1nSrLBVKZFDcTVq5ciSeeeEK8vu2227B48WJ88IMfxP79+8t8szRaZKsCmkXZciUoenI9AEC1bRDW7iZvpR0Tx7qMRXpeg53Cq2BM1aD5NXaBbsXnrt4FXq3IyZkPXA7yBr1gpALcDQOAohC8fylNlP/N86N3EG6heohqRE8cMZX3FFfs30Qfu+fSR8OjbJUwZ6w38oxsKVJIx3vu8PNAJZUT5PNc2eKGpmWULVfythziInkQbzWiRU0lPtLfh/ccchQ1hAWqDyOyC3aGKVsWyzHjJqAi3OuDksqWJuVsBXGQLwwAAKbwPCmubDGYRWFEp/AocM6WS20roWxZZXK2fOBYP8hhRMf6IVPgZEsT0QBBtgop9l0LtuKErrOF0jeanHxHfHy2BlLOcSP/X7hdia7obO62YINgHyjJUiK0ijWPQiBi3Cz43Oc+h8FBeq6uW7cOK1aswDvf+U5s3LgRK1asqGqdLbJVAc3gIE83Lj9lYUSPsqWyOpoUIU5D03oMpcYVmjxXIKLpjkoVm+C66/WF3OaiBGSyFTNpknvK546+FM49ihqcPv1mn2vCaWFs4Shb7vddlayW5ZCtLka2XMrWrIZZuVgaPfaUguyz5R6AQ7b0stYPuqqIoIxZ8OSo+UCX2/VINxq2T2jHNaLh3VIYcXTKVtpmZIRbBQRQtkpRHSNkzlbWphfNqYJsRcqSLV0mWwEPEk5QyvVGFO1wAipbchhRVrZSzPqBEydCVHHDqdo89EdJ7Y6hfhDi/J5yypZN6PiiPEGeq5L5ApJZSdmSdonTT1ETc3chOgEFZrSqR+nxmyVEeN2NB2zcuBELFy4EAPz617/G+eefj6985Su47bbb8Kc//amqdbbIVkU05k649CikBHlPzpbBknDThDg9tsZoLKOBJcKIkeC2D4CT01VW2aL7yAJBlJGtZBnHeS/mToxj3uQ25C0bf3m1vrlwLZRGqZytDLuDb9PbaKuTQoZagnRQRdLVz69zlkepHsOcLb2TbtNKCsLjPXdyjGzphFDXe8A/Z0slQtkyuYpSLmdLbtwsqwukABDbs7T0emSPFEasrik7742YEjlb3GBUSpDnwylhceMFz+20CBHKSjkUFEoYpxtsW5rpIrEy2bItDaodXtnifQlzhNDUBZ+WLqIdjo3Kqj3cPluW5eRseasRVejihpOAJ7XT/byl3z03ZssWB9FjKaa7qxH//sZ2QHFImsxN+W/SFV3M3VbcybXtiNLzL0MIMDB+OnIYhoFkkp6DDz/8MM4880wAQHd3t1C8wqJFtiqgVMlrI8HvePLCcZpOCjonW0qdD+wah1ZtWdkKmhwPOGHE7LBzUfBAEbYQCqLMKyhVdIEpjzMW0O08sqG+VZ4tlEapdj1Zi06I7WbcUbU6ZwJM/YDk2I7E1Iadz7ZOjz0Ltuh/5yUXXNmKcGVFi/gqSnIYUZT7q6VztjTZ+oEpEelcVhgiy3CFFYd7pWrE6pQtfsHmCfI2V0L8yFbRWPxhSGpYoUJoqj81AqJQlWeGIFullS1i61AsfxWnHHhSOa8S9QslusKIAeY32WdLrkbMeBLkVZYgTz9mIVYWXt464L5B5DY7fuA9FbmyxXO29usPQdH7neWk7/AKWl115m4i3Sh3RKiCl1GIk085DnDyySdjxYoVuOmmm/DMM8/g3HPPBQC8+uqrmDFjRlXrbJGtCnCXvDYH23Ia2NJJmf8TDUVStuoZRpSf10LZAq+CMYPbPgA0ROQ1QvXAqUYkiEVo9WJlly033rGAToyPbtiN/SPjyy/mQEGpBPmcTe+4E0YbsH8jfZOHEAFAalUCRfWYM44dVEYQcoQA+zf7br/ALlxRHuaOT/I9wXRNoRYDgDAyLqdsiTAigVDVHPdxT86W/KKGYcSUJ4yoBQgjlgrhmZxIw7nYl8IbfXQ+sS0V3Xx1munqTmFKu4DAEHYx5cbgBU8qz6E02RK9B227cncMuH22XGFEhQCFvNvjis2bNjMY4MqW3BcRAPLllC1C1xc33MqWorsLriyJnLqULTYGrX2K+F92RSnZGlIU7Nv9WsXf3Cz47ne/C03T8Ktf/Qq33347pk+fDgD405/+hLPPPruqdbbIVgU0o7LlDSPyG1STGQemiFKSfNRm+8T3ebWw2UkeNcxwyhZQMUlebtcTjXQDAFIKgRXADJHj6FldOGRyG4YyeXzpdy8F/l4LtUMpn61I/jDkBg/HpOhkoI+RrW6JbHW470IblYNpKJLy0b+laCyAo2xFuRpbokWOrioosJNeUJUyOVu8b19esn4YyflfdF3K1sBWhzRUnSBPLzFJi5EtfnHmv72c9UOJdfIEeXl9pbCZhdEUOwFFtsmQwoiGS9kyQEalbAE5AN9c9Ru82eeeg91kq/L85uez5WdqqhJF2N/YrCKdVwk+t+tF1zpzJSqx84WCyO2KecgW0ZKQSbklFWbxGwRT1R11LdGDI6d3oDOm4+3zDoNRUDGsKLhk8/14dlt9u5vUCrNmzcIDDzyAtWvX4qqrrhLvf/Ob38Stt95a1TpbPlsV0Cxqlgw+N/AwoqNsGYDF4uNl2jKMFrW/Rkn+LmGULb7c/o2lk+RlZUtqppvO9Ltel4OqENxy8SK85/Yn8bu1O3DOET0458iplb/YQs1QKmerI30Btu0YxKy2Q4GNd9I3ZWXrbf9G1ZnFHwTQuLxLnStbQACyxQhEibweQyUO2RLKVpkEeZ++fSlhDeBVtqTX+96QNlqlsiWTLcXJzxQXHlfOlvu7peYZWdkq+ORGydg+SAmAjnZ3dwFp30bkqm6Yro4UQY8XnudkE4K7O9rxo01fw92v34k1Vz0iluG/3QxMtvj3JJ8twslWThAqEXZWNNjshrtg5zCQTuL5/j8AKtCJxejHGuRLtM0ZkawiYiz/jCfIE9XdS1baXcI/S1cMYIDNwW1TcO/HlyGTt9Ae0fG1qe/Fl3fcg116Blf8+TJ8YuZnK/72ZoBlWXj99dexe/duV2NvADj11FNDr69FtiqgaaoRJfC7HO6zxclWRDUBi+VsBWhJUy1qHUa0SQEEbMKqVtna8Htg3+vASZ9x8nVs2xVGjEQ6QWwbNiFIJvcGJlsAsGhmJz7xtoPx3cdexxfuexHHze3GhLbSF7gWaotS7Xpc4UURRpzjLBDrBi78H+d1CXPGeoPnGeUJAfp5GNE/Z6sS2dJVRQS69CBhRJVXIxKRs8UvrrbtMTWVX/S9yVYQd1pjhYRIkC8wssXInm8Y0fPdSqamQOUw4s5hWtUZUTqcG1At4jjIQ1IHAajEFHMGHUOwgyRmOKRxZZy1qNHcajsPeQcNI5b32coLk1ShhMYnUxsI0CrBm//6M0AdBsl34vxDzsdPNq4pqQSOZJ2b87hBjyWes0XJlqRs2fJzbkitSf6IU2BqKkzW5Hv5/FNxxLPfwD/1TMObxjC+9cL1FX97o/HUU0/hgx/8IDZv3lzUG5cQgkKhcmGGF60wYgWQEs8bCR5SETlb7FiIqJL1Q12VLTnvpRZ7hSUG66ZIHg6sbHFS9uKvgUf/C1h3r/OZ7bSZsEFAFAVxtq+SvPdcCHzyjHmY35PAvpFsK5w4xuATvOqZsXhIQ1WIfxjRg0blYPK8qZyUT1lK2Ypx64BYt/+65DAivw6UU7aIbGpKLxJpoWR4yZYNcPNPrmxVGULkYwWANKM0vBhGEBw5sd+rbJX4/yiKAjCSyPdZKexhOXttWqdb2TJiQq2zFOf3acRwk62ya3fAqxEBQPU29wZN1M+rdG47OJcLRLa4z5ZtO3M+YCGlKK6crSgzI0ViCgjbs1k7j5VbfwkAOHnye9AR4W1z/MOII1lH2Yp7qhGJ4r5xd4UReQWtKtn2tHnm7o4Z6CkUcM/uvejC0SBK87vJf/zjH8fSpUvx4osvoq+vD/v37xd/fX3VmYaHIltz5swBIaTo75prrgEApNNpXHPNNZgwYQLa2tpw0UUXobfXHbfesmULzj33XMRiMUyePBmf+9znkM83784v1aagkSiwCw9vRM3/iVFGttKEOBNLHVBLZcuyLBCF/o64pgODjGy1Tw+2Aq8CtuH30sod00abjZqTreEqyJapqbjlYtrx/Q/rdmLXQP32cQtulEqQF2andsFp25SYVnI9jcrB5H0/cwQA80AqTpDnOT1SNaIPdE2B5Q0jqmVytlTJ1JSpIak882iy3S2oCGxgwjz6gqvjVVYiApKDPLd+EMoWQ5mcrbJMh5Etr3eYF31pmiDeYXSJ/S6IKcvbKmhdYnlNMavK2YpICp3mU+z8h1f/ARALk/J59KhxgPVpLQc5P1GQLeLkbPFk9yhX92ITaeseAH/RCihouwArgutP+wjiBt2eDX9lK5XnSqdKyexQL47+61W4z/j/ipStgvT7RPNqVa4k9xDJ9hkACOK5FB5999dxSPTMir+90Xjttdfwla98BQsWLEBnZyc6Ojpcf9UgFNn6xz/+gZ07d4q/hx56CABw8cUXAwA+85nP4Pe//z3uvfde/OUvf8GOHTvwnve8R3y/UCjg3HPPRTabxZNPPom7774bd911F66/vnllRflk8070jRoHv7PIFzjZoq+jbHJOK8TpEVeXsTiD8SYsh4WcK9CWH6JhDqIAiYA5UV6y9cYjjg2EVSgiW23skB9Juat0guKI6R1YOrsLtg38fu34Mekb7yiVs8VvPPS8ZPEQKe0JJX97tMduGBicbIEA3MXb+1u4wssvbCXUKkMlsFgys15hWbq8XI1IL7bJcsrWpMPcKxiFsuVt18Ndyv2ULe//o/x8S9frbUHkxWCWtlbpjna7lS0AOPJiYMI8jEQdJdRQIi5lK6i2pSgKbIsSV81H2frr5mcBAEdlslACpkjI82xO5AzJOVv0dxt8MT0iGp4/GqeP8+PvQE+iCzFm6GrBX9lKcmWLVY3iuR+je+ffMEfbQneBZBPiCiOKY9ZyvOG8v08zRKRCG9qO7533b4F+fyNx/PHH4/XXa5vMH4psTZo0CT09PeLvgQcewMEHH4y3ve1tGBgYwA9/+EN84xvfwOmnn44lS5bgzjvvxJNPPomnnnoKAPDnP/8Z69evx09/+lMsXrwY55xzDm666SbcdtttyGabs6Ter01Oo8Hvcng1Ig8jRtkJVa2ylbfy+PEL9+FXG1aWzYWopTFkUqqKiiWZJ0xiqpN3VQlyxZZq0N/9+sP0tZUvKvSPszu/4XR1/a0A4MKjqer22+frZ6/RghulfLb4vG/kWUsSLVq280CtK2mDwpRztriq5FmGX7gMfrEu4Z2lq46yJVSUANYP1N2cbiMtxuBRtghTtoh0aagyOR4ANMagMqxKLpKjirJwaS9nalpuxTyMKKl1fhjJ9wMAJkUnFCtbp38B+ORqWBEnN85QvDlb5QbhBd3PqsS10mx+e3n/OgDAokwmcD6qS9kScpKjbPEEebEHVRMKcfanbSv4j1M+BgBIcGWL+O8rTr6F0vnqSgBAv0+unhxG5MpWm8Vu7o02fyWUmwyPE6+tT37yk/jXf/1X3HXXXVi9ejVeeOEF1181qDpnK5vN4qc//SmuvPJKEEKwevVq5HI5LF++XCwzf/58zJo1C6tWrQIArFq1CkceeSSmTHEOtrPOOguDg4N46aXSOTCZTAaDg4Pib2hoqOSytUaz9Ea0XdItfeQ5W3y6jDNpOk2UqpStvtQQ/t/zX8R/PvO5slU+tQwjyomZUU62POX6ZTHxEOf5sVfTRx5KlMOIbKBxhZOt/qrGCwDnHjkVmkKwfucgXusdu2PxrYwCL7jyqkHsZNDy3HyzvArTqBxMQ+M+W6Au9yg+d6yiMKK/WqUqgF1VNSIRTZJLki0AiHa6825GoWwRQmBoCvbaNPTCG2h3ZVkYv4oEefZFAO4KSz+kLer23dM2oVjZ4mvSnbCQqVSXswU4REW+qO4Y2g/LstBXoB5Ti9LZQPlagPtYz8thROazxfOlDOkYUBRnf05SlmDx1DkAgIRZnmzxsDKgAcN7gO2rAQD7FX58yHYPkuM+u0GI85udUkSSz+n948NF/qKLLsKGDRtw5ZVX4thjj8XixYtx9NFHi8dqUHU14n333Yf+/n585CMfAQDs2rULhmGgs7PTtdyUKVOwa9cusYxMtPjn/LNSuPnmm/Gf//mf1Q51VHCdbE0ibYkEeU/OFidbqSqVrV1DAwBo3D6dJTBLHB213A3cXNG2FRhDLCwXNF8LoGTrQ7+hJ3N6EHjqNuDVB+ldrF8YUTEAK4WRzEDVY+6KGzjtsEl4eMNu3LdmOz531vyq19VCMDhhRPf7lggjssm+TAgR8By7DcnZcsKIxcoWawRsF6s+MhTFuREKQrZEGBEQxCTDzjvFL4xotAEd0wF+Po6CbAGAqSp4KT8bvWf/Lwr/+HcAQIT36Ctn/VD2H8TCiCBlyVYBlNh2RhPFyhaDZjhkK6aoVeVsUVBikpW+s3OoD/tTw4A6BMUGFmazIZQtZ0UFn3Y9PPrgKFsG8rbTdPqDh71bPE+YrBcj8d9XqSxXtjTg9YfodiYegdeGi9MtCq4wIsu35Tc7pX5bJ1e2xgfZ2rhxY83XWbWy9cMf/hDnnHMOpk0rnYxaK1x33XUYGBgQf+vXr6/7NjmaRdmSIRLkWc6Wyl7HWXPTanO2BlIs5m7pZZsuu0Orow0j8lwB1XG9D6NsAcC8M2ieyfQlNASZHQLe/ItrEubzQ1yld7XDudEpUjyUeN/zO1yyegv1gVUiZ4vvey0XVNmqdSVtMES4sgUpjFiU7M8vnuXJlqo4ZCCIqSm3fsizXB8ASDLiofopW2bCfcMzijAiwP2aCAbmnot+lowe4Qaj5UxNy/x7SEBliyeERzWzpLJlSk2hY4oKyI2oQxwjvFXOiBSC3TW8H3967WkAwPSciUhA2wfA/fvzRWQrJwoqTClvr7fwqvjOFUc5XlAJM8LWaUn5eg7SBU6+NeA1moudnH0G9gmlzN/UlOfgxbKsX2CilLLFyNY4UbZmz55d9q8aVEW2Nm/ejIcffhgf/ehHxXs9PT3IZrPo7+93Ldvb24uenh6xjLc6kb/my/jBNE20t7eLv0RidHdaYdCU1Yg8QZ4pW3xY/O4lU6WyNZChZMuuRLZqSEB5GJHYmnPXE5ZscSgKMP88+nzD7wBbVrboYxsrax+Re+ZVgeULpqDN1LC9P4XVW6rP/2ohGEQ1ouIlKPRRywVsmNyoakTmMJ6Xw4ieZSxCL3gmD2OVIFDEV9kqnbPFla0CIMKIySy9GVOJW74msGnOjUy2RlGNSLdPLzPZvIUcIyKmIFtSfl2IYsSgOVu2aEETKals8eRxAGgjxGXsGuYYISxQlJSO0d0j/Vi9aw0A4KgCI7YBlS1VWg9P6yDEomkitiXCiKaU43fq5PcBAD6+fwCaRELbTYdQDqbdJqWAE2Eg0IQPXKHnaOxX1aJl5XtL0f0jyyIFXtsHjs5Z9HGcKFsA8JOf/AQnnXQSpk2bhs2b6T751re+hfvvv7+q9VVFtu68805MnjxZNGcEgCVLlkDXdTzyiOOY+8orr2DLli1YtmwZAGDZsmVYt24ddu92zN4eeughtLe3Y+HChVX9gHqjUTkeXsjaCb/LL9junK02g/YJTBOmbNnhFJehDJOgbaOCsiU9H+VOScu5AgNVKlsyFpxPH1/5I5DPOGSLK1usl+JwvnjCCYOIruLsI+jE8kCrKrHusEqEEbnKG1zZ8n9eb0RcYUR6bhXlbDGSZfAwVglli1ulKLbt6FJlrB+cakQiTEVHmPJt+JEtI0HDiByjDCOKisS8hawgW4z4lAkjlvsHBVW2eNgsVkbZimpeshVoCEVQ2JiSUlL5vmQ/tiY3AACOzzD1pz1YNMiVsyUnyLOTgDvSy0roLWddg/u39+Ka/gFB6gGg3XSsJgbSxVGPNNs3CtGFR6MSiTsJ8lI1YsGl5LP9y9MySql2fE4fJ2Tr9ttvx4oVK/DOd74T/f39wsS0s7MT3/rWt6paZ2iyZVkW7rzzTnz4wx+GJvWo6ujowFVXXYUVK1bgsccew+rVq3HFFVdg2bJlOOGEEwAAZ555JhYuXIjLLrsMa9euxYMPPogvfOELuOaaa2CaTerG7boTbiDdklskeK0f2AWnnXVYTykKVXIK4So8h7KOstWfKv1dt6np6OBUwWhOpcpoyNbsk4BoF5DcB2x6QjI1pWjT6V36SA18yN52KK1iWrO1f9TraqE8OFn2hhFtQbaYshUp74HTMJ8tXWpELXy2PL+FEYMIN+ospWwRZp8g30wFSJAHHC+vJDv+DRQrF8VhxNGRLZ05kecKFnLskmMWilWmYq5V+h9EhPVD+ZwtnsDdZkZLKltRwyFbCQKPshUijAgeRnS+s2OoF2mFEoxjh3qpG/+sEwKtT76xyElhxBwhyMMxKJULKiK6gYN4cDnvkC1qTcGKg7J+ZIseFwo0YeGgGlEMel2E4c7ZstmxGOXV3aXMqHkYMbUfyI74L9NE+M53voM77rgD//Ef/wFVOn+WLl2KdevWVbXO0GTr4YcfxpYtW3DllVcWffbNb34T5513Hi666CKceuqp6OnpwW9+8xvxuaqqeOCBB6CqKpYtW4YPfehDuPzyy3HjjTdWNfixgDvHo3GwfapBCuwOmDsWt0ccuT/D1a0QGM6w5a3yypaM0RLQTJ7nCqjACFM820dBtlQNOOyd9PlLv4XC7vr4RTnOLhzDheK8hbA4cjq9sG/YNYRcoXT1Zgujh2P94F+NqGabO2dLNCrmPlu27aNssZytCspWIkK/6NKkAoQRAcdUNFlO2SoKI45W2WKJ43lLVCM6ipxUjei1fgiibAElyRY1TGbKi15a2WrTZbJlC9/CSmPwghuKZiRl64V9z4KQAmIFDdPzBeDQMwMZmtJtF4cRucKUIUQcL6a3oII/5t1zHGHGskOZ4utCht2Yq0QT+0k1YxhijI+UyNkCL+pg5rElla1Iu3MjxI2rmxgbN270rTo0TRMjI9WRxdDViGeeeWZRryCOSCSC2267DbfddlvJ78+ePRt//OMfw262YWjUnXA5eHO2+KndHo2LZdKEIBJSvRlmdzS2HSJna5T7hDfE1fmFT4uWbFMSGAvOB9b8DHjzMRCcBsBRRtpMesKPWKMnW7O6Y0iYGoYyebzWO4yF0yrkC7VQNUqGETnZygfL2WrU+RxlLVDyBABsakvi2T5P5jYrKFtzunrwhTlXYeLTX3Xe1PyJGeB4fAEO2UqzC7FJSlUjSpeGgOSgFHgYMZu3KNmE7HwvJ8i7Ue7fw5WtghSW9UI2TE6YZXK2ZGULBdeNbSiy5XM5HQT1ZDo6w375gguCrxD0eLdsOYxIkVIIbJGz5bEK4WSy6IZSB5DyJVv8eFCJLm7SdTOOYdVR1Di4smVZFsCUrUiSk60ybdY6ZgLpAafKtYkxd+5crFmzpigZfuXKlViwYEFV62z1RqwAd/yevvrpmsdw3j2fxo7B6nokVQOZ3zrWD+xCwx4NzRQuxukqlK2RbDBlq5YJ8rxNhJhy26eO/ip40NvFhOMkyFPEI7QaarhCT7UgUBSCw6fTi/uLOwZGvb4WSqOUz5a4ZgdVthpUXSyULT6AfManPQ27eLIk9lJ5WHE9jvfPOwVnJKXzu4yypbnCiIxsFeh3TcVTjahodB/KSdyj7EZhyGHEImWrjPVDmXmAELoG2ajVC5lUJMxoSWUrLitbyLvDiCGOEoUUky2bfX1peoBu95BwrWr48S7ypJg6tVtVRc6WKKjg+1LzV7YU9t0Rn/9nlhFWhWji/62ZUYwoxcIKH0u2kKcmuABM0SqrAtkCnNzcJsaKFStwzTXX4Be/+AVs28YzzzyDL3/5y7juuuvw+c9/vqp1Vu2z9VaBny/P19Z+CgDw2T+345731i8Euje1F5fe92moKmDjA+J9TrKcMCKDooLYBoAUUkr4isQk64VmWzoGksGsH0ZLjDIsV0Dj6zTiZZYOCD0CHHYO8NJvxVtcjW1jZGvE1ZKjehwxrQNPvdmHF7cP4H1LZ9ZknS0Uo5TPFj8XFE62KvlsNagc0dQcB3kbAClki5UtQjseGLwRdRkn/KIQY4mQI+BRtthxn2GqR8RLtk779+LtBm0KXwJC2So4ypYmWLJsahomjCgpWyXIlqu5sqoJ2wsv2UpIyeMJOw/ZFCbMIeKt7JSxMJsFDj4jdGUnJVu2sH7QQMe6Q9NEPlqRCS4n6V6yxXz7R8qEEXWiC0VMNWJIqs55x2dMTrac4ib2/1Q0IFomKsG9toaaP4z40Y9+FNFoFF/4wheQTCbxwQ9+ENOnT8e3v/1tXHLJJVWts0W2KqBcztb+TH2Vrb1DGezIrmWvLhEjEGFETrb4xEVUEBiwkapK2UrxxrNhwoihtlCMdIGHEZnIWqaqKhSW3wB0HwSinQv8aZejbLH2PsM+/cuqwZEzaFjyxe0tZaueKJWzZXnJVpgwYu2GVxE8jAhQNUb3I1vwkK0ySe+QnMKhRcqyAk3KIbJsC7AsZJipaMybJzXvdOfFh38P7FgDHHw6RgNu/ZDJW+KCrfmFEb3KVpl1KgFytoaZrYxtaVDy0lxoxFzL8SbNAJAoZD3KVnD4KVsc8zNZYGG4ECLg7BM+50fUOHIAtmua095J5PjxMKI/2ZpsDWIHgNjmB4Bjz3J9lhVkyzlWiBFDSuW9LIloYc3HIrda022b3uj4tPcR4MrWOAgjplIpvPvd78all16KZDKJF198EX//+98xY0b1+cStMGIFeHM8RjLOATxB6qlVD2Rz/pOtN0FeJHQKZau6/oi8/NeuFEaUn4/yisXla6FslbvAhEHXHOCM60FYOETkbMXo/2ykRlfaw6dRsrV+56CnJLqFWoLvWtXrs8VzgwOSLRljmbMV1x1SwSsSi0NUrJS/UGz4WQRZfapwg6IoiohncSUoy25yokU5W9KY5p4KnPSpUe8onZEtGkZkQxZjL/0byypbRKpGLJGzlWQWBgQawNtzqWZRDlpCztmyclVXI2olyNbkfB7dUIFDzw68Lg5+vOfZgc6rqbdrmjAUdXL8eBiRHQ+enK0ei4oDXZt+Cy/4PGxIlKCgGDhSmYjF6Qwi0nHC1eSURLY0oPzxCkj2D82vbL3rXe/Cj3/8YwC0NeEFF1yAb3zjG7jwwgtx++23V7XOFtmqAG/O1tpdjo1/u1HfhOhMlsC2+b+ouBrREj5bnGxpUCCRrSqVLdsy0F8ujOiyfhjdRMwTM0WCfKUTNiT46IT1Q5xWy6QVghzvUj8KHDQxjoSpIZ2zWhYQdUSpBHmheAUNI9bw2A2DqItsASjkipUtVjln5AMoWzLZCnKDYstKUA45ViAS9YYR67BL5AR5rkHpfmHEom2XHgxXtsrlbHFlC7ZGE7MBX2sQ2fAzXkhV7bOlEv+w74JsDjjobbTnZEjwnC2eIN/Jxr9Dd8KIJv/9ImeLkUfPzbbJ9vmwXpzXyJUtgzgRBlXT8eN5F+AnO3shz8q8GlGQLZsFdSvN3dzYdKj5c7aee+45nHLKKQCAX/3qV5gyZQo2b96MH//4x7j11lurWmeLbFWAPDkrCvDibods5WuQZF0Ow5kCYPGJ1CFbeRFGpGRL5DASBSrr+p5WlNDKFg8thAkjei9+YZFlycAijFgrZYuBj5Xn/MTiTmlycmS331dCQVEI3j6frvPBl0r392xhdCjlsyVytjLhTU1He+yGgWy/QO0ffKphCTc15TlbIcKIFcFznAAUcsgxj6Y24lZj60K2JAd5Hkb0U7a8/9ty/x+FSM21S+ZsSd0pUv30TR/C0xGJIVJQELUsdOdTLmUrDNsqlbN1WCYHHP/x4CuSwHcJ99maFOsEAGzXVNGX0LA9ZEtYP7i9Ek2Lk63ivLEcc/Q3+QZ50QDruKH6XH9482regLtsjiHghBGH95RfrgRuu+02zJkzB5FIBMcffzyeeeaZQN/7+c9/DkIILrzwwsDbSiaTolPNn//8Z7znPe+Boig44YQThJt8WLTIVgW4czwIXuvbIl7nyrSJqAUG0znYBTbh+jj48mRXOUFeI3T5apStLLvbJZaG2an1wJanhJuwjFqGEXmiriGUrQonbEh4x6frMUT4pFMDsgUA5zAn+T+9uLOkLUoLo4NfzpZt24yE2Y6y1aTWD9RUkqkxrBl1sa+Ux6y0jJ2DS0Uot5wzAgCsvY1VQJ6d6zHi7Y1Y+53Cla1cwUKBXbR9c7Y83ysXwguSs5US5rF6WWXL0DTcGl+KH+7cjbb0/qpztjTFf+4aPvrrwCHvCLEmB041Ir2x7mmjCeg7NE0YihoFj1WIn7KV6neULbXYyiPHCL7B9zkjWZx0yUdJQShbdP8qPPpSSdmKT2I3EOE9CX/xi19gxYoV+NKXvoTnnnsOixYtwllnneXqRuOHTZs24bOf/axQqYJi3rx5uO+++7B161Y8+OCDOPNMWkW6e/dutLdXF9Fqka0KcDddBrYObhOv+QFaLwyl87AtftfqHKCFgjtnS4yQqNBlshVS2coyZesd5AXcQ/4D+NFZwK+vKlqulqEYJ1eAoVYJ8gx8fDIHirPnw8nq7rC8eNthk2BqCrb2pfDSjsGarLMFN/gEL6sdPI/LRA6En4th2vWMuXEeVT5ozlbWx1eKWbgIIlIujCipKEGULVtStqwc8kzZiitjoGxJYcQCv5bzzboS5D3ks8w6hfVDuZwtVo2oVCBbALCsewaOzGaBkb1VHyOlcraWzTw88Dq84Mc7DyNO76AFPklFQV6laRBFSign33IHkf7Ngmz5XRXybB3iSOJ5bULZkq4/HmVLEcVNFertFMXdBioEvvGNb+Dqq6/GFVdcgYULF+J73/seYrEYfvSjH5X8TqFQwKWXXor//M//xEEHHRRqe9dffz0++9nPYs6cOTj++ONFy8E///nPvmanQdAiWxUgn2vret/ExuGXxOuxIVvFYURhKsdztkSMRYWu0OVTVShbOZtOTjPsfufNXcWtCUjJF+HB5Wsx5dYrjCjtvzY26BFuxDdKxAwNpx1GE+9/9MTGCku3UA38wohc7UqAH+eEGnKWQSNbbvFwC83Zyvj0AqS/x8/wswhKyJwtl7KVR4Gd63FvBWAddg9PkM8WbCiWgUTBkghlOWWr9DpVwn8PSipbrn5/PEE+0um/Qk5YR/bWTNn6p/0D+HBuOk476IgQa3HD67MVNwxMyLsJslO9WiZna/9mRNg+5yFkGSKMyN/QPcqWdLfKw4ii+wc3EwuSb9vh2OMMDQ1hcHBQ/GUyPqF10AT11atXY/ny5eI9RVGwfPlyrFq1quSmbrzxRkyePBlXXVUsGFTCe9/7XmzZsgXPPvssVq5cKd4/44wz8M1vfjP0+oAW2aoI+WQbKuxEUnlNvM6Xa4BaAwylcwBTthTi3FnkmcNjXlQjMiiaIFtp5rOVzefxyT/civvXP11xe3k2AbfbEolMFttb1LJ8nitbZp0S5Dlcyha7Kx7m/bxqgGvePg8A8Jvnt2PdtoGarbcFCr8EeX4BShBW6GAmypeeo7Yh8PBgZIu17CnePD2vNYB6FpX7LfJ5EkgNdudsWWDKljdnq45hxGzewqJd5+PJLduoigSUTZAvNxZus1AuZ4uHuVSUT5AH4OQo5d03qGGOEd1Dtj4wOITPTl8cfAU+IB6ypRBgssciUM97lC2/nK3+zWhnochBH7LF848j/PdywsZIl2o71x9L+GzR7aoIGEYEHK8tAAsXLkRHR4f4u/nmm32/snfvXhQKBUyZMsX1/pQpU7Brl3+e7BNPPIEf/vCHuOOOOyqPqQR6enpw9NFH02pehuOOOw7z58+van0tslUB5U62fJ2VLTlnq4s44SlhaurN2SIqTJWeJGmiALk0bnj0bjy+9w58+SmptUcJ8LvdDvlkzA4V99hyhVZHNznzfWjwu8maK1vFYcQ2NlGP1JBsHTWjExcungYA+PqDL9dsvS1QCLIlsS1h58GVrQC2D94czLEEV7byPIxYxC5swGZBmUoESq4iDHDOENutBHGy1ebxm6trgnyhgBHLk9dULoxYVtmi53CWoLSyJfr9GWUT5AE4OUoAqnWQ95Itw7ZH3VeS94HmapKiEPQU3JdtQzT1Lq9sTcrT60UfKd5ffB6O8t+uswpNtl80iWx5lS2HbAXIt5WUrfXr12NgYED8XXfddZW/HwBDQ0O47LLLcMcdd2DixIk1WWct0DI1rQD3BOCemPL22OVs2T7VIAV2AhARY1FgqHLOVgp/3nY/oAJZW/ZF9gcnW522R85N9tE2Ogw1VbaEfB0gdFIF+Pjk/RdXTMDOYDhTWwXqX888DPev3YG/vbYXW/uSmNkdq/ylFgKB52fJYUR+0+FStirCnYM5liDQYEMKIxYtYQtSVDHpnRAaSrRyAasR3dV7NqFzV8yTrFyPXSIS5PM2hgua+xa/yvM9riWAAjCkKCVzttJc2SKVc7YgtewhktoXStnykA0Doydb3jCiQghmWhoA5zc7vmzenC1pHt+/CZMKlGztV4oT1Au89Q//7Xx/+IQRebK+ULZ4GLFEgYALEtlKJBKBks0nTpwIVVXR29vrer+3txc9PcXdDd544w1s2rQJ559/vnjPYmPWNA2vvPIKDj744MpjrTFaytYoUP8wYl5YP8g0Ly8S5LnPFgAWGoswZSupEDyzbxsyKs0hslCZGBbY3W5XEdkqnds02gsWv6My+Q+sm/WD814bm5RGspUJaBjM7I7hxINpAuuvn9tWYekWwsCvXY+Ts8XIVgWPLaBxDvJ0e1KCvI/PFiVbvIw+wHkgfJUCKFuQlK1CDjah53piLK0fChaG8577ew9BCVot2m5Q0tSvKBWVLS0I2XIpW9XBq2zpNmpGtnIsdUQhwCEePy+Dk80iny1pHu/fLJGt4oppTrYcZcuTIC+1NxPKFt+/VYYRg8IwDCxZsgSPPPKIeM+yLDzyyCMicV3G/PnzsW7dOqxZs0b8XXDBBXj729+ONWvWYObMxrRVaylbFeDujWhjSde7gEIUqwd/PgbKVk4kyFs+uSoWuytVABFWmNMxC2uHgRdME3sHXheZ53xyLQe+TLdNwzIZLQEzP1REtlzViLUiW0LZqi3ZUnzCiO16HMjvQV8d2i1dvGQm/v76Pvz6uW341OmHuMJeLVQPrmzJxx7PHWnj9VUBLmyNzNlSoKIAlrOVz/iGEZWgyhZAq79yCEe2CIFVyIE3vW4rKsOvX85WOlfAsF06jMi3LmwDy4ylM9IBDAEDqloxQV5TDCDNcntKJchLypYi7ROv91c5yGRLs226xwPcAJSD6FtuOdYn89UIwG8wINlo8GPG2xsxn6VhRGYVsU8jsCzLlYtUsPMAAaL8t3usH3RpH/ObfR5GFN0/QoYRw2DFihX48Ic/jKVLl+K4447Dt771LYyMjOCKK64AAFx++eWYPn06br75ZkQiERxxhLsoobOzEwCK3h9LtJStCvCe8LPbp6KbnbCFMVC2bKFsFcu4Bbk3okJ58xVHnwNiAxtMA3/T9orv8LBBOfBl4uwupldlEq2XbLmejzJny/aSrRr7bLFHef/NSNDy423J2vhsyTjr8B60mRq29qXw6Mu1X/9bFU6CvBRG5M152UUkSBijoTlbLM8oT1AiQV4iW3VUtoYzIyJUFvcYM9ezGnE4k0fGLt9AO+iN3ASWezVYRtniKQqjUbbC7A9D+i2iojRE+yg/OA7ylnh9sKe3o1BMvGFETra2/QMoZDCRKVs5QrBt0D2nF9hxwO0hvKamuqRs8fQVR9ni2w9wg9A+HdUQ+ve///245ZZbcP3112Px4sVYs2YNVq5cKZLmt2zZgp07m7sNUEvZqgD3yWYjbphIZ1gYD2MQRrTpQS8rW3lPb0QCiDDiwRN6MDXfhR36fmQVGyhEADUNVCBblmUBTNmK2jYsm+DNXDdmAT7Klv/zaiByBbyd62sEvzDirO7DgN1/x+Z8bcOIABA1VHzw+Fn4/l/fxPX3v4jjD+pGIlJbAvlWRDmfLaFEeFrP+MHrmzeWUFxhxOJG1IAdrrKLk8sAOVtEclwfTo+I99s8RKWeOVsjmTzS8PwurVjZEs/LDGYic1IfKJOzxZUXXQ2SIO/MO+7eiKXH4IXcJcAhW6MNI9JH+fjXFR3dhQL6VPo/FYqJN4zIc7befJwOZc7b0F54HYOqilf3bsOsTqe3L08ziXLyzRPkGekypQR5m6SRL1jFfW2DHLOaQc1NEX7uvfbaa3Httdf6fvb444+X/e5dd90Venu1RkvZqgDieR43DBhMfSnUuV3PoBxGlN7Pe8KIKuAqEz8u4hi4HdZGvUkIsZDMZUSioBdD2bS4241YNoYRxY5snH5YhmyFxY2P/RSPvvGC81vEHRXvKFxr6wcWRpTemTXlGADAVhRgl9gfo8Gnlx+Cmd1R7BhI47///GrN1/9WRDmfLZWIGGPF9TTQZku0mCndiFoiW0HDiECgc0ZWtobSw3RrtoJIkbJVvzDicCaPDCqEEQMqjz1tNDdyQFUAq+C7DO+IoRMjQIK87KpeXTWiKf0Wg6+iRjlbeSlBHqqO2TkPwVR05xqgepQtRrYw/1xMZurW5r1bXF/nXSsjXMHS3MqWIu0TTRlBJu+QLaevbUDt5kO/CbbcAYYW2aoAr7LVZpow2URo1TFny7JsDGfkakQHXFLmpqaKbQtlCwAumn4Mex/43AlXivd/8I+VWHTXSfjy4/cUbW9AutuN2DbSShx9YBNFURixOuuH+9Y/hXu3fA2fefxfBOkrCOfieitbzh6cPnUpVNtGSiHYs2d9TbcHUJPTL194JADg/57Zgr6RyvlyLZSHn/UDf08TZKvydOZWZceWeamMaOSAogR5Ajo2raowYgBlS7S3IRjOsHwfW4dS8Nq61B6GStc6nM4jCw2WLW2lKGcrmPI4LUHb1gwoCq3I9AFvp2YqGpBj81slU1OMJozoEEmjVsqW4qlGVAigaJid89zoy/OmnCCfHgC2r6avDz1bhBK39bsLeCzusyWULZ4gb8JLOTV12E22fLoBlEV8QrDlDjC0yFYFeO9s2gwTJjuprDoqWyPZPGwbUoK8j/WDXI2oOHcVi6fMwTd69+BrqXYcO+Ng2Gxye3DTo4A6jMe3/rVoe/tTdDIiFg122GYb9tpx7FGV8mHEEL/pjT7a7d3S9uL3L/+D/gbwMCK7o6p5ux4KmazqZhxTWVx2y85na7o9jlMOmYgjprcjk7dwz9Ob67KNtxLKmZqKJrkkZBixdsMLBMXVYibjIX6MOPJRBVG2woQRJVPTkSwlW8TWixpi17Ndz1AmD4A4oUSiFId+A84t09upf1JSUYTFgxdc2YrKFZelcqj0Uj5bweHK2eLrGHXOFn3My2F0P7Ll1ysznwFeewiwC0D3wUDXbExk0+yeEbcZqM3DiJaHbBEC6FHXPlHVEWTyBR9lqz6G1AcKWmSrArzViDHDFHKxVcecraE0W3ehuBqRV4NYjKAoNtyTlh7FO5IpnE0UWnFiUyI2nOsHACQLA0Xb48qWykrPn2iP4rfznsQXJ04oW40YplpnMDMsnv/fSw+w3+CVr2vss+VTjQgAs1Wak7Bl74s13Z683Y+eTMO5d6/ajEzeP9TRQjD4+WyJji8hlC356jnWhaLciNM3jMg5lrhwBVG22A1WgHNGkdrbjOQ42TKAfLbq6rugMFhu0UiGnutZwqvmisetuAho6bFMTXSKO6jdaf/8H17pHOPHh5EoHepyKVvSZBFif0Q0R9mqXc4WKX6taHjv0DDMgo5p1qH0Az9lKzsMPM5c2Y+4CADQzS4kfSl3X1ibeJQtqWAAWsS1TxQ1hWzeEu3qxK+ucXHTgYYW2aoAr6mppmgw2UlloX4XUE62VEtj2/KpRuRhRLjDiF4HYcLKrVMF6kKfsYonp4E0nYA1RrbmRjpgKRbWRkzkRva6lg0js1uWhWV3vg8n3/VB9Gec7W4Y/DssyxKE1bDGTtkCgJkRKmVvHthY0+3JeOeRU9HTHsGeoQx+v7a5K2WaHX4+W041Yogwovx8rMOIzB+JmprmfFUcR9mqUxiRECRZz1QFBjNXLfZdqiV0FkbkhDnLL88+ZCtoGFFTVUQs+pv28bCoB7wHYIzng5ZKjgc8yhZ8n1eCqck5WzagxwMVbZSD9xhVCABVR6dl4W+Hfhh/OP1z9AOZ6PA5dOcaYN/rQGwicOInAQDd7MZ7IOe+gbbZtczkBqlyDpsede0HW025wojiVwcxNX0Lo0W2KsB9qNvQFd1RtuoYRhxK0wO5xyw2Nc2xWcsWZAvuPmr8RMkxssVOh6xNyVbOdhQmjmEWWjDYIXFkfDIUS8OwouDVVPVhxNf27cKwsgEDZB16RxwHYEvbg4feWOsoW1y+rrmyxZ54pK3Z7bMAAFuTvagXDE3Bh0+cAwD4wd/edOWNtRAOvj5bIkGeXUwD5Ww1rhpR49YPID4O8vS3RL1JzuUgwoiViRlrAoQ8gBS7CVOJwRS26qrvgoKHETmyhPfwK744h5lboqz1z77ciO/nPB80zq1BSiXHAyWVrTD7w3RZP2DUqhZQrL4SpmwBQJQAGr8GyTep3uPhtH8Xfl8TGdEdsTzRDa5s8fw3vbSyZatpZHKyslWf7h8HGlpkqwK8J5umaIjo9KCy6xhGHGRka2YbmwTknC3uIM/IlmqjhLLF7mCZt02BUJJlkeLJaTBD3zNZfpceaUcXZgMAXkDKRVbCKFs7hxzj0H1pt+/UT1/4vSg5Ni2fSaMGEAnynvdndR8GAHWxf5DxweNmIaqreHnXEP722t7KX2jBF3K7Eg6HbLE3Alk/+D8fC6iK1/pBComyx7jKHeSDVCNyhSiAzxabHyxCkGI3YSo42ZKXq181Ike5MGKYsUSZYr9f7gEoIS+ULU62OkuvrBTZKjsCNyJeZasmZMs/jAiAqqO84lArQbYmzAOWfES8nMg7jMB9w82vZRGubMlqqUfZKqgZpHN55JmPmai8bIURy6JFtirAT9niJ1U9yRYPI06MAFGPPYFwkOc5W3BMTQEUKVuKmNyY3K6mkfQklQ5nKTGL8Eohsx0HtdOKuhdMFcg5Ur17Eiw/He0e6RfPB/M0T4DkaSXRi/1PCPna4HdUNe+N6J+zVW/7B46OmI5LjqOuyV/+wwbRdqOFcPBv10MfNZ5zFNb6YczDiJxsgTaidn3KTEY5YQyi8B56NpCYBsw8ruKiiqhGBFIsKV4jZrGyVXmrocHb9XDklHLKVvAChhib1/pLkC1uzRNnpKussqVqYg4l0o1rGPJZRLZG6R4PFCtbPEEeADVz9bbqAdxk6x03uvZzF8vFyhNn/qcei2weLrB9KStbiupWtoiFvnQ/8iz1w2gpW4HQIlsVQDy6totskfrlbA0ystVlFNBmWZC1mbxlw7Ztx/oBcN/Vy8qWbUPxetsA2NrvDg2OsDyOiKiiSeCEmbTv1POm6UqSD3O92jOyXzxPWXQdR3S8DbatIK/thKX20+3WKYzIB2t7tK3pU5dCqaP9g4x/OeMQdMV0vNI7hB+valUmVgPfRtRC7aoyZ6tWgwsITeE5W8XViJxMxrlMF0ThPfFaYMV6YELlprqE7Zs8Ie42NrkkMMZhxJwIIxb/xjBzSxz0+wOWfzVigStb3n5/peBpUeMdTyXwiAfAEuRroGwV52wRINpFXwzvdqpJZYI18TCg5yjgyPcBh73TPUb2Gy3p2pUu5ByPRa6UyfsqMwx3bMBGb3KnCCO2lK1gaJGtCvD6bGmKJt3B1D9nq0O3ELe8QTDa1FWYmnrDiPGJlHBZeWDPy/QO1oPtg+6Q1ghTtmKSGd8FC5aB2Da26xpe275BLBsmr2JfyskNsFT6fFrbNHRgIVsX/Q2OslWnBHnPLtTNOKbV2f6BozNm4PNnzwcAfPvhVzGcqa8Z7oGIcj5bTjVigDBiA3O2eO88mrOVcycds3O5jatAQf3mAv4IlZEtC0CahX90xQRyyZq23/KD7lG28krpMKK7aKD8WNoUSooGLX8fO97hIxK0dx8jWUQOKYbYHVHpRlEHgPYZwb9cAqpH2lIUAnTPpS/2b6R9DwH3vtQjwMf/Blx0R9EPMBnZKpCC8DocyThkVWfREFcYMTtcpMLuSu4UrdYMriy3yFZZtMhWBXjPNV3REWUToUYywK51ddnuln00bDc5aqPNditbAJAr2OJkoWFE6V+pmcCck+nz1x6CRoontR1DXmWLbk9U7pgJ9CS6MDlP98DLu14Wy4YxNe1PO2SLsLupdrMNJ0893bWcySeNMbJ+AOpv/yDj/Utn4qBJcQym8/j5M1sqf6EFFxxlS3qPz/EIkSDvej62bEtzhRG9jajpD4zx92oeTufViECG59oQg5GtsVW28owk+YYR5ecVxtKuUuIwWMJcmptOR/nmlRK2DxyMiBBdbt0TfIdEpe9tixwBLP9S4O+Wgm8YsYuRrb6N/spWGUR4X0UCpJgLfSrvkC2Th2R1qf9iZqioYnXX8F7kWTSiFUYMhhbZqgQf64e4wQ5sUgC+d0pdNrtmaz8A4LAJur+ylXeULRpG9Ewk82ibHrz+MHSl+CToHe5zvU7mKdniTai5BK4zR+tMSlreR9m6f/3TWPnqc0XbGcgUJ6B3mgl8bOn5wmwVAAyemDlG1g/A2Ng/cCgKwcdOob5bP3xiI7L5Vu5WGDg5Wz7KFoKHEV3H7lhXI7rCiJ6cLabOiUbANe6koHJDVRDRoDnCxlP3nC0P2SqInC2fBPkQ/5QOjRKC4RIRBl58I5StSmSLK1tSGDHMDjE1Z/2DxgygbXLwL5eAbxiRK1sDW4Esz8MNRnQiRpt4PsS+O5KlZMu2CXRBtqR94CHkIDb2JPeIak/hlt8iW2XRIlsV4Ktssdh8npCiXKBaYCSTx6u9lKTM69bQXSi4qhEBRrZkU1NvCGXeO+jjllUwSfEks3vETbZ2jewAAEzjflfM+VhlzXOzaSf3ynv3OZBO4j+e/md87omPI5t3T3xD2WKy1RVN4OAJPWizDxPvGTzRcwza9XCMhf2DjAuPno5JCRM7B9J47JXdlb/QggAnVvK1pyAIGJe9mttBnrdz4WTLPQAP2apxSEYR1YhA2uKKD53+3aSvppsFUJwgX1DLkS3/537o1Gnv1mHiT7aEKzoC7lONhxFlZSs4otL3tBp5ThVbP4AWRagmTRPpe5N+EJDoRE2JbGUosRLFUrZUCCUrWyjeD32ZfaKvrcFv0CuR2bc4WmSrAvxytqJSImQ9sm/WbR+AZQNTOyLo0gs42Nt0FECuYIkEeRV28YVmwsFA52ygkMVEZmYqoy/d736dpaabczhZYsqW6OeWkcOBcD3fNdQPomQBNYW9Sfe2hnPF2+6M0HWf2HOaeK9ed0dKmTDiWNk/cER0FWcunAIAeGZjX4WlW5DhlyBvVWNq2kBlS1d5zhaAfMZT1k9/g3Me1Pamw3GQJ8jZ3FmdVd9Jy9UjtFpMtoKGEcuPpYupNMPEXyXmruhRzlgqhhEjbDFH1QnjqC+HEf2iCdXA3/pBAbqoLQ/2sPSOgDepihGDyeLvw1lGtpiyRWxp/8g5W9OXuLoMADaGcvtRsFvKVhi0yFYFeFtq6IqOmHRS5eowY69lIcRFMzqBfBrzsjm4Kob0fcjkLZFUqwDFyhYhIpQ4J/VG0TYGMv2u10mLqjtzWaI8J1sKN0TNSmTLpQ4Ql41EX9Lt35IqFBuoTopT1ezqJe+CbalAIVK38IkTRixmW2Nl/yBj6RxaSfTs5v0VlmxBhuXrs0UflRBhxObI2SpOkBdhRH4c1vg8cPoyAhmmSMSEslXfnC1FIdAkicYOGEasNJROkypbqRJkixt1RvncEjSMaEhu8iH2R1yuRqyZsuVDtgCgm6YkYM8r9DEo0dFjonpwiDnvJ1nOFs/ro8tJ1Yjv+wlIzxHSSmwkC/0isuL0tW2RrXJoka0K8FO2YjlHCcnEJtZ8m2u39QMAFs/qBPJpHJx1T8yRqb/GzuFdjrJlwz+EcggNJU7OFoeshnIOeRpIJ0Wl4Bzev5CTLUInoHzeIU1em61kzvG56Uu5VaJ0odhAdVKc+t0smDwDX1hyK2486gapv1adrB98lK2xtH/gWDqbeoy9tH0AqWyrX2JQlG1EXaWyNdZxRN6o2M/6QShbIuu/tucBz9myQJBjZKuN7cwwBKdayBWJFieSo0yQj7KE9izxObkhhRH5egImyMtEN1Q1oky2ahQG9m5fHP88SX6Ipn8EJud6FBHbrWylczSHT2Gt2qBo7v9Nx3TARbaArD0gfMx0u1WNGAQtslUBRTlbqg5jzwYobPLP1qEf1Jot/QCYspVLY0Y+X1QN8mb/G4JsEb8wIgDMOQVQdEc1kjCSd8J7z+14g/qsWCYmcOLEyRarHCqUIFsEQCrnlF7vT7mVrKxdTLYmxh2zv0uOOhXvPmQRfaHoNb+1FqamPp/J9g+bdz5T0+2WwoyuKCYnTOQtW5DqFipDqFiyQiIS5MNUIzYyZ0tuRJ31jIX+BtFJoY7KVpYlP8R9zrV69YuUk+QtHqKqlLNV4T8UYwpUzodsWZYFotCbmUjQnC2RIC/3SQy+PwxVE0U/Ro3CiL7WD4CTJC8WDK9sJRnZ4tWIKqcDWrEfmTvCY8NSBlHg3T+EstUiW+XQIlsVUKRsEQ3ofVF0dc+VKDuuFrsH09gxkAYhwJEzOoB8Ghog7kb4OEZyOdEbschni8NsA2YvQ8SnmjGS2yyer9tFkyyjVhc9pYgqkS0q1eetlFjea/2QksKI/Wk32cp7yJZtq0gYnsa5okVEbS8wdHx8u/53vz0spLFvcGvNt+0/HiJCiatbocTAsHyqEUWCfJgwoitna2zpFu+dlwdYux7pQ28YsU7KVgFAjnVtaGPbD+ObVy1ksjUcnU6fdM70WVKeW8qvM85sDLI+yw1lHbU9ysl4pRtjlhQuVyOGOUQURaFJ5qidslUyjNjlIVshlC1+8827hvD5W+VhRJ91eY8RouSRtWgUw7RaYcQgaJGtCnBHHWzoigZsekKEvXJWbVPk126j4bxDJyfQZmoAK8WNymSL0F6GbuuHEpVY85b7KlsRu1cY4r3SR60PphF2RzN5gVifyryoCpYzecknnkLgytni5cQcFkm5XhPLpJOSDD9jvhrBydnyR4KRrcH02BGfJSyUuOqNfRWWbIFDpN1Ix56TNB+iGtFz7I4luBmyXxiRO3jrhfooW3IY8bz+HiQ3X42lkWkA3BeBMAnhYSAnyW+cciZw5Z+B064rWk5x7ZPy62xjZCunQHgOcoy4yFbA42PRJcCsZSAzTyi/XFnQbdRK2fL12QKKuwYELaiQyFaSXVvSrBLcUbYiRV+Th8ET4vPKfvaa97VtKVvl0CJbFZDzqELaIzcBr64Uylbeqq2ytWYrPYAXzWR9vFgbHXcrUBuDmWF3NWIp9+wj3wfbmCRequznDKgKkKLb2jZEVZ15nNBNXSyWVzQa8ivYGXHFc+d4uJWtwYyjbOULBdiKh2zZxSdyWGO+MBBjLcG2EmxiGZKqLeuN0+dT/50n39iL3YP+fd1acMNP2eJJ86F8tkIoJ7XGMZOWIfHGFfjanr3sBkMeC1e26tMjVFV4ux5gUkZFIXkwpuvMZ2sM8thkZUvXdWDW8b7tc8KEEdsijj3BYMY9z3BbAyBEmOug04ArV4J0OYpbWPWTMGXLrJE5s6/PFkCVLWZ9ASC4GbQeE2SLz9tp9qgJslVe2WrjSpZC97HeUrYCoUW2KkA2nySEgDz5Hfo+Iw3ca6RWWLuVXvQXz6ShJt7VPe4iUzaGsiOiiTMplSAPAO1T8eaRnxcvJ7PfM6AosEZoy569GZpkuYBbIExbLJZXdUq2sgQAc4P3JrGmCk7Olt77lBjz3uSguIiI9cGnP1k9la3yXAvtbMIazBZbVNQLcyfGccysTlg2cP+aHWO23fEMP58tp11PiJytEBfzWmNSWwes7ERMLFhFYUTetsrkylaNz4XZ8SOwoG8ajk1nYOWZ9QNYFVoINalayMqW1wpCBglBhhOmQzYGM25FnavttqVB4TeRAX2gRpfXR7dh1knZEvtEUdxJ64Fzthxli/fDTbP5tyzZkvZEwnIX9og8wzrkLx9IaJGtCpDJFr2rsAGzA2mwXCbbAqzaVJVZlu3YPnBlK09PiC6PTDySG5EShFFWIo9Jd5CTmGRcIATpIVqlOFKgj4eNbKcLScqWoVMvmxQhohm1N36fyTtka9q2XwEv3QcA2D5Y7CWlkzFWtthjqZytdvb7hnLFifz1xHuOoX3Tfv6PLRhI1lYdPRBR4NdLv0bUQtkKYmoqPR9jZas9oiHLExCsHIjrmGTViHUKIx7VvQxH7T0YpydTsAvc7JOX/I9BNaLmrNnUypCtEPljptkGje1DWckCgGGmdBFbA7hhclCyNQryyb2qaqVslczZAoCeI53nIcgWDwOm2U1xlt0s65wOVChcaPOEbHWrFUYMghbZqgA32WJPVB0Ebs+cWuChN1aj0HMrYlPvx2FTWMd4Vh3orpCxMZKTc7bKhBHh5DYAQEI6UdLD1FvLIvSk60730/VId0zcFTlDCJDsY9uXQJyTFgCSRBHlyL3D/fTNQpuo0tEVtzMxAKGE1drIEaisbCWYU/5QPlliifrg/KOmoc3U8MaeEZz5rb/gL6/uGdPtjzf4t+uhj471Q+Ur41gTLBltpo4snAu+uwUKV7bqE0bsjOkosOleBfdHYnOLqzl3/XO2vO17ZLjJcIWxxLoFcRjyKFsjwo5Gc26GA5KB0URVuVfVmJOtUNYPnGxRksVvlnX+ayuEERMesmVY9VFjDzS0yFYFZIqULQCqAcKd1QmcarpRYu3ON6HGtiCe6IXGJyeWxEg8J0Aqn3RXI5ZRttqk6r8O6UTJjFCliocjNRvApPmuXAruZZMhBLv7NmNfcqgoZ0tWtlIKAbJUJdo9vJ+NLw5i0TGYik8YUVQj1uNkpWMtIWyh3aTJ6oOFsc2d6ojp+NlHj8dBE+PoHczgwz96Bt946NUxHcN4gp/PljVa64cxZl6JiIacTLZsWRGnv8HI16cytyOqi21rKEBVCPQCy3Ma42rEsmQrjOdXfJIgDsMj/a6PeBiR2DpghVO23P0zw+0RhYcRa0Q8SvpsAR5lK+DxkpgmCGqOGVWnvcqW77HnbLjdE8kx6nSDcKChRbYqwKVsCeZvgJDaK1s7Rqi6EVe7nDc52VKliYLYyOaHhLJF6AIl19smOSInLEt4hGVGuNkpPXl02EXmdTFWBj2kKDj32a/i9P97F+BxnJbJVpIoojnqvhQ9mTUSg2JTRSuillO26pmz5c+2ElFKtoZqXOgQBItmduIPnzoFHzlxDgDgu4++hn3DmfJfeovCz2dLEDBR2h+uGnGsRa6YoSIn9SnNF+R8z/oqWx1RR9nSUEAiooGwPnhuZaummxXQXTlblf9PgcZiJgRxSA7tcn3EjZYJ5DBiUGWr+rBqTKXN7ed1zwj5TX+U9NkCgMkLneepgO2/2iYBLJVDS24D4IQRxRHnQ9zKKVt6UB+ztzhaZKsCMgWHxTthRMMTRqyNsrUnSclWu9HtvJnjd5/uf9WhQ4+hK0/DgDRnq/RdW8J0CE7MskUlZTZJlSebcGXLBqSu8IBDtnboGtJaBpa2B3npZCNw7owAIKkQIEsrEvel+gEAptImEuOjmlRBw1EYA+uHUspWlHYAGKpxoUNQRA0VN1xwOI6Y3g7LBh58aWyaYo83+PpsFeVshWzXM8ZsixCCuGkgxyrWrN2OksnVZbNOxSIdUR15FuLiZIvPLd72W/WAGVjZkp5XGgsh0G26roynmTw3Wlag04bNQCAyXjSGkLvjrvO+gX9bdCvOX3BsuC+WQNkwolzNGUIJzbGbeT1N0z2yFidbZcKI0vPDsxmYeec6wa8nLbJVHi2yVQG+ypZqQOXKFuAkeI8S+9P07qTLnOC8yftWKe5qxKRCEGWmcopt0+qUEkiYUljQtoRHWF7cDTFly0bRCRNnqthe6W60IMnIhBBkCsw5Ig4AAG/nSURBVHLOlhNG7E/TCr+IGkcHuyB2+k1edTU1LR9GTLTRxtCDJVp+jBXeeeRUAMAf1+1s6DiaFX4+W2KOD1WNWH9iUQ7tER0joDcw0Z1Pi/dtkSDPyVZtL1ztUR052wkjtkd0gBWFjIWyFTyMKL+ovF5NkC23Z12KKVsqkchWFTlbYbWtQyZOxYcWvz3Ud8qOpVwYEQA+9Gvg2KuBRR8IvM4cswLSWRs3oWyRMmRL2m7ctvD7JZ/DDO1t6MExmJ5vWT8EQYtsVUAm55cgb4jYfC3DiEN5qjRNjkv9Flk1IvFcSJKEgDCSpAJlw4jtJZStPG9GzS5WGuwihSyu85YYztmWc5EtICeF4JKKIshWio09okYxq0B/26zMNvfg9r4G7N5An9dV2SpRjRjvAQAMEoxZM2o/nMvI1qo396FvpDZK6YGEcsoWCVGNKKMRyfJtpoYbc5f5DIb7bNUn2ThhaigQdxiRh/vHIndNr4P1AwCojEBm0+4wGi/acStbQasR608+g6KssgUA85YD597i61lWCjmDznlanu4zPn+XDSN6QqtT7ST+dOl38dBFtzgkImhO3FsULbJVAa4Eef5EVrZChhFHMhl88Fc34P/W/qXosxQjJNPaJjtvCmVL/lfZGFEUENbjjABlJfLOiBO6i9oy2RpANp8XXliaj7LVZhafxC5lC86dEcAsItgdc4YlneuKiQvtCA7PZPCu9inOijLDwA/OAFZ9l76ug7LFJ6eSPlsJ6qKdJwTpdH/Ntx8UsyfEcfi0dhQsGw++tKvyF95iEMRKutYURIJ8le16aja64EhENPzGOhWvL/hnJ9cMDmHU+YFa4wuXohDoOj2/NFJAIqIDY5izFbgaMeT/R2E6fT7j7gAh+v0RPXTOlqwejXWXAS9K+myNAvkIvbEjdgoY6kWO7R8x+/oUKrn2CSxgxxr6Qk4BaTQzbXK0yFYFZPIOsVBFGFGHQqqrRvyXld/CupFf4ytrri36LGvTsNusTomQ8LwKTxhxRCGimkm1y1s/dERkZcsSE3ohO4iUlNyu2XbRhCQn13PkPcpWtiArW04YMcPladXAuwH8fEcvFhvSRWTnWmGUSn9IPXsj+n8ejU2i+w/A4NA2/4XGCK1QYmk4YUQivedJkA9i/dBAB3kAaIvQ4z9dUDxkwoau6NINXe3zXyIGvYiKMGLWJ4xYJwqqSwSrrM+W/DzAP0ghdM4oZAdc7/OiHRpG5GGuKny2GkLJHVRUtqqAblC7mwwhwK4XhLIl2rr5tevxVoluX01f8Gtfy9C0IlpkqwJ8fbY0kzakBpBDuDDiK/vX+b5vWRYshZKtOTLZ4tWI8l07oVV/PIxIeyOWnkgMTYPNknJlZcvKDiKdd8auAUWTfIdZXD1YcJWsE+QsbzWiW9kylYhDqmRytXONe8V1sX6gKFWNSBQFCfbR0HBjFSUeSnzyjVYo0Qv/MCJ9FAnyIasRG6FtJSL0/Epbisdny3a7jtchpG6YTNkSCfJc2XLmlrFQtsqbmoarBORkK59zd4BIs7lHI0Z464cGE3IZJdv1jAK8R2eGECA9IOZvU1QVlj/2CGyg90UadRHVsy2yVQktslUCe1N7sal/u6vvnzuMyJUt4lgXBEDaGvZ9f8dgP4hCScy8CVOdDzjZ8kmQ5+Xi1Ger/L+S2HS8McsS5dJWPolM1ukpRpUt94SU8CNbZXO2iMgF4SdxRCtBtrgUzdEAZQsA2pnh6pCwwmgM5kyMY+HUVijRD4JsKT7vhfLZkp43KGcLAJJ5D9mCBUO+YNUh/yUaoeeXjgLapZwthCQ41SBwNaL8PMBgCPPts/LuDhCiBY0SPow4Fu2LgsKb3lZhmg+EiEqVqwwhQD4t+vs6YcTyCfLEaKOKVu+LDpFtJcdXRItslcCl930G599/Nl7pf1G8p/CKNVWHplSXs5WzJbIlhfBe72M98gpRV9hPOMh77toLhCDN/nuVHOQBAIxsRW0bCisBzxEgPexU8ahA0R1KewVlSyHE1Yw7RRRh/ZCzKAmNqEbDlC0eBihXa5hg+24w2ViyBQDnHkWJ9s//sbVkUv9bEZZPGHG0vRFroRKERYKFEVOWJ0AlK1uKXpervMmULRUFtEednC3Zu6le+0SvImcryFgU1pGiYLkd5HkKg64Y4RPkXc8PvDBiRJe6guRSKDDbG5PftFRKkO+eS59sf66utj0HGlpkqwS27Ke2CsMZJ4wo6IwrQR6hwogFIt2BfXOhIB+b9lMlQ7UTzueWJWwlvAnyADDI3qNhxPJk69D4qZiW1XFYNidaSuQIQZapOarNE+09ZCvikyBvuxPk5TBiSiGwWBgxZ9P321QV4N/hZCszRCsRZdTT1LQMb0mwi9xgUGPAOuL9x86EoSlYu7UfT29s/HiaATLpdJGtKqoRw4apao0EV7YKXmXLhsnPvTqFZKKNDCPKyla5asSwju0q9XsqWCnX+7xoR1Nk64fw1YgN5lp1IVtRlpOVJQByKeTZPB3hDbsrKFvoPog+bn9OCiO2KhEroUW2SoEZfcJ2dpETRjTpSYzwypatSJPCyB5gzysAgG2sKbSpdDqfS/5VsrLFT4phTrZsVLxr+/X7v4oHp74NUdt2GbLmWMse1ebJ/54wohEpkoUKsqkpAfK2QzbzhCBfyACFPPKMhLURyVKBk62dLwCwgZjkKZbxD7GOBs4kUZpttbM7uSFPRVMjMLHNxMVLqPv0tx5+FQOpVpNqS/rXyVVRBa52jRNTU8BJkB/JE8BDtgx+DteJbMUiDtnqMJx5a0x6I9YpjKjoNNm7YKVdd1RZpqobijlKZauxqOizVQViTNlKE4WGEZmyFUEZsiU/72Tu+PtebylbIXDAkK1svrYO4ISTLekwc5QtHTojW3kEJ1uWZUnrZRlXzIxv17CnVc/gTmDldc54JLIVZWQnqfDGsgHCiAAQ6WC/yKmkzCU52WLLeJQtRVFECJLDrWwRFDytbpKKAuRGUGB3THH4ka019HHmCc5nfW9U/g0hIcKI5ZQtlap3g5mB0guNIT526kFQFYKn3uzDSV99FDf/aQN2D41t78ZmgiX982QyYAvrh+BhRHffu5oMLxR4gnwyV1yNaHIyUKfKrmiUtWkhBXRoznxJapEIVAHVhBGDhPA42coRG8g4SfJZEUYcbc5Wc4URazEe3hWEKltJEUY0+E20D3Fy7ZMY63AyvEtStlpkqxIOCLJ12a9vxDE/OQEPvbamhmtlB55L2XKqNXRFDiMGI1tb+ve6XmcIEWRrb4p+1m50AUO9wF3nAqvvpAtGOlxSf8F2l+bSMGKAf+UU2vcwp1DfrSwI8mnWLJov43dXXY5seZQtgCbJP//6dhRA90tCIpjIDNJSbJ4cP22x81nXnMq/ISSc3oil0a7T/TGUHar59qvB7Alx3HH5Ehw2JYHhTB7/+5c3cfLXHsPNf9zwlszjKlhyGLH4fRKmGnEMWtOUA0+QH84D3mpEg/dNrNOFKx6h84aOAtpVNmcRVeyHevKKwGHEkJWAukbDiGlCgBFnfuWpDYZqStYP9e+NWGt4laxaKFu8KwjN2UqLm+IIn9crWT9EOumToV5J2WpVI1bCAUG21gzfC6Jk8Pm/fr52K/UJIwrzRM2AxvJ8woQRX97r9nFKEwIkaV5Of4Y+TjHiwE8upCpPxyzgtOuAD/zcNfGklG7XehQbwZStIy4C/mUt9ulzxNjzLJm9lLIFAATuyb/g6fpe8JItouBff/Z3WOwkbrc8qmNm0FG2ph0NfOJJ4IRrgNP+v8q/IST4bitHUtqZ78yQp6KpkTh9/hT86V9OwR2XL8XRszqRzVv437++ib+9trfylw8w2K4wopwgz94L47PV4BhRQgojuociKVt1yn+JM2VLRQHtGpuzjLi4kNZzd5iMYBmqUladCZsuFWEhr2wJsmWqpmT9UP/eiLVGPXK24gZXtgiQT8HiYURBtnyULfl5jEVfChmaCgO0fLYC4IAgWxx5rXZmkIpSHJqQrR90V85WsLyaN/q2u17LytZInoawTtr5ALB7PZCYCnz4fuC0fwdmn+g66TXiPhlU2MEmEkKArjlQpUrKPMsLE6eKz0Sv2O7tWR5ly4KbTKUIQQxpWLRzJNrg2T8D253k+KmLgSmHA2d/BYhPQK0RRNlKmDS8OphPlVlq7KEoBO9YOAW/+cSJokrx+S39jR1UA2CVSpDn1g92ldYPNRldOHCyNZglRQnyBr9hqpeyxchWNxnGhFd/Sd/UY2JH1DNkpmt03eVCiF4EGQ9P9k4TAkjKNFfbDTV8GNE1htDfqC3q4bMV16V9lkuJ+du0eSGBjwWPTEA1Q6SkYIAJCK0wYkUcEGTLtpyD45U9O2qyToV5Xtl+ypZqCE+cXIicrc0DbjKYUhyyxStCJiS30wP58vudqg+4pW2N+IURg98Na5L7fZ75eGlllC3FQ+4seHK2bDfZSioEMWRgERZGtDz7Z9PfANhA+3SgbVLgcVeHADlb0W4Q20ahRj0uaw1CCJbOpneTa7f1N3YwDYCLbMk+W1VVI8rPGxhGzHnIFrEcslUnlSDe1i6ex565lT7JpcTMUs+9YbBG9pXIVthqUZ5/lFGI6LYBQBTnRLQqEuSbKGerHj5bbbKylXOULdMqE0b0hnfbaH9F9G9lA20pW5VwQJAtIikv97zwUI3W6lONaMs5W+Hb9ewc7nW9zkhhRB6KM20b6DkKmHSY+8vSOW+o7pNBDRpGZOCVlFlCkGeGrJr4bT5kC+73LCksSAhgFeVsKYiRNMDumOKWJ7n7zb/Qx6mLA4+5WjjWD6XZ1lnTT8WaTVvxnZFG38eWxqKZnQCAF7b1v+XytqwSYcSCMDUNU43Y2HwcniA/lPNu34YplK36hBG7e2bjq7lL8GdIRSndc50w4hjkbJVzjwfCVyPGNMmgUyJbfD6NqDqErj0uc7Zqr2xxo+oMIbByKRGBiPB53S+M6C1cSLAuJwNb6GNL2aqIA8IcwyYZcVI8ueNJAB+uwTo52ZKrEWVlSzY1DaaI7Eu7TTPThADM26nADnjDtv2rQaTnpkfmJUHDiAxyq6E896PhH/rc/WnEhOyR35Xdiv2gDZzlMKJq2ygQgiQhiCMDm+RAALQVPGRr89/po5wcXyeInK0yy6hx1vg7ua/MUo3Fwqnt0BSCvcNZbO9PYUZXsdnsgYqSPlsiZ4udq2EbUTcwZytna6IBPB2MDYPf+9bpwjW5PYLD3vtFROImMNUCnrwVOPh0kN+xIdSRWuhqsDBiWFUpxpK9aRjRybnM25TNRuX5rIqcrUazrXqEERMmJag2IcjmkrDZ+WMIN/jy1g9oKVtVYdyTrXyhAKI4ytJArrfM0mHAQ2VSGNF2SmO5LB6mXc+ezBap7I8nyNMLPFeHdNuuWA1iKh5lCwhW9s6gq05yP8/ZEmFEn5PGmyMmGTmABkPo2DssC32qiqRCECUpEIWSsLa8291ZlGiPibJVOYwovL6SfdRIdgxK4cMioquYPzWBF7cP4oVtA28pslXKZ4uHEav22WrAldTUFOgqQc5S4fXZMvn465hs/O6jZzgvzvoyAIAQpjTXcXd0xugc0hEt/9vCcgmef+RVtvh8GlWlCbca64eGO8iXf10NEqZjVJ3KJWFrXNliZKtSux5AUrZaZCsomu+qEhIDGfeFPI8aNfAlxTlbiiRHm4KwIFAY8fkdG5FW3wRsYDLzBJMT5G2mDhk2KprKxXT3hVaxAUS7AvwoCl3K2SpYPEGeywTFJ42ueKoRpTtyQhwVsJ11Bk4RBSZxJr6YdMfpwhgoW3xyKht6474xdgFoEq8tPyya0QkAWLu1v6HjGGvI1g/EpWyFt35w9QFswHWUEIKoriIHzWn/BQCwYQpla2wvXML6oY7bWDSjAze963Dc9K4jgo0l4GBiItlbEY74gBMpiMkqYcD9KqtHtSA3o0E9fLbiuil4fjafAk/3MHn7OJ/rj3ufEEfZEu78rTBiJYx7srV3xO2NZNnBm0KXg+2Ts6XyC7ZmwuBkC8HCiN979tcAgGn5iZiVowdoSlGA1H7AsqSKkFLKlvO8w2x3fVbomAPMPzfQ7wLgcr/nSeHlcrZ04j753MqWQxRv3LsPf9+8FR8cHEKEOBNfNMec4XkFC8CS4ycHHnO1CNIbEZoJGKxNUrJ5W+TwvK1VbzZvuLMe4ETZe+ETOVu8OjaI9UOJ52MJU1eRh+rjs8VGNNZki7gf67MNgsuWzRHHcMWxBFxvmyl7RsnKFp2TonKGeTUO8g0nW/7PR7VORQGx6Y1JOpcWN8tGGYPSon2S6HEv0FK2KmLck639KQ/ZIrUhWxAtZmRlywkjRnnLAyVYNeI/9j4KAFgenUsJFdgEYVtAuh82kcOI5Q/2ibFO12f9y67zvRspBVFJSYio2tE52fKZkLwJ+QVv3guhE1t3wUK7ZUMDoKt04rNtBUaahQ07ZzlfHIMQohgfKoQRAcd2YqR5faxOnz8ZqkLwwrYBvL679q2NmhV+TagBv2rEcDlbjWJbpqYgB7U4Qb7OOVuV0OiQGR0DewzIchJGCbLFlK04LzYgamDm1ExhROJVlGq1Xpvul2whLeZvg7eH87uWuKpECdA2xf15y2erIsY92epLuS86dg3CiLStDiVWJ89z1BcnZ0tHu0FDeakApqZ/3fgScupW2LaCSyfORJRd+Yc5iUn2gUu5BirnbE3x+FFZEw8P/uMAlyrHW+3oZSp2DMV98hVc5zwRypYu3akbhCXF27rToqdztvO1MQghyrDLa1tS3lbzqkYT20ycdii1yvj1c9sqLH3gQPhpeW7tOQkLZf3QYAd5gCaJe5UtAsuxDg5h41ILjEU1YmCENFiVK+vkMCK/eY1pbD4LtU8bG2qWoUrHvPf4Hw0Ulq6dsrLiWhfhypav9YP0nID6QLoG2gojVsK4J1sDaUq2bIsePDYZPdlK5Zyw4MVLHILgkC0THVHaJiJFlIpk6wfP/xYA0IHDMY1YQtka0ViiYnIfbCKHEcsnKE5LuB3k9ZCl4rJtBU8k1fhv87lD4dWPMdY7y/Ik9vJEeEOSjzSmMBIX2WpiZWsckC0AuIg1qf7tc9uRL1gVlj4wYJUKI4oE+eqqERuVj2NqKvLQisKIpmgGP7YXLr4faqmcVIuwY2lnYcSsQlCQckP5DWBMCZ8H1+iKVRn1CCMCbF4GMCJ5IBqStVHR8qUS5DlaYcSKGPdkqz9DTzDFojk3RCkgnRsd4UpJ1YURzSEymqRsdUUo2UpWCCNaloUX9j8GAFg+8ywgM4QIu0gMMWXLGtnrSLmlcrake4uZnW4jUC2E7QPghBGzhCDPlS3x24qJW4SNZ0KBXtRkZSsreW7ptg2wvlmaQpUtF9nqmOl8cYyULVGNWGnBcUK2zlgwGRPiBnYNpnH/mtoY+DY77BJhRLsqny3peYOupKamIGtrRWFEI6QfVK0QNk+qniBFT8qj3XSKhVIZiWwRHkbkRrHB50hS5tVYoygxvUbgLdhGJINqXcpJLl7ew7bMBBCXcm7j9TanHv8Y92RrmJ1gBnGSxr2hxbBI5R1ly5DIlsoTcVUDHRHavDhJlLIJ8n989TkUtN2wLQ2fOO4CSrZ4GJFV+WVG9gjPHb2Uz5Z0rE9PdMGW/b9C3vLIOVs8kbRcNeJ7F56J7lwH3jNE97XcGTGdle+MAMQn0jExhVGFTqv8AKB7Ln0co+R4QJoqDxBly9RUfPQU2lngu4+97qrUO1DhKFvu45wnyBPBxsI6yNdmfGFhsjCi1/rBKGO/Uk+IC2kTsK2wfRo7IhLZysnzPlO21PB2Gi4X+wbvk7rlbLGK9CGu/NnE8YGqqGyxF5feC7zjJuC8bwLHXlWzsR2oGPdka5CTLSUhCMhAOlnuKxUhK2MRSekRPdg0ExOiVEnLKgTZnMe0U8Iv1lPHwAnKYvQkuoDMICJsPYJsDTlmp9T6oXzM3NA1ECsuXushczx489YcgWi1o0uqnRfnHrYUfznx81g+QverrGylco4KaNi2IC2Kws1S+QWQAAefDhxzufD3GQs4vRHHf84Wx+XLZqMrpmPj3hE88MKBr26J3CxS4n3RiDpIGFHO2WoMSlUjmmVaZtUTzahsBeUVhqaBsHk/laXzE/VepDd4QtkKE0Ys8bwRUOp0c6CAXmMcssUKNlTTd0O+FZrTFgMnfQpYeiVVulooi3FPtkZyXNmKAKxtz/4SytY/tr2O3294puI6M0zZsm0FimRwqQllS8eEuHNwjRRKV0C2K4cgPzQfCxOnsZUPOTlb7O4ixTung4cRK5nKEWh2mzOuKsOIORAU2G9yqhFLTEpmm1DkXGSLebMoNjNXjVFl6yCVmt2Z/DTVY/R3XfAd4PB3hxrvaCCsHw4QZQsA4qaGy06guYQrX9zV4NHUHyI3q0Q1YphG1DIapVoYaolqxDL2K/VE2ArAesIhfsHHojAbgwxrJD8i3QDGhbIV/Ia0GdRPjnqFEVVmVM3JFt+Hparai3K2WgiN0GRr+/bt+NCHPoQJEyYgGo3iyCOPxLPPPis+t20b119/PaZOnYpoNIrly5fjtddec62jr68Pl156Kdrb29HZ2YmrrroKw8PVhf6GWQWKqUZFj8T+dLGJZiaXw1UPXoHrnvoY1uzcVHadaW7uZqtuGVcKI1JjOPqZfHJ7MVk9BqltH8Hc2PFsIEOIsovEMLs4ZNgFXhCWAI1AdeKQPT0k2TIlB3kRRhQTfYlJyUiIi4ElnW1plq+mcjIzYwmySoS64wNoZw22oRf/prGAo2xVwDgiWwBw2nwahl315j5BOg5UlPLZckxNwyhbrlc1GF14mLqCnDdB3pWzNcaVXU1UjRjW1BQAFG5jkKcRhqGME2lo43NjGLLVBBWrHPVKkFc9YUQFlcgW8X3eQnCEIlv79+/HSSedBF3X8ac//Qnr16/Hf//3f6Orq0ss8/Wvfx233norvve97+Hpp59GPB7HWWedhXTaOQEuvfRSvPTSS3jooYfwwAMP4K9//Ss+9rGPVfUDkoxsRdQoFJseKIOZYrL1q5f+DlvrB1EK+OOrq8quM82ULWKrrpNesxyypSgKVFYBOeLt/Schm6cXAtETLDMoSEuKXRxyrD+izufeAO0SIqpDttSQZCui8zAigcUuVE5vxNLKlrjzli4SqRz36WJvdB+M/136R/QzI1RucwG9Me1l+G6r2Lx5nJGto6Z3oM3U0J/MYcOuwUYPp67gXNKbm8iLMUP5bDVBWb8prB8kEAumMBQbY+sHz2NDUUVIk1fWpdk8PJJ15uMIPyTGq7IlHfNhc3PLQRPKFl2nULZ8+iJ60eh9Ml4R6qz+2te+hpkzZ+LOO+8U782dO1c8t20b3/rWt/CFL3wB73rXuwAAP/7xjzFlyhTcd999uOSSS7BhwwasXLkS//jHP7B06VIAwHe+8x28853vxC233IJp06aF+gEpJh1HtCgUGLAADGVSRcvd/+pD4vna3S+VXadQtjwToqxsAYBm6yggh2SZBPlisjWECJOBkmzt2XQ/EAlOtkCAuNaO/awQUFfDCZQRkSAPEUYs5yBPNxITYUQZ2YKHbGkRaLFObEUXgDxiXHXQo0XfHRMEVbZYYv94IVuaquC4ud149OXdWPXGPhw+raPyl8YphIJVohqR2NVZPzQsZ0tTARBYtjwCG4bUe3Us4TjIN/4qWk1IUwUjW6z12HCWGSpbmjRnVxeabfQeUeqkKOmKCViOsqUKZcv/2GuG82a8I9RV+ne/+x2WLl2Kiy++GJMnT8bRRx+NO+64Q3y+ceNG7Nq1C8uXLxfvdXR04Pjjj8eqVVRNWrVqFTo7OwXRAoDly5dDURQ8/fTTvtvNZDIYHBwUf0NDjmt8ukBPrHaiYJZFc5+GfJStV4ecdW8ZfrXs78wI8uRVttx9oPhJnrRLk60MJ1uqQhOHpGpETgnzrB+fTFiK4Za2E4ZTfRnWZyuiSWFERobEtkvdAUa7oAAwLBsydeHEVIRA9AjipookoeuJ2ZwRNoZshc3ZeslK4f6/XI8Nu1+o88hGjxMPpmP+w7qdGMnkKyw9flHSZ6uK3ojuRN8GhRHZjZcNN9kyrdJFKvVEMylb1STrRwi90djD5ppklnv8aU7vvjDWDy5l68AMI3KvRU62NN6WroSy1QyK8HhHKLL15ptv4vbbb8chhxyCBx98EJ/4xCfwqU99CnfffTcAYNcumqw7ZYrb8GzKlCnis127dmHyZHfZv6Zp6O7uFst4cfPNN6Ojo0P8LVy4UHyWYWRr7sjLmFSgJGw4565GfGLTBhS0XvF62N4MyyptCJmRwojyaa967jw15lWSsiorW6am0HYSVl6QrQwjOvkcDQMZAZUtQoAOo1O8DpsgbzKylXWFEW1KtEqdSZF24KOPIG+7iaAgW2LsUcQMDUnm5h3nk12jwohBJ4ZIJ6Do+HUiji9s+i0ee+bb9RxWTXD6/MnQVYLnt/TjnG//DRv3lmj4Pc5RymdLVCOGSJBvhjt0TrYs1wWsgWSrCXO2wvxz2jXq8dSrWLjo5/+Gzz7+BfaJDhQ42arO+qHRqJf1g664E+Q1fu4ESJBv1Jlz2223Yc6cOYhEIjj++OPxzDOli93uuOMOnHLKKejq6kJXVxeWL19edvmxQCiyZVkWjjnmGHzlK1/B0UcfjY997GO4+uqr8b3vfa9e4wMAXHfddRgYGBB/69evF59lLEq24kRDlE1WKW6iyXDfq38GAEQKh8C2VUBN4rkdb5bcHg+NqSAg9/+zeF9QGs1DtuzSqkK2IIURM5QM8tyMLCM6hTyrqBShPD9TOffzbmYeCgC6Go5sxXjOFgjybMWajcoT0oylsGBCVraygmzxcGEEcUOl/mMA2rhDccOULQdl87YUBTj7ZrSzY2gotaf0sk2Cgya14e4rj8P0zii29CVx2Q+fRu9g6fzB8YpSPltOb8QQZKsJ7tD9yBZgw+A5oWNt/eDzrFGoRtmaGKWpJ+tNHa9m/oikQguyiK0D/Ea4WuuHRuds1YlsmcxQe5iRLd2uQLbk5w3YJ7/4xS+wYsUKfOlLX8Jzzz2HRYsW4ayzzsLu3bt9l3/88cfxgQ98AI899hhWrVqFmTNn4swzz8T27dvHeOQOQpGtqVOnulQlAFiwYAG2bNkCAOjpoZ3Ae3t7Xcv09vaKz3p6eop2UD6fR19fn1jGC9M00d7eLv4SCSc5PGfRi0vEaHOSsYc3ub4/W1+O1NbLsDByEQyLnpiPvvlcyd/Jla2onQfZ+4p4X4Vb2dIJPWDT5chWvphs6exAzzF70BwjLyIBvYKypRCCiTGnKEEL0BNOhhNGBPJsxZptB5qQFFt3nXlpFnIVRFGLIGZqSLMLX9zizU0bQ7bkCapi0d5xVyNx0BkAgMHc+FCJTjx4Iu675iTMmRDDtv0pfP5XzR/+DAtu/VDss8VztjjZCmlq2rBqRDpObxjRsNw5oWMFJ2drTDfri2ryx6azzhTPRN2quwI5jBg81aJeBKca1Mtni/fHHeRkq0IT9Ebvk2984xu4+uqrccUVV2DhwoX43ve+h1gshh/96Ee+y//sZz/DP//zP2Px4sWYP38+fvCDH8CyLDzyyCNjPHIHocjWSSedhFdeecX13quvvorZs6nnz9y5c9HT0+P6QYODg3j66aexbNkyAMCyZcvQ39+P1atXi2UeffRRWJaF448/PvQPyNuUbLXbliBbSnKra5m+YSA/fDjmdx2DHvNgAMCa3S+WXCfP2dI803EnL39id566Qk/uFEqHJAXZUlUgw8KFGg2pWSQLmB3IsoPX4OsJYP3Q0+b0R9SUUP9GRDWnGpEHQEUYsQIUGHApW0wFNG1n7HFDRZqNqY1XajZK2ZL+gRUrEgEkTJr/MZQfPwrRpISJH37kWCgE+Mure/Ba71DlL40jWCXCiIKEiWrEcBeBplK2iA2Th7xC5mCOFsJuYUy36o9qrB8O6qbXn2HPPEhcYcQqqxGDD6MuqJ+yRa8Bwwq/9nBly9+ipx77ZGhoyJWLncn4Wyhls1msXr3alQuuKAqWL18ucsErIZlMIpfLobu7u/LCdUKoq/RnPvMZPPXUU/jKV76C119/Hffccw++//3v45prrgFA70Y+/elP47/+67/wu9/9DuvWrcPll1+OadOm4cILLwRAlbCzzz4bV199NZ555hn8/e9/x7XXXotLLrkkdCUiABQY2epAQfQcVDM7XcvsHqL/xCmJCOZ3U2Vu07Db+0tGlp2gKojLC0cQEnZSG5xskTL5X35hRJ26v9vIAbFu2rEewZUtAoK3H7RYvFbCki1D6o0olC0EUra4GR4HDyMKsqXTnC3+m6Jcxm9wgjwQoCIRQHuEKoZDVmnvtGbEwZPa8I6FNFfyric3NXYwNUZFn61Q1YiNvnw6lcm2a/ptnLIl0qQav2uqCiMumDTb932VVKdsNTpkJsMd0ajdejnZsvmNPt9QiTBiqTGNBgsXLnTlYt98882+y+3duxeFQqFsLngl/Nu//RumTZvmImxjjVC3UMceeyx++9vf4rrrrsONN96IuXPn4lvf+hYuvfRSscznP/95jIyM4GMf+xj6+/tx8sknY+XKlYhEHMb8s5/9DNdeey3OOOMMKIqCiy66CLfeemtVP6AAerFvQwFRdsHXcntdy+xmeSyT202c1LUID+0GhqxNGMlk8Mb+nTiqZ45r+SwLI6ogTpNbsIa3Uj6VocaAPJAOomxpilC2InobgAxskgVi3cjlaNhVJMj6HPCK59ZiRkc3fvyO34n8qzCIM4PRPCHIyWHEALkilGw5vzdT4DlbThgxbqrIsOGaDfbZkmfOAMIWElFa4TdYpsK0WfGRE+fiwZd68ZvntuPzZ89HR3Rsc3/qBaFsea42ToJ8ddWIjQoR+SlbCgE0fmMyxjlbfLc2OmQGOGQ4zFiO7PEnWwqMqnK2XIJjg/eJ7K3lPf5Hg4hHwTL4jy5B9N1trmozjvXr12P69OnitWmGv5YFwVe/+lX8/Oc/x+OPP+7iIWON0Hr1eeedh/POO6/k54QQ3HjjjbjxxhtLLtPd3Y177rkn7KZ9YROmbFl5EUa0rCSQ7ANiVDLkytbkRASHTV2ML/1DAVGH8bafvRcZdRNuOvYOXLjwBLHOrMXDiHApWwTuvKaIFqdki9j0Su5zYmbz9G7VUBUgSZWtiNkGYB+Ikkch0o3sAP1ehN+hl5ByxTjYZo6eNrfi/vFDVHdOKO70rgGBwhcqcZ8Qf9p1OxSNOdBrEaZsWSI0ytXGpggjBtC22mP07mnILk2gmxUnHNSNQ6e04dXeYfxuzXZctmxOo4dUE5RsRF1NgnydcmDCgPpsucmWqihOyOstHUZkjyEGkzCj6Mrb2K+5v1S9stU8JQP1CiNGPH5aYlYvFUaUn9doGIlEAu3t7RWXmzhxIlRVLZsLXgq33HILvvrVr+Lhhx/GUUcdNarxjhbjvjeiTaiy0m5lRRuclEKAIRpKtG0buwdZGLHdREckBr0wFQCQUTcBAH7+0h9c68yynC3dJkAR2XIOUpPlXqUUxTmpPfCrRoyZjgFlMtopiImgcQEc5EcDWQ1L8m0HVLZ0YoCQYtKi2wDO+gqg6oibKnLEk/TfFNWIlZdPxKktyZDPb2x2EEJwybGzAAD3PLM1UI7aeEApn63iBPkg1YjS84bnbEl9VxVFUmHewqamYgjhxjK5ULy8RoxR52w1mm3Vy2cr4rnGOGSrsqnpWMMwDCxZssSVC86T3XkuuB++/vWv46abbsLKlStdvp6NwrgmW6l8FkShJ1PCygj/qjQhQIb2WhzK5JHKUcVocoKy9iksSZ6jO+JOmssWZGXLAQFcRCiq02bQSUKAfInkPtlni4URo1GnknDEaHcS5MtaP8gJ8qM78mWylVK4shWsGnFm+1TPO3TM68wJwLFXAQAimoqcQt+PNJpshdxXiTb6+4YVgoLoJDB+8J5jpsPQFGzYOYi12wYaPZyaoLTPVjXViLUPh4SFqReHEVVFBVhIfsytHxrPsQSqUbYAYJJV/L9XSQ2sHxrMturlsxX3zMemCCM2p/XDihUrcMcdd+Duu+/Ghg0b8IlPfAIjIyO44oorAACXX345rrvuOrH81772NXzxi1/Ej370I8yZMwe7du3Crl27qu7BXAuMa7LVn3TK8xP5jOOzpRChInFVKxHREDXoCTl/gtu+Yijrrt7KsbuhSmHEuEEtKFKEOBOlB77WD9EO2GxyGNLjgmxRFYj4Tgy1VLYMTYPNfFW4Hxb12ap89/eOQ46CW+2jOEKfI95TFIICO7KMBudshVa2Ek6RxvBIsOTLZkJnzMC5R1LCeOffNzZ4NLVBqXY9PMVxvIURDZWHEWVlS5XCiGPtsxW+ArBeEAarIb832aaKDJFaIOmKUV0Y0dUiJ+RAaox6teuJenJ9I3xOD9KIugEE9P3vfz9uueUWXH/99Vi8eDHWrFmDlStXiqT5LVu2YOdOpzDu9ttvRzabxXvf+15MnTpV/N1yyy1jPnaOsU0OqDH6mXpl2wrMXBIxucEzI1C7h1hyfMI5iJZNPwqPSFZfIx5PpZxVjmw5MmvMoMpWSlGAEv0Rs3K7Hka2YCZY89QChjSHbIm8J5+TquZ3FrYKEEsoW3pAn635Ew4G4HiUHWmp+NedvZj3zv90LWcp9Hc3XtlyngfJ2TLMNkQsG2mFYHBoBzo6ZgXf2Mg+bBx4E3p8CmaE+V6NcdXJc/Hb57fjgRd24rNnHoaZ3Q0qTqgReG6WX7seIhenhA0j1mBs1UAoWxIx0GRla8wd5N2PjUS1ytY8NQ4ghen5idimU0NiTdGdebnaasRww6g56hVGjBUlyFciW/7PxxLXXnstrr32Wt/PHn/8cdfrTZs21X9AITGula3BFHWPJ5YBZEecnoNSGJErWzyECADvWnA8JpJjxetUwUu2WHNl+JxsEtlqj0hhxFLKlpyzlaZhREq26Hh6bccmwbDtYAd7DaYAwrq8p2TrhwDhi7kdcyErW3OyaSxNZ9A5yR2atZVmydly9lXQFKZ2ttxQGGXr77di5f8cgfc+eAVuuP9i2GXaQdUTDz3xFWx67RaccshEFCwbd/ytdKeE8YJSYUTbtl3VwghigdIEqoXI2ZLIoUYal7PF0eiQGSBbP4Qby/vbZ+J/du3Gt+e8Q7xn2RYgXPmr9NlqImWrlmHEmOEmW1HJJ9EPjQ4jHggY12RrJMeNJ3UgmxQJ8mk5jMiUrSntDomJ6AYeu/xHOGPyx+nyRWSLKlsGyitb7QZVDJKKP9myLBu5Av2+bP0AM4GoQvt5bUiPIMcOXrMs2ar1RUJjY2dhxIA5W5Oik6BLF4lZKabWtc9wLffFI67Al/fsw6wck/Eb5CDvVraCIcFOi6ER/1YQvtj0BA7PZKHAxtN2Evc99u/Bv1sjJJN7seKN/8N1m+/Dse20QfzPnt6CF7b1j/lYagmRIO+5tS9YHrIVUtlqlG7BqxEL0vY1VVLHxzxnq3nCiKgypGmYcZySSmOeIeWjFoarzNlqfF4fh1In64c2wz0fRySfRF80Qa7jeMe4JlupHO/urgPZEcHOU4QAWaps9XJlq72YsXewMCBvZs3Bc7Z02y6bs9URZWFEovgmyHNVC2Bka4T124tPQrdBc4NeTQ26E+QD9KaqBQgjW652PQHu/ggh6FSdfTkrn6d3QzF3kcH5B52GC4ZH0GVVOInHEEGr8xJsPwwl9wVfebofM/N5/LNNK01v2fJHvPTyb0KPcTTYs2eDeP7DkafwvkNeRMGyseKXa5FmRSLjEY6DfPH7StgwYhOoFn7ViLqiOWRrzHO23I+NRDWmpgAAduOr8I4VADLWcFU5W2iCY4SjXmHENo+yFWkpW3XHuCZbSUa2FOhAbsSxfiCOiuR4bBWTmM4I9fjI2knX+3l2gnrDiAQApDYuXTyMqDjkToaLbKkKMMyUkrbJmBanStCWzH4RRtRtlPbYqvEEwMOIHNRnK9gk3yUlu8/KZ4H2acWDMjx5Qo1KkK9G2VKoejmU7gu+odR+AMBlp96IhZaKQYXgg09dj1/++dPB1zFK7Nn/hnhuEYKJiVWYlDDx+u5h3Pvs1jLfbG6UbERt2x6yFcTUVL5DbwwcB/lmy9lq/FXUydkKORZ+M5dz5vKsNVJdzlYNi5FGi3qFEeNFYcTyHo/NtE/GK8Y12Uqx0nwVGmBbTs6WIudsUXI0yYdsdUVoNWHeditbeZv7bPkoW1kn5NgRoW13khK5k8GT4wHAUIhEtqbgoE6aQL0334csuyMvG0assbRNPLURQR3kAaCTWV4AwPRcHmifXryQV8kaTzlbKh3rYGYg+IZS/QAArW0Kbn/Xr3C22gWLEHxr+8NIDocIR44Cewc3u16/mdqJa98+DwBwx982Il9oTB7ZaFGuXU/oMKLrpqVRYUQ6zoKsbKmqo8KMtc+W57GRqPpfwm/mcs5crhKpGrFK64dGyzj1atfTbrpvfiN8P+mllK3G5zqOd4xrspVmypbG7mh5GDFPCDJpeqHcN0IJ2aS2YhIzIUaVLYt4wogszm942vAQ2ILEAcCEKCVrWYUgM1KsgnCypasESqbfyR+IT8IRk2lCeQq7kWQkx7Dt0j4nNVa2FOKefPSAvREBoMt0XH9N2CXIVtzzuvHKVlBpK8HyywazAcmWbQtlC9EudHfPw1cveRizC8CQQnD5ve/AP919HLZteyr4wKvA7uEdAIBp7Cb11cII3rd0JrrjBrb0JfHHF8eflQUgteUpcpAH1HFZjchytmyv9UN4FaYWIFXH7mqPqm0ohLKVwpXzboSen47/Ovk/R2/9EHIYtUa9rB8SplfZKp9b6950o/fK+MT4JlssPs87lkcV5wAaSTGyNUwJ2URZ2UoPAL++GnPe+C0AwCJOaBAACqyCRYdX2YJL2ZoQT4jngyPF+T0u2weubkQ6Ac3EkumH0NfqEPbJZCtAzLwWcrICbxgxWII8AMyedIR7XB0+ZEvV3WGdBilb8r6yguZsMeVuyCc07Itc0iHSkU4AgKoZuGz62wEArygWnkQKK5//nutr//yrc/G1n74de3/6buDH78KX7zwB7/nxsXhh+5PBtuvB3iQ9xpZFqc/WHpUgNbwRH2Zte778h/XoGxl/Rq2lrB9sj/IcqDdiE+TjmD5hRJqz1aAwouexkajahoLl3yI9gM8Y+/DcjtU4Wx2oqujAnSfV4AR5VxixdutNGOGUrXrljr2VML7JVp5bNNBJVjPiwtQumR1GvmBhf5KebBPiTJpPDwI/vQhY90tMXfMDAABRMsgXnATiPFe2fMOIzgU4rptgDgcYHPYhW7LtwzDr69RGTdhmdHQDBXrAb9Z5zlZQ64fRQxlFGFGXlC0Cm+ZseUEIYEjqVjO06wn4nXaD/r6hfLLCkgxc1VJ012++4JQbMKXgbHVX0gknbtnyBP42sgU/z++B/uZjwJuPY1NmL16z03jz5fsDjtSNPZl+AMCsxAzMZIfza5sexQePieHM6d9FJPMsPnfvWlhW0D3RHCjXGzF0gnwTVJppCgEh7jCipqgAz5tptesJ/79h8yqGe4HXH6XE9aXfStYPlYm4GEMThczqRfyiuu6aECPM7qikslXDDiZvVYxrspVhZMtkP0MxYlAtShgGskPoS9LPFUKdtZEZAn72XmDbPwAACcmuYfeIEzIqsElPt63iU952SJmiKNAtZhPAL7gSXO7xUnI8hwnWh0+VPKlKKlu1nQAU4iVbCNwA1038bCA+2X9BmWA1g6lpUGWL9a4cKqQrLMnA8rUQ7XRtMBrrxs8v+BWuajsMANDLyBAAPLbubgDAEktHx/nfBd5zB2bHqCK1ud9JdA+DPXl6IzApPg2H6pQwvrprNe5+5Bqsat8Ge9ZPsOHNJ/HVlS9Xtf5GoZTPVsG2obpytgKcGE2gbBFCYGqKq12PLnuEjXEYUdgtjPFW/VB1GDHBGhIP9QJDNJyOrU9XZ/3QRMng9WrXoygKYDv7JMoVwBLKFppon4xXjG+yxciSyQ9Cow1Rm15ktueHsW+Yft4dN6AqBPjtx+kJGOkAzA6YNkAYWdo94iS48wR5wy647pwVUpxgrLGqvnS6OEE+ky+tbAFAh+7uWE7DiP53tYrrIjH6w1315GxpCK5sucaiR4GZx/kvKOdplaqyrDPkfRVY2YpOAAAMWQFDblK+lhcTJ87HMTNOAgDskvzcHt1DXfhPn3YicPSHgKPehzmTDgcAbE5Wl1u1p0BD5pPaZ+LQNlqA8dy+l/Cr4dcBAPs1BV0z7sAP/voqfrV6W1XbaARK+WzZNkK16gGap4Td1FRYUijfkMnWGCtbfLc2OmQGOP+T0GMRytYuYJC1bdnzMjCyl62wOgLb6F2i1slnCwCI7ewTkxcWlFS2pOeNP0zGJQ4MssUPBSOOTpWahW61M9jL8rUmxFlo7o1H6eP7fwpMOhQAoDF2v1dWtpj0bMCtbBEAOORM1xhUNmFmc6WrEU1N9SVbPTF3rlN564faHuH+ylYwsuVKIP30Wueu0gtOtvRYU5yhQasRE4xsDfK74kpI99NHlq/lxZRueqztAj2u9u19FWtAj83TF31ULDeb9ezclB9CNdjDbgYmdR2MQycdBQB42BrAiEIwuwBELRtbTII5kXW44XcvYWtfwDBpg1HKZ8tlahrA9gHwHLsNPCZNTXGbmso/7q3crqfK3ohiXi1kKeHi2PoMfazaQb7ROVv+z2sDSdniLetKVSO2TE1HjXFNtrJMeRCHhx7DRJPmD+1UCkLZmtBm0Cav3INlyhGigi7CKoL2JR2yVGCVGYZtuXO2jr4MeM/3XWNQ2N1BVkqcF+MrSAny3NBUCiPO7ZjtWr689QN7rNFxHlXc1YJBTU29INHO0h/yJMwGG5ryfRakNyIAJOKUsA8ioFVCGWULAHomHwkA6FcIUsk+PPb892ARggWWip6pR4vl5kxdAgDYggIs3pQ4INKp/Rhis/GkCYfh2CM+gBmSj+mVs87CkYSeKYdPehXDmTyu+826UNtoFMr5bKmjUbZqMbgqYeoKLKka0ZDJ4lhXIzbRxZMUPQkIPeJ/s8N65FbrIN9o1MtnCwCI7SioTm/ElrJVL4xrspVjypY4PIw2zOyYAwDYphHsG6QEaGKb6Zx0bDl0UFPRODvG+tLO5xYjW7pVgBx8InNPKrqg8kTzXM6HbFUIIy6YNNe1fNlqRJE4Wht0mhNcr8NUIwbOaeAkq0GtejjEGAMqW13tNATXryBYj0M5Z8sH7YkZwnC3d/c6/GH7XwEAZ0082rXc1J4l0GwbGYWgt3dtoLE+tvZOPLfpUezZ9woAwLRsJBLT0dE5B7+79Cl8eeb5+OzEE/Cu076CIxNzAACJju3QFIInXt+LNVv7A22nkSjps2XZUAhXtgKSrSbI2QLoDZjLQV7Ez/QxH1hzJshXgVIKOzBueyPWy2cLYGbgDIboYVvZ1LSF6jCuyVaWka0In3CNGOax8OA2XcPQEA0NTmgzRK9EaBGaF8Uq6NrZxTS9d71Yb4GFewy7ULEBtAJ6d5ArOCGZXCGHkexIiQT5SWK5Y6bOc63L8PRelOEkjtbmqJ8Yneh6rQdsRC2PpeJ4uNdWw5UtOsagOVtdnQcBALKEYGSkt/IXKihbRFHQw1SMNW/8Ec8SGkI8d+mnXMtpegQzWQ7hpp3/KLvJvr7X8fmfvg2fWvMNfPGRa7H1sRsBABNtAsLyf3QzjgtO/wo+fO4dUDUDR02luXUbsnvwrsVU2f3+X6tLxh9LcC9W77FmyTlbAavNKp3PYwVTU92mpvxK2oAm1E1ks1W9gzzgupEFK3IRCGH94FY/Gx1GrF/YW/ZaNO0KylYTEdDxinFNtnIsjCg6lhtxHD2D5r1s1zT099PQ3cQ20yFbJvPGYmHEqYV+AMC0V/5HrFdWtlxhRJ+DTCGcbDmVaw+9+RRO+L8T8IVnPkY3WULZOmziNNiWc8Drtl3yolHrCbFHIn1i27VWtpoljMgeg+ZsRWPdiDElqm9/ADJSIWcLAHpYP8m7Nv8JALDUNl0hRI7ZGvULevDV3+DJZ/8Hz665C+teuhevvPYH9O/fCAB49Mmv48L7L8SfCn1QbBtnDI9gX+/zAIDJSumL9VGHnA8AeF21kcj9O45O/AErX9yFV3uryxEbK1gllK2C7CAfOIwoX7xqMryqYOrunC2Djz9gRXAtUXUFYB1Qdc4W4Fa2Zp/oJlxhrB9IcxwjQP18tgBAhefaA1LyGtCyfhg9xv7MriF41WCcT7hGG46YPBOGZSOrELze+waAg6jHVoZVpXjIVhtXtmyn8qxg5wECGHbebZroA0JojlVBIltPbHqVrjNDD9yIYjtVMRLZotYRk5BXaKmyabsd6l3bEdsrO5zAmNk+xfVaAwJL7YHj95xkNcg9niNszhYAdNkESQB9A5sxCyeXX7iCsgUAPUY7kE3hDWbzce70U32XmxObAowM4dfZnfj1S7e7PjNsGz9cch0+++qPkVMIDrEU3HT8F3F4cghPbfglTrH3Y970E0qOYeKkBZhQsLFPJfh1fhum9GyF9dq5+Oy9a/HrT5wIXW3Oey+7RM6W7crZCnZiNEtZP7V+cAiACCM2QNniO6IZrqGjmudkZat9GjDzWOD1h+nrKtv1NHqX1NNgVSXOsWbaoPN1iW00y3kznnFAkC3RRFOPQVNV9ORtbDEI9iQ3AzgIE/yUrQ432RqRSq8ttj7DKlRk9Apzrc/bGfHe1sHtdD25bgBANxkEYNO775g7Vyqh9WA/KNkybNu3xyLdNnus0aE+t2uqeK7azMu6GmVrPIQRQQDYgZUtAJhANGxHHvuGAlgkVMjZAoApkUlAlqqbum3jHcd92ne5kw86B3e/cCvabKAHKnK2hSyAfmIjqRBc94+vIqcSzCsQ/OLSJ6GbdB+fsPQKlKZZDk4wJ+EPeUr8ezWCidEMXtg2gO//9U1c8/Z5Fb7dGIhqRM+tfcGSTIcDViO60EhlS1ORI4qIbZtc2QoR7qoVBMFphsvoaOY5Wdlqnwokpjpka5zmbCl1tH7QJLJVLl/Yi0bvk/GK5ryVDQjew1CQLebePbVAJ17FpP0KJ7YZDonh7ufsLijOyNaQJDNbYNWIVq64XY8HikKJRN7O04pHAL1JSp7sHFU6JqCfLhyfVCRnT4467uuGDdpKyAe17l82b4Kz3YKcnBsAgSfCJlG2+HDD+KZ3s2bUfcMBPK+CKFttzv4+Re1AR8cs3+WOP+ZjePp9f8UTl6/Bb65Yi99fuQ4PXrkOXznsMgDANnb4vL1zviBaYfCpt/8/fLp7qXj9yWX0JuGOv72JZDZcBeRYoXQ1IsKHEZukhN3QFFdvRHHZa0QYkYfumuAiOqqQpqxsJaa6/f/C5Gw1yTEC1LcaUWP7RLNtSgTK3BQ30z4ZrxjXZMtiylaM93ViZGsaK2kdatsEwPbP2WKkp43dNo8oEH20bJYgr1vuMKLfsa4qlEhkCBEVj/05mgxvMbLVbfXThSXbB47Z7c5F17BtoGtu2d9cq8N8WqKz+M2QDvIVz33euqaUK/EYwcnZCk63upkq15faW3nhIDlbnc7/9dw555RdXTTWDcXzvzh1ybXokNrsnHbYRZXH5YNp05biqvPvxAKLHv8z27Zj9oQY+pM5/OIfW6taZ71RymfLskZp/dBQZUtxJcgbjUyQ9zw2EqP6n8jKVmIqMH2Jc1xUSWIbTUDr6bOlsxQYUYlYRtlqlvNmPGNck60Cy7OKMdLFL+4fNKfAtCwMxPdA73zaXY1oJlzr4MrWsKLQvolwEuQ1u3JlisYSnzOEiG2kbZqYb2Up2eq0mfLh09bmsAlznHUt/Shw2r/7/tZaGw8qis+/PmQIo+JQJi+gj5MWhFpvrSFytkJIW90GTa7tyxS3YfLiJj2FL0/owt4yZ9OcacdBsW20Wzbeduwngw+EQTfjOCdGifmEgo0j5ldHtjhmaPQ82N7/Gq4+hVZf/uBvG0UFbTOhVM6WZUu9EauqRmwcqIO8lCDPb+oaEUasdULoKOAMpZow4lT3c7MNmHUifd0+3f87pcbR+F0BoL7Kls6IvWP7UE7Zkp7XdBRvHYxzskVJUZQ7fbODZX5iEj69n4bjzCl/wP5srz/ZuuQeFBK0b90IIUKhEMoWbKCCssUP2CwhQHoQI5kMLJVum4cROy0aznTJ3AzHTp9Pl7V06OfeUjLvR8jrNTzUbctzFx3SQb7ihLjgfODTLwKnfraa4dUM1eyzbuYi35f1z6HjsAt5/DZm4OftCeSN0mG96dOPw20L/wk/OPHLMCMdJZcrh8uW/X+YUwCu7DmxSPkKi1kxeixuGdqK9y6ZgckJE9v7U/jxqk2jWm89wMOI3sOtIDeKr6oasXGXDVNXkJOVLT6uMXaPB5pT2apqLIkedhwQmrMFAO//CfDxJ4BJh4UbhxhPY/dKPfPHdCWMstViW6PFuE6Qt+w8CGSyxS52ZgIfHBzCHyYcilkz34WeWI8/2Zp/LnbtSQJv3oikohSTLXkyh/8xZrADlitb63o3gRAbtqXBLtAy/o4CU0d8wohHT5uL0yZejQnlnNhRe2ULABSrDbbSJ70RNGfL/VgWnTPDDqvm4PvMChNGZGSkL1++pU1qpBc5toH29vK/9eTjwitaMmbNOhm/v7I2ru8zO+YAw69ga3ofIrqKFe84FP/+m3X4zqOv471LZtDG7U0C7rPll7MVOozYJNeMV7L/h9enbgR2HAJAztlqhLJ1gORsGXHgvG8CVt7Jn4x107+w4yAE4IVDDUQ9lS1DqVbZavReGZ8Y18qWBZYg7+1YbrZDAXBPz4n42vJ/gaqoUoK8O4wYZd/JEojkdE62aBhRvhMuHkOEtdfJKATIDOKl3Zvo2HJd4NN5Ilda2QKA75z7Kdxw+uVlfyvfdC1POJ14lJha52w1Cfg+CxVGTNCE9j4eoi6BQVatqNk2orz4Yhxg1kTqR7eVmfFevHQm5vckMJDK4TO/WIN8oXnCiaV8tiyrGp8t6XkDj9+ZHV2A4vRTMrgxcytny/UYGks+Ahz70YqLVYLTnHvUqxrlOGSyVdt1mypXttgbZZSt+vZofGvggCBbEWZuKqre4tQdnSSl5GahbLkviFFNDgMyskVYzhZsV7jGT1LmByxXtl7r20LXwUKIAJDI76NPfJStoOAnXS2Pc5N4yEFAZUuMZZywLZEgH+I73Uyl6kOh7HIDQ7TytMMeP/sDAGZOOQYAsEOxkc+loSoEX73oKER0BY+9sgdfvP+lUAUF9USpnK2CnLNVVbuexv2/Lll0iuu1McCKE4y2MR8L3w+1Vk6qQbOMpdYdO6pFPX22DHbtEu7xZQuZmiP8Pp4xrskWJ0XRPPO44sycJ0oO7XQWLpEgHxPKFpFsFyRlSyIgfoeYqclka1Dy2HLIVqyCshUEThJr1asoQlRz74uwPlvj5pQTCfIhwoisx2Y/AQr5bMnlBlhngA5U4fPUQEyefAQM20aeEOzcRd3nF8/sxLcvORqEAP/3zBb8z+PN0crHz2fLtqlvWliyhQpK9VjhiAlHQKb/kZf/QJ8s+fCYj6WZlOqmmVuaZJ/U02fLZCqqHiRnq0nC7+MZ45pscVIUKTCyxWPOvAR4MDjZyhDimFMSOoHrtg2iSWTL58yLeBLkd6foNmVlK5YdvbIlxjDqNTg4rHOh+42QDvKNnoiCohplq5NZcFiEYGBwS8nlBpO08rSDjH2uzWigqBpmMJ+nrazVDwCcdXgPvnQePS7+34Ov4M8vBfAZqzP8woiCgIkqvvFVjdhmtKHLcOYis5AGZp8MzD9vzMfi3Mc1/oQWI2i4suV+bBTqGUbkKTQt64exwbgmWzZhYcQca5XDw4hC2ZIuFCXJFlWmcnIYkStbsEHU8spWjBG8DKHbGMhRpWNShI7BRBZmnm17FGQrcAVgCNxy9j/jGOPt+MFO1rcxaHKuCGmOj7NONKIOwbZ0PSZ8rfr2v1lyuYE0JdIdjWizMkq8W5+Cj+0fQE/a3SLqIyfNxRUnzQEAfPbetdi2v3yRQL3h+GwR6T1OwEaTs9XY43dmzJkPDNjAWV9uyJWsHqp5tWgWZatZ7DDqGUZsZyHrWKAEeUkRbvh/Z3xiXJMtotAwoulNkOfKVmYAyI6w5yV8tgxJ2eJhROKEEeHK2SoeQ1Rz52ylQfPEDp1APZEmgq1TNcqaXlZCPc79mG7i7otuxvFppgxy0tqAsdQTzjjD5SB1M+Wnr4yyNZCmlabtaoNd8qvARyYei0/2D+CgkYGiz647ZwEWzezEYDqPy3/0DHYNBDs26gG/nK0CY2Dhc7bki0ZjMTPmmHAqh7wDmLa4IeNwbGUaj2Zpit0s+4S4lK3ajubqpefi+MLBuLqfnf8Bla2G75RxinFNtjicBD92wTPbHRsIrm6VSJCPGXLOVj8sywJhYUTN9ihbfmRLCkOmMsOwFFr1ePQ0ahQ5ibADuW3KqGaQut3xyd5QWf8m2GM2ljpBhBFD5ntPYqXRvf2bSi4zwHy42st4bDUtummoFH0biz4yNAW3ffBoTOuI4M09I3jf/65qmMLFiZV8+tjeMGLA3ojNFA45cvIi8Vw59sqGjaOVs1WMZtknsrJV67FMa+/GD46+GIszvMAsoPVDo/854xQHENkiTtk0IY66NbQLsKySylaCka0cISgk+5HKOaX+NIzohIf85NMYU7ayhGBTso95bOk4fAq1DphE+umCo8zXctSkOhzpp/wrMH0pcFj5NjLOWGof0qwnRBgx5PemGZ0AgB2Dm0suM5CjBLXDqM6otKEQZMs/TDqjK4Zf/NMyzOqOYUtfEu/73ips2jsyhgOk8AsjFmyvshXsWGwmv6A5k48Qz5Ua5HNWC4fgNP58rus8V804DuDeiADcBKuVIF9XjH+yZTNnVj3mPiLkisTcCMSl1kO22kznYMtk+pEuOJVnug2XsuV3lMWkMORm5uWlWRPQFTcxAQP4gPoo21D1lYiAlLM1qrWUwBnXA1c/UvbOxj0W1G8sdUC1ytY0Zmy6I1k6SXwwnwIAdJidVYysweim6iv2b6I3JD6Y2R3DL/9pGQ6aFMeOgTTe97+r8Frv0NiNEXIY0XnPKiJb1Vg/1GR4VaN5VLbmCN3RMTRX+K7R+6SeCfIA3HN+OWWrZf0waox7skVsjR4G3gNFKFs7HVVL0YrYe5vkLZJJuZUt3RtG9Nl+XCTIE2xjKkebOgldMQM/Nr6K09U1bEOjVLaaRNYG5IqhRo4iOPg+s0NqW9PbZwMAtmf7Sy4zWKC5TB2svc+4QsdMGn7Lp4Dh0oSypyOCX3xsGeb3JLB7KINP/t/zJZetBwSxkq42lsjZClmN2EQHrXwuqw10imymuYWj0WNplpoBuYVtra0fALivh2WUrVa3ntFj3JMtxWaTbEmytcsdQvScxVFdF6JXNjuIjOSppAEgmhRG9JkB2kxH2dpl0Qtvt9mDzoiGeYR6bqUSs4ElV1Tz85xt+zxrFMabssVHGlbZmj6B9lPbXiidHD7AWkV1xCZWN7RGQtWBTlrI4Ze3JWNSwsRPP3o8VIXg5V1D2No3dvlb/tWI7L2QDvKui0bDL+h1DhEFHgd/bPwZ3TQhzSYhoPUPI0qFPWWVLel54w+TcYkDmGxJYcQS+VoAoCgKYNOKw1x2GKkcJVvEpqc7kasRfbbfZtDtZgnBLkK/OzU+De1aDiYzXX3mzPtGX2nUJLI2MB5ztuhjaLI1+SgAwC7FLmlsOsBsQtrjowsTNwwV8rZkTGwzccysTgDAX1/bU8dBueHns8WT5kP3RnQ9b3BekLT5RrZAaSZlS/xPmoNrNfwYqX8YMZiy1bJ+GD3GPdnSuHO390DhXd97XwLW3Uufl+pdx8mWnUM6TfOuiM0IhZwg73OMtUk5W72s19mcjplQmSVAxtagmKNvwdEssjaAprnrCwqRsxUyjDhp0uHQmMv67t3+DaAHmM9TR9vU0Qyxceg+CElC8Mt1P8Ktj36u4uKnHDIJAPC3V/dWWLJ2sH2ULZ7HpQqfraCmpnLuSW3GVy3cakEjla3mOZGbRTUnTTKQevpsAQC0gAny8vNG/3PGKcY92dKZF5JLDgUcZWvPy8DT36PPI6UqxmheVpYQ7N5PGwubFsvVcilbxUdZwqTbtQjBDo1O/AsmzQZStEVPRm/HSfMmhflJvmiuu0/3Y7OjWmVL1QxMteiXt/uQrVx2BEmWVNGRmDmqMTYMEw8FANyEvbhj60oMDG4ru/iph9Jj+e9v7B2zZtVc2ZIJSaGIbFWjbDUYTadsNXyPNI2HX5NwLc/NQT3CiHKCfLBqxBaqw7gnWxo/HbwHStcc5/mEecCsZcCJn/RdB7EZ2QJBbz9tCBu3qaJFWLNOwP+AS5jOdgdVusBRPXOBJCVb7V1TapLY6JjsNf6or4ebfT3B91k1fZWnq3Qy2t73ctFng6x5MLFttI1XZeuo9yN2wjXoZsRp+45nyy5+5PQOdMZ0DKXz+M/fr0c6V75Rdy0gDEylw43nbGmCbAU7Ft3ViA0OIzZdzlbj0Sw5Ww7pa/xe4cd93asRtaDViHUYx1sA455sGSihbLVPAy79NXD574BrnwWuXFnSR0qh5hHIKAR7BncAANqY2lWpGpH7dHHYlo45nZOAFA0jItYd8hf5g9TzhAuJZpqcg4Dvs7BhRACYLry2il3kB4bpsZKwAVUbX70RBaKdwFlfxgxCbyq27Xmx7OKqQnDt2+cBAH7y1GZ89t619R6hf4K85VG2qqhGbPTxW/cQUUA0i80BHUNzjIX/P5phvnXGUo8wonT9KqNsNcuxOp5xAJAtrmz5sPJDlgMHva3imasQp5n0QIom/naCvqdUaEStKApgORO9arXT91gYEdGuwL+l/BhLj2GswcuRm2EsQVBNb0SOaSzxfXtyd9Fng4xstdvjYz+UwwyD5jNu66+cKP/RUw7CDz+8FADwh3U7sbHORqd+Pltc7dLChhGJ//NGoFnyx5wbucYfx80ylmZK26gr8QuobLWsH0aPA4dslTtQKoArW1kQDGRoY+FO4qNslTrKbCevyyQsLyzJlK0aka1Gy+oymqV/WVhUwbUwNUG9tnblBoo+GxihBKyDaEWfjTfMYAau20Z2BFr+jAVTcPr8ybBt4K6/l7eNGC18fbZEztYoqhGb5IIONNhnq0n6AALNNLc0T9oGv7mti8+WojqFYyVzmr1hxMbvk/GIcU+2TH4FDeh+7geVEasMAYbztBqxm11AiSblbJX4PoFTsRhXmb0EV7ZqFEZspgrAZkkeDQonQT483ZrYPgMAsK+QKfpsMEkr8jqUcRpClDCjfQ4AYFumL/B3rjxpLgDg3tXbsL0/VY9hASjls8WVLfZGwGrEpjqPpOeNDSO6HxuJZhgD0Fz7pK5hRAC44DvAO28BEqXta1rtekaPcU+2RJS5TLy5ElQ4YcQRi7rAT1IY2VLLhxEBJ8EeADo1dpfAc7aiNcrZEmOoyepqgmYaSzk4DvLhMbFjDgBgHymuvOtPUxW0Q62e6DcLZk5YAADYVghOmk6aNwHHzOpEMlvA53+1VuRR1Rp+PlvVmpo2lYrjyoNp4Dh8njUKzZKY3vg94aDu+WOHXwgcd3XZRdyKcJ3GcYDjACBbbLL1JsiHgKo41g8joBcbQba08j5bAEDgkK1JOiNbydoqW07PsMYf6c00liAYTTXihG6aDL5fIcjl3K7pO0Z2AgAmjce+iB7M6FkMANip2MjnSjvmyyCE4JaLFyGiK/j76/uw4pdrMJjOVf5iSPj5bDk5W9X1Rmz0xZyiOUIzzaTiNItq3kzHSTPksdXdguItgPFPtvhMPIowoibCiARJQi8Wk3l1k2xqWuL7ikS2JnMD0xonyDeTstVMYwkCZ5zh2VZnxxyo7Bjb73FZ35yiOVtzOg4azfCaApMnHQGdGbj27n4h8PcOmtSGm951BBQC3LdmBz78o2eqCteWAydWxJdssTeCWj94HhuJplG2xM1T49EsOVvNpIByktUMhLyF6jHuyZZpszvbUSTIa6waMUcIhlXaYmcKz0pUjYp3fipxCNn0CMvZ4spWrcKITXLHBzTXWIKAj7MaDqCoGrrZIbZ3/+uuzzbnaRXenElHjGJ0zQFF1TDdosf8tt41ob578dKZuPfjJyJmqHh+Sz+eeL227vJ+YcQU8/cyeW1CUOuHJrEWAJooZ4s/NsM+aZK5pZnUvrr6bAVEMx0j4xXjnmwtZKrUaJQtnYURBxUFGbZHeviRrRoVcxp4gj0AzIzE6ZM6+Ww1g4Q7/nojsjBild+fyIol9g5sFu/lMiPYrtA1zpl+wqjG1yyYodJQ/LZ9r4T+7pLZXXjfUuqi//2/VraPCAO/MGIyS8mWwWewsGHEhl/O3edPQ60Ommmf8Mcmydlqhn1S9wT5AGgWEjyeMb7JlhXBRawR9OjIFlWmdmn07ti2VXRyxUzVK94Ny2RrVjQGWBaQ7qdv1CxBvnlk7XHCsQRGo2wBQLdCK1L3DW8X723d8TQKhCBm2Zg4ccEoR9gcmBGhx+q2wc0VlvTHVSfPhUKAv722F6s376/ZuPysH1KMbEW4shXW+qEJjmGXstXAmbhZQncAxCAaPZRmUkD5cV8X64fAGF832M2IcU22ZpnHoa3AEnJHQ7ZYXlavRmduxWqDYtFwIjSzYp5HwXaSgudEY5RocbJWq5ytJko2GXeSMhtntblEE3UaGt430ive27zrOQDAbGggjbxS1hAz2qjNxbZUsYFrEMzsjuE9x9B1/Osv12Akk6/JuPzDiHTdJo8eBm5EzR5rMrLRwZ2z1QSqRRPslGabW5phGE0RRmyi82a8YlxfJT50xAUArxAbBdkymHLBlS3NTgB5VpHlytnyP9SytlOl1k4KTgjRaAOkasZaoBkO9maanINAKFtVfn+CSc3+9qUdD6rN+2ivxDl6aSPA8YYZXQcDALblhqpexxfPXYhpHRFs2pfELX8OH470g5/PVipLb2b0kGFENJGK03y9ERu/U5zd0OAwYhPNcU0RRuSPTbA/xivGNdk6o2cGkGO+QKNIkDeZsrVLpWQrrsSALPXbgtFWMYRXkMgWcumaJ8cDsqzdDEc73x/NMJbKGE27HgCYGJ0IANibHRDvbRqivRJnx6eNbnBNhBks0X8bqrdv6IjpuPmiowAA9zy9BXuGis1gw8KvXU8y61W2xmPOlv/zho2j8bukaUKazUL6gObo0zje7H6aEeOabGFwGyU3wOiULUa2Miwc1K7EgCzr92a2OQmkJY4zi0hGkPmUZPvQWfWYvGii+bCp7vqCwFG2qmNbE+I9AIB9eYdUb2ZO67O75o1qbM2EGVOXAAD6FYKhwe0Vli6NUw+ZiMUzO5HJW/hRDVr5cGWLuJQtT4J8wFBus96hN1a1aI48KaB5wlXNQvrkMTTU+qHoSQthMc7J1o6ahBFN1XS97lZNIMOVrXhFmd1WJBPIXArY+yp93jW76jF50UwEp5mIXxAIB/lqla12WmW318qK9zZZ9H8+d8ri0QytqRBv60E3Yzbbd66uej2EEPzzaTQkefeTm/DQ+t4K3ygP7qnlCiMy6wcjtLLVfMQCaLRqwR8bv1eahQw3C+kDmiSM2ET7Y7xi/JOt/OiVLdOTVzVZNYAsy1sxEpKpnP/3JylUETgqnaFK26519IOeRVWPyYtmOOE4mmksQcDHWS3ZmtBOSTNv2TM8tBN7VbrOWQeI7QPHDGbQu23PS6Naz/IFU3DiwROQzBZw9Y+fxZf/sB65QnHLoyDwS5B3rB/CtusBW1fjj115DM3gDt7QYjcGUXnX4P9PM81xzZAg30z7Y7xinJOt7Y6yNYqcrYhH2ZqiqK4wYiVV6ccX/D9cYR2E7/TuoeMRZOvIqsfkRTMd4s2U4xEG1YYRJ02cDwAYVAi2bXsKm7evAgBMKNhItE+v2fiaAdNZu6lt/W+Maj2KQnDXFceJZtV3/G0jPv6T1VVVhPKvqNLVJl2kbIWrRmyGY9elbDXwStpEu0Sg0df0ZlHYAOfYUJuADTfD/hivGN9kq0/KBxmFshXV3WRrOikA3PohQBhxZucErJixGN3cX2sPq8KqJdlqJql/nEnKo02Qb++YiWWgx9f/rboZm3vXAgBmK2a5r41LzIhNAQBsG64+Z4vD0BRcf/5C/O9lS2BoCh55eTce3hDeVoIrW/Kxz5WtsNWITZWfhOZQlJpzbmn0TmEPjd8lTdWupwl2x7jFOCdbklP1KMiWV9maVpCqC422YAZ3vBH29ucAuwDEJgDttaxUa76LRDNMzkHgJMhXjw/NvxQA8JvhN7B+NyVbc8zaVZs2C2aykOm2TF+FJYPjrMN7cNXJVOG6+Y8bQocTy7XrCR1GbEJi0ejQTDMS0EYPptIN9liiGcKI423Ob0aMb7KVZD3YFA1Q9fLLlkHUiLheT8v20yd6DFDUYJKyztYhhxBreGA2U4J8k8yHgeEkyFdPt05eeg3mFAiGFYJfDNMeibPbZtZieE2FGRNoyHSzfMNRA/zzaQdjQtzAm3tHcPvj4UKU/j5bHmUrYG9EjmY4j5olf6wZQ6uNHgpploGgOfKlmmF33HbbbZgzZw4ikQiOP/54PPPMM2WXv/feezF//nxEIhEceeSR+OMf/zhGI/XH+CZbHGZiVF+Pqu4E+Z7kHvrEYH0Og0jbPGfMLrCV1C6EKA2hqS4SzTCWIBBkaxTrUFQNH5p+GgAgzW4x5zBiciBh3uzTAADbVFoIUCskIjq+eN5CAMCtj7yGF7b1B/5uOZ+t0GHEJrhocDTLDVQzqTjO3NJotc/92Eg0QwFDown5L37xC6xYsQJf+tKX8Nxzz2HRokU466yzsHu3f1rCk08+iQ984AO46qqr8Pzzz+PCCy/EhRdeiBdffHGMR+7gwCBb85aP6usxSdnSbRsJ7jFktAEISC68Ycyeo0Y1Ji+ayVSumcYSBGKco2FbAM4/5Xq0W85K5jBfqgMJXd0HY2qB/sYNb/ypput+1+JpOOeIHuQtGx/4/lP43dodgb7HrR9cPls57iAvTLgCrau5wiGNVyyA5iF98hgaPZRmCjcrTTCWRoeav/GNb+Dqq6/GFVdcgYULF+J73/seYrEYfvSjH/ku/+1vfxtnn302Pve5z2HBggW46aabcMwxx+C73/3uGI/cgVZ5kXGAoy4Z1dfjEtnqLhSg8HY7JiNbQbx5ishWS9lqFvBx/uzpLfjra3tGta5J/Zdjd3Y/CIC7106A8uLoLBKaEfm9FyFtZ/Hfg8NYsLW2v68rbqCn3cSuwQw+9X/P4+H1vZjQVr6l1ZY+ahrsDiNyZYuTrfHbG7HxCfLNl7PV6Lml0eRCRlOFEWs4hqGhIQwODorXpmnCNIuLjrLZLFavXo3rrrtOvKcoCpYvX45Vq1b5rnvVqlVYsWKF672zzjoL9913X20GXwUODLJ10Gmj+npMqkbslpN3DRqebI9qGEjl0BYps7tksqVFgAmHjGpMXrRHaU5aotwYxgjNNJYgaI/Q8T68YXTmmhSHi2d3PzX6ir3mxHEAgKf2A09t31TXLQVVtwCgXTreeIK8Jph/MJGeH7PNcOwmTDoGfj41Cu1inzR2HACda4HG/3+aZRyAM5b2Bo6lHufNwoULXa+/9KUv4YYbbihabu/evSgUCpgyZYrr/SlTpuDll1/2XfeuXbt8l9+1a9foBj0KNP5IGi0WfwhQR/cz2gyHKHUXCs4HLGfrW+8/Gjv6U5jaUabiUZOS7CcvHPWYvFg6uwtffc+RWDK7q6brrQaLZnTgaxcdiUUzOxs9lEC44YKF+N2aHSiMIkFeRiGfgaKaDb/7rhe2bP8HHty3Bp02wcWLPlqXbQym8vjZ05th2cA5R/TgoEnxsstP7Yji2DlO9aewfiDhqhFndMXw7UsWY2Z3rLqB1xCT2yO47YPHYEp7Yy1E3n3MDAC0arTROH/RNGTzFt6xsLFj+a8Lj8SarfuxuAnmuBvOPxzPbt7vOv7HGlM7ovjOB47GtM7qq/69WL9+PaZPd3wK/VStAwnjm2y9+w7gxPeOejVtUhixy5KULRZGXDK7qzLJkZWtGocQAWpsd8lxs2q+3mpACMH7j22OsQTBvMkJrDjzsEYPY9xg717g8T98HWnbxjWn/DtisYl12U5bRMPtj7+BZzfvx1fefSS64uXDiTK4qanGw4ghqhHftbh5jGjPPWpqo4eANlPDZcvmNHoYAICY0RxjOawngcN6Rld4VSscMiWBQ6Y0fiznL6qllRGQSCTQ3t5ecbmJEydCVVX09rojE729vejp8SflPT09oZYfC4zvBPm5J9dEQYobDqPucilbbcFXUmey1UILY4WJE+djcsGGTQhefq1+5dL/csYhmDe5DXuGMvj4T1fj9d1Dgb6XK1jIsST+sGHEFlpoYXzBMAwsWbIEjzzyiHjPsiw88sgjWLZsme93li1b5loeAB566KGSy48FQs1QN9xwAwghrr/5853y93Q6jWuuuQYTJkxAW1sbLrrooiJ2uWXLFpx77rmIxWKYPHkyPve5zyGfz9fm11SJhBRG7JL9FsOQLbldUI0rEVtoYaxxuNYBAHhh61/rto2IruIb71sEQ1Pw9MY+nPnNv+Jz967FvuFM2e/xECIAaKxfZYtstdDCgYsVK1bgjjvuwN13340NGzbgE5/4BEZGRnDFFVcAAC6//HJXAv2//Mu/YOXKlfjv//5vvPzyy7jhhhvw7LPP4tprr23UTwivbB1++OHYuXOn+HviiSfEZ5/5zGfw+9//Hvfeey/+8pe/YMeOHXjPe94jPi8UCjj33HORzWbx5JNP4u6778Zdd92F66+/vja/pkrI1YgJIsWNzRBky+A5IASYsrDsoi200OxY1HUoAGDtfv8E1FrhqBmd+MMnT8aZC6fAsoF7V2/DJ372XFkDWh5CVBUCBeFytlpooYXxh/e///245ZZbcP3112Px4sVYs2YNVq5cKZLgt2zZgp07HV/AE088Effccw++//3vY9GiRfjVr36F++67D0cccUSjfkL4nC1N03zjngMDA/jhD3+Ie+65B6effjoA4M4778SCBQvw1FNP4YQTTsCf//xnrF+/Hg8//DCmTJmCxYsX46abbsK//du/4YYbboBhBM/ZqCUMTYNtKyDEQociJc6GUbY6ZgLHXE4fR2my2kILjcaimW8D+p7F2lw/bMsCUepHZg6ZksD3L1+KZzf14dIfPI1nNvbhkQ27sXzhFN/lubIV1VUQtJStFlp4K+Daa68tqUw9/vjjRe9dfPHFuPjii+s8quAIPUO99tprmDZtGg466CBceuml2LJlCwBg9erVyOVyWL7cMRidP38+Zs2aJbwwVq1ahSOPPNJVknnWWWdhcHAQL71U2s8nk8lgcHBQ/A0NBcvtCAWblj136VLCXhhlixDggu8Ab/t8jQfWQgtjj8MPvQCabWOPSrBz13Njss2lc7pxJeuh+NWVLyOb9++hyFv1RA0VsFpkq4UWWmh+hJqhjj/+eNx1111YuXIlbr/9dmzcuBGnnHIKhoaGsGvXLhiGgc7OTtd3ZG+LUt4X/LNSuPnmm9HR0SH+vP4ctYBpT4Zt6ThIrrwKo2y10MIBhGisG4fZVPhe+9oDY7bdT5x2MLrjBl7fPYxvPfyq7zKpHM3xjOoqYDOyFbI3YgsttNDCWCIU2TrnnHNw8cUX46ijjsJZZ52FP/7xj+jv78cvf/nLeo0PAHDddddhYGBA/K1fv77m2/jDxT/Dz8+5D5MTEhlska0W3sJYFKf+S09sr1+SvBftER1feTfNq7j9L2/g2U19RcukspRgxQyJbLWUrRZaaKGJMaoZqrOzE4ceeihef/119PT0IJvNor+/37WM7G1RyvuCf1YKpmmivb1d/CUStc+J6kl04YieWUBU8tMKE0ZsoYUDDKcdfB4A4He5Pbhn5TVjtt2zj5iK9y6ZAdsGbnxgPSzLnSzPm1BHDdVp/N4iWy200EITY1Qz1PDwMN544w1MnToVS5Ysga7rLm+LV155BVu2bBHeFsuWLcO6detcnbofeughtLe31yU0WBVkstVStlp4C2PZko/jms5FAICv7voLHvzbTWO27X8/Zz7ihooXtg3ggXU7XZ/xVj2uMGLA3ogttNBCC41AKLL12c9+Fn/5y1+wadMmPPnkk3j3u98NVVXxgQ98AB0dHbjqqquwYsUKPPbYY1i9ejWuuOIKLFu2DCeccAIA4Mwzz8TChQtx2WWXYe3atXjwwQfxhS98Addcc03zWPVHO53nLbLVwlsc/3T+j/H+yEzYhOC6N36BZ57/wZhsd2KbiY+/7WAAwFf/uAGD6Zz4jCfIu8OIB2jvpBZaaOGAQCiytW3bNnzgAx/AYYcdhve9732YMGECnnrqKUyaNAkA8M1vfhPnnXceLrroIpx66qno6enBb37zG/F9VVXxwAMPQFVVLFu2DB/60P/f3t3HNH3ncQB/F2hpEeShSjEnOJxON+eCQ6fFzSw+5jQ+bOz8Y7l5dzPZmJjd4pbczGDgcpmeGqNhWWaYIbLLhmNhLgFnpoBu87gJ2KIcWnDRczqKhJMnQZT1c38wG36zxYr99cG+X0kT++u3n+/v985H+u3Tr3/EunXr8N5773n3qO4H30YkctKEhWFz5kEsCYvFLY0Gf7Xuhq3ZNx+YX/9MKlISovBz1w3kHmzE4K8/En/71A96bTjg4NuIRBT4NDLS2QMD1OXLl5GcnIyffvoJEydO9G7x80eBf2YO/ftvF5WLL6IQNXCjC1kli1CnGcBURxjK/tLgk3nr/3sNf/joX3AIkDBGB/M0Bzr6evHvc3qsnT0R22/9A7BVACv3AOl/9sk+EdHoqfr4HcD4dPC3FJ/Z4slJiQAgUh+LXasOIEwELWEOtP5c75N50yfF4+9rZiIuSov/XR/AsZ4t+I8mHxExZ37zmS3+KSOiwHX/v+LsB45fT2Q4/PT8XtM1CHQ7AO0YoNX9ub+IQk8kJncATWE3UX68EL9/xvUZ3r1twe/CkPGn6ai92IacUx0AgPCoIlxqG4/LN3qH/r9e7QQuX/bJ/hDR6N1+3L79OB4qgvJtxNraWjz11FP+3g0iIiIahZMnT2LOnDn+3g2fCcrF1uDgICwWC0wmE8JU/M02X+rp6cFjjz2GpqYmVc4jFmqYp/cxU+9int7FPL1LrTwdDgfa2towa9YsREQE5ZtroxKUi60HUXd3N2JjY9HV1YWxY8fe/Q40IubpfczUu5indzFP72Ke3vVgvCxEREREFKC42CIiIiJSERdbASIyMhJ5eXmBcyb9IMc8vY+Zehfz9C7m6V3M07v4mS0iIiIiFfGVLSIiIiIVcbFFREREpCIutoiIiIhUxMUWERERkYq42HJj69atmDNnDmJiYpCYmIg1a9bAZrM5b7948SI0Go3LS2lpqdu6IoJ3330XEyZMgMFgwOLFi9HS0qIY09zcjNWrV2PcuHEYO3Ysnn76aVRXV991n0tLSzF9+nTo9XrMnDkThw4duue51RJseZaVlWH27NmIi4vDmDFjkJaWhk8++eSe51aLP/M8deoUlixZgri4OBiNRrzyyivo7e296z6zP72XZ6j2Z1lZGZYuXQqj0QiNRgOr1XrHmBs3biA7OxtGoxHR0dHIzMxEW1vbXfc5FPtTrTwDvT/9QsilZcuWSVFRkTQ2NorVapXly5dLSkqK9Pb2iojI4OCgtLa2Ki5btmyR6Oho6enpcVt327ZtEhsbKwcPHpSGhgZZtWqVpKamSn9/v3PM1KlTZfny5dLQ0CDNzc2yYcMGiYqKktbWVrd1T5w4IeHh4bJ9+3ZpamqSnJwc0Wq1cubMmXuaWy3Blmd1dbWUlZVJU1OTnD9/Xnbv3i3h4eFy+PDhe5pbLf7K88qVKxIfHy9ZWVly7tw5OXnypGRkZEhmZuaI+8v+9G6eodqfxcXFsmXLFiksLBQAYrFY7hiTlZUlycnJUllZKXV1dTJv3jzJyMgYcX9DtT/VyjPQ+9MfuNjy0NWrVwWAHD9+3O2YtLQ0efnll93e7nA4JCkpSXbs2OHc1tnZKZGRkfLZZ5+JiEh7e7sAkG+//dY5pru7WwDIkSNH3NZeu3atrFixQrFt7ty58uqrr3o8ty8Fep6uzJo1S3Jycjye25d8lefevXslMTFRfvnlF+eY06dPCwBpaWlxW5v9OcRbebryoPfncBcuXHC5OOjs7BStViulpaXObWfPnhUAUlNT47ZeKPbncN7O05VA7k9f4NuIHurq6gIAJCQkuLy9vr4eVqsV69evd1vjwoULsNvtWLx4sXNbbGws5s6di5qaGgCA0WjEtGnTUFxcjOvXr2NwcBB79+5FYmIi0tPTnfd76KGHkJ+f77xeU1OjqAsAy5Ytc9b1ZG5fCvQ8hxMRVFZWwmazYcGCBR7P7Uu+ynNgYAA6nU7xA/AGgwEA8P333zu3sT/VzXO4UOlPT9TX1+PWrVuK454+fTpSUlIUx83+9Mxo8xwuGPrTF7jY8oDD4cAbb7yB+fPn4/HHH3c5Zt++fXj00UeRkZHhto7dbgcAmEwmxXaTyeS8TaPR4OjRo7BYLIiJiYFer8euXbtw+PBhxMfHO+/z8MMPY9y4cYraI9X1ZG5fCYY8gaE/aNHR0dDpdFixYgUKCgqwZMkSj+f2FV/muXDhQtjtduzYsQM3b97EtWvX8PbbbwMAWltbnfdhf6qbJxB6/ekJu90OnU6HuLg4xfbfHjf70zOjzRMInv70lQh/70AwyM7ORmNjo+KZ5nD9/f349NNPkZube99ziQiys7ORmJiI7777DgaDAR9//DFWrlyJ2tpaTJgwAQBQWVl533P5S7DkGRMTA6vVit7eXlRWVmLTpk2YPHkynn322fveL2/yZZ4zZszA/v37sWnTJmzevBnh4eF4/fXXYTKZFK/OsD89cz95sj9Hj/3pXcHcn77CV7buYuPGjSgvL0d1dTUmTpzocswXX3yBvr4+rFu3bsRaSUlJAHDHNzna2tqct1VVVaG8vBwlJSWYP38+nnzySXz44YcwGAzYv3//iLVHquvJ3L4QLHkCQFhYGKZMmYK0tDS8+eabeOGFF7B161aP5/YFX+cJAC+++CLsdjuuXLmCjo4O5Ofno729HZMnTx6xNvsTzuv3mycQev3piaSkJNy8eROdnZ2K7Xc77lDsT0+MNk8gOPrTl7jYckNEsHHjRnz55ZeoqqpCamqq27H79u3DqlWrMH78+BFrpqamIikpSfEsoLu7Gz/88APMZjMAoK+vDwAUz2pvX3c4HG5rm83mO55dHDlyxFnXk7nVFGx5uuJwODAwMODx3GryV57DmUwmREdH48CBA9Dr9c63CFxhfw7xVp6uPOj96Yn09HRotVrFcdtsNly6dGnE4w7F/vTEaPN0JZD60y/89MH8gPfaa69JbGysHDt2TPF12r6+PsW4lpYW0Wg08vXXX3tUd9u2bRIXFydfffWVnD59WlavXq34umt7e7sYjUZ5/vnnxWq1is1mk7feeku0Wq1YrVZnnYULF0pBQYHz+okTJyQiIkJ27twpZ8+elby8PJdfXR5pbjUFW57vv/++fPPNN/Ljjz9KU1OT7Ny5UyIiIqSwsNDjudXkrzxFRAoKCqS+vl5sNpt88MEHYjAYZM+ePYo67M8hauUZqv3Z0dEhFotFKioqBICUlJSIxWJRnMYlKytLUlJSpKqqSurq6sRsNovZbFbUYX8OUSvPQO9Pf+Biyw0ALi9FRUWKcZs3b5bk5GTFV7dH4nA4JDc3V0wmk0RGRsqiRYvEZrMpxtTW1srSpUslISFBYmJiZN68eXLo0CHFmEmTJkleXp5i2+effy6PPPKI6HQ6mTFjhlRUVNzz3GoJtjzfeecdmTJliuj1eomPjxez2SwlJSX3PLda/JnnSy+9JAkJCaLT6eSJJ56Q4uLiO+qwP4eolWeo9mdRUZHLusOz6e/vlw0bNkh8fLxERUXJc889d8c59difQ9TKM9D70x80IiLef72MiIiIiAB+ZouIiIhIVVxsEREREamIiy0iIiIiFXGxRURERKQiLraIiIiIVMTFFhEREZGKuNgiIiIiUhEXW0REREQq4mKLiIiISEVcbBERERGpiIstIiIiIhVxsUVERESkov8D5hKtM5iwKk8AAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "transformed_data = await load_transformed_data(4, '4', 'CI-W3W01A3')\n", - "\n", - "display(transformed_data)\n", - "\n", - "ax1 = plt.subplot(1, 1, 1)\n", - "l1, = ax1.plot(transformed_data['timestamp'], transformed_data['previous_value'])\n", - "l2, = ax1.plot(transformed_data['timestamp'], transformed_data['previous_prediction'])\n", - "l3, = ax1.plot(transformed_data['timestamp'], transformed_data['selected_value'])\n", - "\n", - "ax1.set_xlim(\n", - " transformed_data['timestamp'].min(),\n", - " transformed_data['timestamp'].max()\n", - ")\n", - "\n", - "ax1.set_ylim(\n", - " transformed_data['previous_value'].min(),\n", - " transformed_data['previous_value'].max()\n", - ")\n", - "\n", - "ax2 = ax1.twinx()\n", - "l3, = ax2.plot(transformed_data['timestamp'], transformed_data['sensor_in'])\n", - "ax2.set_ylabel(\"sensor_in\")\n", - "\n", - "ax1.legend([l1, l2, l3], ['previous_value', 'previous_prediction', 'sensor_in'])\n", - "\n", - "plt.show()\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.14" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tests/helpers/status_engine.py b/tests/helpers/status_engine.py new file mode 100644 index 0000000..a859f91 --- /dev/null +++ b/tests/helpers/status_engine.py @@ -0,0 +1,174 @@ +"""A recording stand-in for the import log's SQLAlchemy engine. + +The import's status writes are the part of the feature with the strictest contract: which table they +name, which predicate they carry, which columns they touch, how many transactions they open and +which of them may fail without failing the import. Asserting that against a mock whose `execute` +returns a `MagicMock` proves nothing — the assertions need the statement text and the parameters. + +So this double records every statement it is handed, answers reads from a queue the test fills, and +lets a test make a write fail on a chosen attempt. It deliberately understands nothing about SQL +beyond the first word of the statement. +""" + +from __future__ import annotations + +from typing import Any + + +class RecordedStatement: + """One statement as the engine saw it: its normalised text and its bound parameters.""" + + def __init__(self, statement: Any, parameters: dict[str, Any] | None) -> None: + self.sql = ' '.join(str(statement).split()) + self.parameters: dict[str, Any] = dict(parameters or {}) + + @property + def verb(self) -> str: + return self.sql.split(' ', 1)[0].upper() + + @property + def is_write(self) -> bool: + return self.verb in ('UPDATE', 'INSERT', 'DELETE') + + @property + def table(self) -> str: + """The relation the statement names, for the "one table only" assertions.""" + words = self.sql.split() + if self.verb == 'UPDATE': + return words[1] + for index, word in enumerate(words): + if word.upper() in ('FROM', 'INTO') and index + 1 < len(words): + return words[index + 1] + return '' + + @property + def assignments(self) -> set[str]: + """The column names a `SET` clause assigns, so a test can assert what is never written.""" + upper = self.sql.upper() + if ' SET ' not in upper: + return set() + body = self.sql[upper.index(' SET ') + 5 :] + where = body.upper().find(' WHERE ') + if where != -1: + body = body[:where] + return {part.split('=')[0].strip() for part in body.split(',') if '=' in part} + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return f'RecordedStatement({self.sql!r}, {self.parameters!r})' + + +class FakeResult: + """What `Connection.execute` returns: a rowcount for writes, mappings for reads.""" + + def __init__(self, rowcount: int = 1, rows: list[dict[str, Any]] | None = None) -> None: + self.rowcount = rowcount + self._rows = rows or [] + + def mappings(self) -> FakeResult: + return self + + def first(self) -> dict[str, Any] | None: + return self._rows[0] if self._rows else None + + def all(self) -> list[dict[str, Any]]: + return list(self._rows) + + +class FakeConnection: + """A connection that records and then answers from the engine's queues.""" + + def __init__(self, engine: FakeStatusEngine) -> None: + self._engine = engine + + def execute(self, statement: Any, parameters: dict[str, Any] | None = None) -> FakeResult: + recorded = RecordedStatement(statement, parameters) + self._engine.statements.append(recorded) + if recorded.is_write: + return self._engine._answer_write() + return self._engine._answer_read() + + +class _Scope: + """The context manager `engine.begin()` and `engine.connect()` both hand out.""" + + def __init__(self, engine: FakeStatusEngine, transactional: bool) -> None: + self._engine = engine + self._transactional = transactional + + def __enter__(self) -> FakeConnection: + self._engine.open_scopes += 1 + if self._transactional: + self._engine.transactions += 1 + else: + self._engine.connections += 1 + return FakeConnection(self._engine) + + def __exit__(self, *_exc: object) -> bool: + self._engine.open_scopes -= 1 + return False + + +class FakeStatusEngine: + """An engine double that records statements and can be told to fail a write. + + Attributes: + statements: every statement executed, in order. + rowcounts: rowcounts for writes, consumed in order; the last one repeats. + rows: rows for reads, consumed in order; the last one repeats. `None` means "no such row". + write_errors: exceptions to raise instead of executing a write, consumed in order. A `None` + entry means "this attempt succeeds", which is how a retry is modelled. + transactions: how many `begin()` scopes were opened — one per write, or the writes are + sharing a transaction. + open_scopes: scopes currently open, so a test can assert none is left open. + """ + + def __init__( + self, + rowcounts: list[int] | None = None, + rows: list[dict[str, Any] | None] | None = None, + write_errors: list[BaseException | None] | None = None, + ) -> None: + self.statements: list[RecordedStatement] = [] + self.rowcounts: list[int] = list(rowcounts or []) + self.rows: list[dict[str, Any] | None] = list(rows or []) + self.write_errors: list[BaseException | None] = list(write_errors or []) + self.transactions = 0 + self.connections = 0 + self.open_scopes = 0 + self.disposed = False + + # ------------------------------------------------------------------ engine API + def begin(self) -> _Scope: + return _Scope(self, transactional=True) + + def connect(self) -> _Scope: + return _Scope(self, transactional=False) + + def dispose(self) -> None: + self.disposed = True + + # ------------------------------------------------------------------- answering + def _answer_write(self) -> FakeResult: + if self.write_errors: + error = self.write_errors.pop(0) + if error is not None: + raise error + rowcount = self.rowcounts.pop(0) if len(self.rowcounts) > 1 else (self.rowcounts or [1])[0] + return FakeResult(rowcount=rowcount) + + def _answer_read(self) -> FakeResult: + row = self.rows.pop(0) if len(self.rows) > 1 else (self.rows or [None])[0] + return FakeResult(rows=[row] if row is not None else []) + + # ---------------------------------------------------------------- convenience + @property + def writes(self) -> list[RecordedStatement]: + return [statement for statement in self.statements if statement.is_write] + + @property + def reads(self) -> list[RecordedStatement]: + return [statement for statement in self.statements if not statement.is_write] + + @property + def tables(self) -> set[str]: + return {statement.table for statement in self.statements if statement.table} diff --git a/tests/laborious/activities/test_activities.py b/tests/laborious/activities/test_activities.py index 54fdd89..dd07c1f 100644 --- a/tests/laborious/activities/test_activities.py +++ b/tests/laborious/activities/test_activities.py @@ -156,6 +156,121 @@ def test___init__( ) +BASE_CONFIGS = { + 'postgres_config': { + 'host': 'localhost', + 'port': 5432, + 'user': 'postgres', + 'password': 'postgres', + 'dbname': 'postgres', + 'min_connections': 1, + 'max_connections': 10, + }, + 'minio_config': { + 'endpoint_url': 'localhost:9000', + 'access_key': 'minio', + 'secret_key': 'minio123', + 'default_bucket': 'test', + 'retention_hours': 24, + 'secure': False, + }, + 'mlflow_config': {'host': 'localhost', 'port': 5000, 'username': 'm', 'password': 'm'}, + 'opc_config': {}, + 'pi_web_api_config': {'base_url': 'https://pi', 'auth_type': 'bearer', 'auth_token': 't'}, +} + +IMPORT_CONFIG = {'bucket': 'sientia', 'prefix': 'imported_models/'} +IMPORT_STATUS_CONFIG = { + 'host': 'bff-db', + 'port': 5432, + 'dbname': 'sientia-core-mlops-bff', + 'user': 'u', + 'password': 'p', + 'schema': 'public', +} + + +@patch('laborious.activities.activities.ModelImport.__init__') +@patch('laborious.activities.activities.Storage.__init__') +@patch('laborious.activities.activities.MLFlow.__init__') +@patch('laborious.activities.activities.OPC.__init__') +@patch('laborious.activities.activities.Gates.__init__') +@patch('laborious.activities.activities.ModelMetrics.__init__') +@patch('laborious.activities.activities.API.__init__') +@patch('laborious.activities.activities.MongoDBRepository') +@patch('laborious.activities.activities.MinioRepository') +@patch('laborious.activities.activities.MetricsController') +def test_the_import_activities_share_the_worker_s_mlflow_repository( # noqa: PLR0913 + _mock_metrics_controller, + mock_minio_repository, + mock_mongo_repository, + _mock_api_init, + _mock_model_metrics_init, + _mock_gates_init, + _mock_opc_init, + mock_mlflow_init, + _mock_storage_init, + mock_model_import_init, +): + """Setting the MLflow tracking URI is a process-wide side effect, so two repository instances + would be two chances to disagree about where MLflow is.""" + repository = MagicMock(name='model_monitoring_repository') + mock_mlflow_init.side_effect = lambda instance, **_: setattr( + instance, 'model_monitoring_repository', repository + ) + + Activities( + **BASE_CONFIGS, + logger=MagicMock(), + notification_handler=MagicMock(), + mongo_config={'connection_string': 'mongodb://mongo', 'database_name': 'sientia'}, + import_config=IMPORT_CONFIG, + import_status_config=IMPORT_STATUS_CONFIG, + ) + + passed = mock_model_import_init.call_args.kwargs + assert passed['mlflow_repository'] is repository + assert passed['minio_repository'] is mock_minio_repository.return_value + assert passed['mongo_repository'] is mock_mongo_repository.return_value + assert passed['import_config'] is IMPORT_CONFIG + assert passed['status_config'] is IMPORT_STATUS_CONFIG + + +@patch('laborious.activities.activities.ModelImport.__init__') +@patch('laborious.activities.activities.Storage.__init__') +@patch('laborious.activities.activities.MLFlow.__init__') +@patch('laborious.activities.activities.OPC.__init__') +@patch('laborious.activities.activities.Gates.__init__') +@patch('laborious.activities.activities.ModelMetrics.__init__') +@patch('laborious.activities.activities.API.__init__') +@patch('laborious.activities.activities.MongoDBRepository') +@patch('laborious.activities.activities.MinioRepository') +@patch('laborious.activities.activities.MetricsController') +def test_without_a_mongo_config_the_import_gets_no_repository( # noqa: PLR0913 + _mock_metrics_controller, + _mock_minio_repository, + mock_mongo_repository, + _mock_api_init, + _mock_model_metrics_init, + _mock_gates_init, + _mock_opc_init, + _mock_mlflow_init, + _mock_storage_init, + mock_model_import_init, +): + """It refuses rather than guessing: the notification handler keeps its own client either way.""" + Activities( + **BASE_CONFIGS, + logger=MagicMock(), + notification_handler=MagicMock(), + import_config=IMPORT_CONFIG, + import_status_config=IMPORT_STATUS_CONFIG, + ) + + assert mock_model_import_init.call_args.kwargs['mongo_repository'] is None + mock_mongo_repository.assert_not_called() + + @mark.asyncio @patch('laborious.activities.activities.Storage') @patch('laborious.activities.activities.MLFlow') diff --git a/tests/laborious/activities/test_model_import_bundle.py b/tests/laborious/activities/test_model_import_bundle.py new file mode 100644 index 0000000..5a95bb5 --- /dev/null +++ b/tests/laborious/activities/test_model_import_bundle.py @@ -0,0 +1,565 @@ +"""Downloading the object and opening the bundle: the two activities that touch the secret. + +Three things are asserted here: + +- **the digest is enforced against the bytes that arrived**, before the header is parsed, with no + path that skips the comparison — a mismatch means the object behind that key is not the file the + user chose, and no amount of well-formedness makes it the right file; +- **the envelope is opened inside the activity that uses the password**, so a plaintext password is + never an activity input, an activity result, or a value the workflow holds; +- **nothing observable carries the secret** — for every failure path, the password, the envelope key + and the derived key are absent from the log, the notification, the metric labels, the exception and + the return value. + +The bundles here are real: the shared factory writes them field for field like the producer, at the +test cost parameters, and the production reader opens them with real Argon2id and real +`secretstream`. +""" + +import hashlib +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +from pytest import fixture, mark, raises + +from laborious import metrics +from laborious.activities.model_import import ModelImport +from laborious.activities.model_import_errors import ImportGateRejectedError +from laborious.utils.bundle.steps import ImportStep +from tests.helpers import bundle_factory as factory +from tests.helpers.status_engine import FakeStatusEngine + +# Distinctive enough that a substring search for it in a log line cannot match by accident. +BUNDLE_PASSWORD = 'Zk9-bundle-secret-Passw0rd' +ENVELOPE_KEY = bytes(range(100, 132)) +OTHER_ENVELOPE_KEY = bytes(range(200, 232)) + +METADATA = { + 'model_id': '77', + 'model_name': 'factory_model', + 'workflow_name': 'import_model', + 'schedule_name': '-', +} +WORKFLOW_ID = 'import_model-77-abc' +OBJECT_KEY = 'imported_models/factory_model_v2.sientia' + + +def import_config(work_dir: Path, **overrides) -> dict: + return { + 'password_key': factory.envelope_key_b64(ENVELOPE_KEY), + 'bucket': 'sientia', + 'prefix': 'imported_models/', + 'work_dir': str(work_dir), + 'max_object_bytes': 8 * 1024 * 1024, + 'max_entries': 5000, + 'max_uncompressed_bytes': 4 * 1024 * 1024, + 'max_compression_ratio': 200.0, + 'models_collection': 'models', + 'retention_days': 7, + **overrides, + } + + +@fixture +def minio_client() -> MagicMock: + client = MagicMock() + client.stat_object.return_value = MagicMock(size=1024) + return client + + +@fixture +def import_activity(tmp_path: Path, minio_client: MagicMock) -> ModelImport: + repository = MagicMock() + repository.minio_client = minio_client + with patch('laborious.activities.model_import.create_engine', return_value=FakeStatusEngine()): + activity = ModelImport( + status_config={ + 'host': 'h', + 'port': 5432, + 'dbname': 'sientia-core-mlops-bff', + 'user': 'u', + 'password': 'p', + 'schema': 'public', + }, + import_config=import_config(tmp_path), + minio_repository=repository, + mlflow_repository=MagicMock(), + mongo_repository=AsyncMock(), + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=AsyncMock(), + ) + activity.emit_metric = AsyncMock() + activity.send_notification_async = AsyncMock() + return activity + + +def serve_object(minio_client: MagicMock, payload: bytes) -> None: + """Make `fget_object` write `payload` where the activity asked for it.""" + + def write(_bucket, _key, destination): + Path(destination).write_bytes(payload) + + minio_client.stat_object.return_value = MagicMock(size=len(payload)) + minio_client.fget_object.side_effect = write + + +def download_input(digest: str, **overrides) -> dict: + return { + 'metadata': METADATA, + 'import_run_id': 77, + 'workflow_id': WORKFLOW_ID, + 'object_key': OBJECT_KEY, + 'expected_digest': digest, + **overrides, + } + + +def open_input(path: str, envelope: str | None = None, **overrides) -> dict: + return { + 'metadata': METADATA, + 'import_run_id': 77, + 'workflow_id': WORKFLOW_ID, + 'path': path, + 'password_envelope': envelope + if envelope is not None + else factory.build_envelope(BUNDLE_PASSWORD, ENVELOPE_KEY), + **overrides, + } + + +def observable_text(activity: ModelImport) -> str: + """Everything this activity said out loud: logs, notifications and metric labels.""" + logger = activity.logger + parts = [ + str(call) for call in logger.custom_info.call_args_list + logger.custom_error.call_args_list + ] + parts += [str(call) for call in activity.send_notification_async.await_args_list] + parts += [str(call) for call in activity.emit_metric.await_args_list] + return '\n'.join(parts) + + +# ------------------------------------------------------------------------- the download + + +@mark.asyncio +async def test_a_matching_digest_yields_the_downloaded_file( + import_activity: ModelImport, minio_client: MagicMock, tmp_path: Path +): + payload = factory.make_bundle(password=BUNDLE_PASSWORD) + serve_object(minio_client, payload) + digest = hashlib.sha256(payload).hexdigest() + + result = await import_activity.download_import_bundle(download_input(digest)) + + assert Path(result['path']).read_bytes() == payload + assert result['digest'] == digest + assert result['size'] == len(payload) + assert Path(result['path']).parent == tmp_path / WORKFLOW_ID + + +@mark.asyncio +async def test_the_configured_bucket_is_used_when_the_request_names_none( + import_activity: ModelImport, minio_client: MagicMock +): + payload = factory.make_bundle(password=BUNDLE_PASSWORD) + serve_object(minio_client, payload) + + await import_activity.download_import_bundle( + download_input(hashlib.sha256(payload).hexdigest()) + ) + + assert minio_client.stat_object.call_args.args[0] == 'sientia' + + +@mark.asyncio +async def test_the_request_may_name_another_bucket( + import_activity: ModelImport, minio_client: MagicMock +): + payload = factory.make_bundle(password=BUNDLE_PASSWORD) + serve_object(minio_client, payload) + + await import_activity.download_import_bundle( + download_input(hashlib.sha256(payload).hexdigest(), bucket='other-bucket') + ) + + assert minio_client.stat_object.call_args.args[0] == 'other-bucket' + + +@mark.asyncio +async def test_a_digest_from_another_file_fails_before_the_header_is_parsed( + import_activity: ModelImport, minio_client: MagicMock +): + payload = factory.make_bundle(password=BUNDLE_PASSWORD) + other = factory.make_bundle(password=BUNDLE_PASSWORD, salt=b'\x01' * 16) + serve_object(minio_client, payload) + + with raises(ImportGateRejectedError) as failure: + await import_activity.download_import_bundle( + download_input(hashlib.sha256(other).hexdigest()) + ) + + assert failure.value.details[0]['step'] == ImportStep.DOWNLOAD.value + assert failure.value.details[0]['gate'] == 1 + + +@mark.asyncio +async def test_a_file_that_does_not_match_is_not_left_on_the_worker( + import_activity: ModelImport, minio_client: MagicMock, tmp_path: Path +): + serve_object(minio_client, factory.make_bundle(password=BUNDLE_PASSWORD)) + + with raises(ImportGateRejectedError): + await import_activity.download_import_bundle(download_input('0' * 64)) + + assert not (tmp_path / WORKFLOW_ID / 'bundle.sientia').exists() + + +@mark.asyncio +async def test_the_digest_comparison_is_case_insensitive_on_the_input( + import_activity: ModelImport, minio_client: MagicMock +): + payload = factory.make_bundle(password=BUNDLE_PASSWORD) + serve_object(minio_client, payload) + + result = await import_activity.download_import_bundle( + download_input(hashlib.sha256(payload).hexdigest().upper()) + ) + + assert result['digest'] == hashlib.sha256(payload).hexdigest() + + +@mark.asyncio +async def test_an_object_over_the_ceiling_is_refused_before_its_body_is_fetched( + import_activity: ModelImport, minio_client: MagicMock +): + minio_client.stat_object.return_value = MagicMock(size=9 * 1024 * 1024) + + with raises(ImportGateRejectedError) as failure: + await import_activity.download_import_bundle(download_input('a' * 64)) + + assert failure.value.details[0]['step'] == ImportStep.DOWNLOAD.value + minio_client.fget_object.assert_not_called() + + +@mark.asyncio +@mark.parametrize( + 'object_key', + ['other_prefix/model.sientia', 'imported_models/model.zip'], +) +async def test_a_key_of_the_wrong_shape_is_refused( + import_activity: ModelImport, minio_client: MagicMock, object_key: str +): + with raises(ImportGateRejectedError): + await import_activity.download_import_bundle( + download_input('a' * 64, object_key=object_key) + ) + + minio_client.fget_object.assert_not_called() + + +@mark.asyncio +async def test_an_unreadable_object_is_a_download_failure( + import_activity: ModelImport, minio_client: MagicMock +): + minio_client.stat_object.side_effect = OSError('no such object') + + with raises(ImportGateRejectedError) as failure: + await import_activity.download_import_bundle(download_input('a' * 64)) + + assert failure.value.details[0]['step'] == ImportStep.DOWNLOAD.value + + +@mark.asyncio +async def test_a_failed_transfer_is_a_download_failure( + import_activity: ModelImport, minio_client: MagicMock +): + minio_client.fget_object.side_effect = OSError('connection reset') + + with raises(ImportGateRejectedError) as failure: + await import_activity.download_import_bundle(download_input('a' * 64)) + + assert failure.value.details[0]['step'] == ImportStep.DOWNLOAD.value + + +@mark.asyncio +async def test_a_missing_minio_repository_is_a_configuration_error(tmp_path: Path): + with patch('laborious.activities.model_import.create_engine', return_value=FakeStatusEngine()): + activity = ModelImport( + status_config={ + 'host': 'h', + 'port': 5432, + 'dbname': 'd', + 'user': 'u', + 'password': 'p', + 'schema': 'public', + }, + import_config=import_config(tmp_path), + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=AsyncMock(), + ) + + with raises(ValueError, match='Minio repository not initialized'): + await activity.download_import_bundle(download_input('a' * 64)) + + +# ----------------------------------------------------------------------- opening the bundle + + +@mark.asyncio +async def test_a_well_formed_bundle_opens_and_returns_nothing_secret( + import_activity: ModelImport, tmp_path: Path +): + path = tmp_path / 'bundle.sientia' + path.write_bytes(factory.make_bundle(password=BUNDLE_PASSWORD)) + + result = await import_activity.open_import_bundle(open_input(str(path))) + + assert result['metadata']['model_name'] == 'factory_model' + assert result['parameters']['target_variable'] == 'SE' + assert Path(result['extracted_dir']).is_dir() + assert (Path(result['extracted_dir']) / 'artifacts' / 'prediction_model' / 'MLmodel').is_file() + assert set(result) == {'extracted_dir', 'metadata', 'parameters', 'digest'} + assert BUNDLE_PASSWORD not in str(result) + + +@mark.asyncio +async def test_opening_a_bundle_measures_the_kdf(import_activity: ModelImport, tmp_path: Path): + path = tmp_path / 'bundle.sientia' + path.write_bytes(factory.make_bundle(password=BUNDLE_PASSWORD)) + + await import_activity.open_import_bundle(open_input(str(path))) + + observed = [ + call.kwargs + for call in import_activity.emit_metric.await_args_list + if call.kwargs.get('metric_object') is metrics.MODEL_IMPORT_KDF_DURATION + ] + assert len(observed) == 1 + assert observed[0]['method'] == 'observe' + assert observed[0]['value'] > 0 + + +@mark.asyncio +async def test_a_wrong_envelope_key_is_not_confused_with_a_wrong_bundle_password( + import_activity: ModelImport, tmp_path: Path +): + path = tmp_path / 'bundle.sientia' + path.write_bytes(factory.make_bundle(password=BUNDLE_PASSWORD)) + envelope = factory.build_envelope(BUNDLE_PASSWORD, OTHER_ENVELOPE_KEY) + + with raises(ImportGateRejectedError) as failure: + await import_activity.open_import_bundle(open_input(str(path), envelope)) + + details = failure.value.details[0] + assert details['step'] == ImportStep.DECRYPTION.value + assert details['gate'] == 3 + # The platform's own transport failed, not the user's password. Same step — the step is where the + # import stopped, not who is at fault — but a distinguishable cause. + assert details['cause'] == 'envelope' + + +@mark.asyncio +async def test_a_wrong_bundle_password_is_a_decryption_failure_of_its_own( + import_activity: ModelImport, tmp_path: Path +): + path = tmp_path / 'bundle.sientia' + path.write_bytes(factory.make_bundle(password='a-different-password')) + + with raises(ImportGateRejectedError) as failure: + await import_activity.open_import_bundle(open_input(str(path))) + + details = failure.value.details[0] + assert details['step'] == ImportStep.DECRYPTION.value + assert details['cause'] != 'envelope' + + +@mark.asyncio +async def test_a_missing_password_key_stops_the_import_at_decryption( + import_activity: ModelImport, tmp_path: Path +): + import_activity.import_config['password_key'] = None + path = tmp_path / 'bundle.sientia' + path.write_bytes(factory.make_bundle(password=BUNDLE_PASSWORD)) + + with raises(ImportGateRejectedError) as failure: + await import_activity.open_import_bundle(open_input(str(path))) + + assert failure.value.details[0]['cause'] == 'envelope' + + +@mark.asyncio +@mark.parametrize( + 'envelope', + ['', 'not-base64!!', 'AAAA', 'bm90LWFuLWVudmVsb3BlLWF0LWFsbA=='], + ids=['empty', 'not-base64', 'too-short', 'wrong-bytes'], +) +async def test_a_tampered_or_malformed_envelope_is_refused( + import_activity: ModelImport, tmp_path: Path, envelope: str +): + path = tmp_path / 'bundle.sientia' + path.write_bytes(factory.make_bundle(password=BUNDLE_PASSWORD)) + + with raises(ImportGateRejectedError) as failure: + await import_activity.open_import_bundle(open_input(str(path), envelope)) + + assert failure.value.details[0]['cause'] == 'envelope' + + +def bad_magic() -> bytes: + return factory.make_bundle(password=BUNDLE_PASSWORD, magic=b'NOTSIEN1') + + +def wrong_digest() -> bytes: + return factory.make_bundle(password=BUNDLE_PASSWORD, digest_override=bytes(32)) + + +def tampered() -> bytes: + return factory.tamper_ciphertext(factory.make_bundle(password=BUNDLE_PASSWORD)) + + +def traversal() -> bytes: + return factory.make_bundle( + factory.ZipSpec(traversal_entry='artifacts/../../escaped.csv'), password=BUNDLE_PASSWORD + ) + + +def absolute() -> bytes: + return factory.make_bundle( + factory.ZipSpec(absolute_entry='/etc/passwd'), password=BUNDLE_PASSWORD + ) + + +def missing_data_model() -> bytes: + files = { + name: content + for name, content in factory.default_tree().items() + if not name.startswith('artifacts/data_model/') + } + return factory.make_bundle(factory.ZipSpec(files=files), password=BUNDLE_PASSWORD) + + +def missing_target_variable() -> bytes: + document = factory.build_metadata_document(drop_parameter='target_variable') + return factory.make_bundle(factory.ZipSpec(document=document), password=BUNDLE_PASSWORD) + + +def script_in_the_tree() -> bytes: + return factory.make_bundle( + factory.ZipSpec(extra_files={'artifacts/run.sh': b'#!/bin/sh\nrm -rf /\n'}), + password=BUNDLE_PASSWORD, + ) + + +@mark.asyncio +@mark.parametrize( + ('build', 'step', 'gate'), + [ + (bad_magic, ImportStep.DOWNLOAD, 2), + (wrong_digest, ImportStep.DOWNLOAD, 2), + (tampered, ImportStep.DECRYPTION, 3), + (traversal, ImportStep.ARCHIVE_INSPECTION, 4), + (absolute, ImportStep.ARCHIVE_INSPECTION, 4), + (missing_data_model, ImportStep.STRUCTURE_VALIDATION, 6), + (missing_target_variable, ImportStep.STRUCTURE_VALIDATION, 6), + (script_in_the_tree, ImportStep.CONTENT_POLICY, 7), + ], + ids=lambda value: getattr(value, '__name__', str(value)), +) +async def test_a_gate_failure_reaches_the_activity_as_its_own_step( + import_activity: ModelImport, tmp_path: Path, build, step: ImportStep, gate: int +): + path = tmp_path / 'bundle.sientia' + path.write_bytes(build()) + + with raises(ImportGateRejectedError) as failure: + await import_activity.open_import_bundle(open_input(str(path))) + + assert failure.value.details[0]['step'] == step.value + assert failure.value.details[0]['gate'] == gate + + +@mark.asyncio +async def test_a_rejected_bundle_leaves_no_extracted_tree( + import_activity: ModelImport, tmp_path: Path +): + path = tmp_path / 'bundle.sientia' + path.write_bytes(traversal()) + + with raises(ImportGateRejectedError): + await import_activity.open_import_bundle(open_input(str(path))) + + extract_root = tmp_path / WORKFLOW_ID / 'extract' + assert not extract_root.exists() or list(extract_root.rglob('*')) == [] + + +# ---------------------------------------------------------------------------- secrecy + + +SECRET_PATHS = [ + ('happy', lambda: factory.make_bundle(password=BUNDLE_PASSWORD), None), + ('wrong-bundle-password', lambda: factory.make_bundle(password='another-password'), None), + ('wrong-envelope-key', lambda: factory.make_bundle(password=BUNDLE_PASSWORD), 'other-key'), + ('malformed-envelope', lambda: factory.make_bundle(password=BUNDLE_PASSWORD), 'garbage'), + ('gate-2', bad_magic, None), + ('gate-3', tampered, None), + ('gate-4', traversal, None), + ('gate-6', missing_data_model, None), + ('gate-7', script_in_the_tree, None), +] + + +@mark.asyncio +@mark.parametrize( + ('build', 'envelope_kind'), + [(build, kind) for _, build, kind in SECRET_PATHS], + ids=[name for name, _, _ in SECRET_PATHS], +) +async def test_no_failure_path_says_the_password_out_loud( + import_activity: ModelImport, tmp_path: Path, build, envelope_kind: str | None +): + """Every path through this activity, checked against the three things that must never surface.""" + path = tmp_path / 'bundle.sientia' + path.write_bytes(build()) + if envelope_kind == 'other-key': + envelope = factory.build_envelope(BUNDLE_PASSWORD, OTHER_ENVELOPE_KEY) + elif envelope_kind == 'garbage': + envelope = 'not-base64!!' + else: + envelope = factory.build_envelope(BUNDLE_PASSWORD, ENVELOPE_KEY) + + raised: BaseException | None = None + result = None + try: + result = await import_activity.open_import_bundle(open_input(str(path), envelope)) + except BaseException as error: # noqa: BLE001 - the point is that *whatever* it is says nothing + raised = error + + secrets = [ + BUNDLE_PASSWORD, + 'another-password', + factory.envelope_key_b64(ENVELOPE_KEY), + ENVELOPE_KEY.hex(), + ] + surfaces = [observable_text(import_activity), str(result)] + if raised is not None: + surfaces += [str(raised), str(getattr(raised, 'details', '')), repr(raised)] + for secret in secrets: + for surface in surfaces: + assert secret not in surface + + +@mark.asyncio +async def test_the_envelope_never_leaves_the_activity_that_opens_it( + import_activity: ModelImport, tmp_path: Path +): + """The result is what crosses back into the workflow, and therefore into the event history.""" + path = tmp_path / 'bundle.sientia' + path.write_bytes(factory.make_bundle(password=BUNDLE_PASSWORD)) + envelope = factory.build_envelope(BUNDLE_PASSWORD, ENVELOPE_KEY) + + result = await import_activity.open_import_bundle(open_input(str(path), envelope)) + + assert envelope not in str(result) + assert 'password' not in str(result).lower() diff --git a/tests/laborious/activities/test_model_import_errors.py b/tests/laborious/activities/test_model_import_errors.py new file mode 100644 index 0000000..910563b --- /dev/null +++ b/tests/laborious/activities/test_model_import_errors.py @@ -0,0 +1,410 @@ +"""The failure catalog and the one channel every import failure passes through. + +Two contracts are asserted here, and both are about what a person is allowed to see: + +- **the catalog is closed and total** — every pipeline step that can fail has exactly one code and + one sentence, the sentence never carries a path, a table name, an exception class or a traceback, + and the classification is a function of the *step* alone, so rewording a sentence cannot move a + code the frontend switches on; +- **the channel is the only reporter** — one call reaches the log, the metric and the notification, + the exception's text reaches nothing but the log, and no other module in the import path logs an + error or notifies (asserted from the source, so the rule cannot rot). +""" + +import ast +from pathlib import Path +from typing import Any + +from pytest import mark, raises + +from laborious import metrics +from laborious.activities.model_import_errors import ( + IMPORT_FAILURE_CATALOG, + STEPS_WITHOUT_A_USER_FACING_REASON, + UNCLASSIFIED_FAILURE, + ImportErrorCode, + ImportGateRejectedError, + ImportInputError, + ImportLogRowNotClaimableError, + ImportStatusRejectedError, + classify_import_failure, + failure_location, + report_import_failure, +) +from laborious.utils.bundle.steps import GATE_TO_STEP, ImportStep + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] + +# The modules that make up the import path. The channel lives in the third one; the other two may +# not report a failure of their own. +IMPORT_MODULES = ( + REPOSITORY_ROOT / 'laborious/activities/model_import.py', + REPOSITORY_ROOT / 'laborious/workflows/import_model.py', +) + +# What a leaked exception message looks like: a work directory, the table, an exception class and a +# traceback line. None of it may reach a recorded reason. +LEAKY_MESSAGE = ( + "KeyError: 'target_variable' while writing public.experiment_run from " + '/tmp/sientia-import/import_model-77-abc/extract\n' # noqa: S108 - the leak under test + ' File "laborious/activities/model_import.py", line 1, in write_import_model_document' +) + +# Every fragment of the leaky message that must not reach a recorded reason. +LEAKS_THAT_MUST_NOT_SURFACE = ( + 'sientia-import', + 'public.experiment_run', + 'KeyError', + 'File "', + 'line 1', +) + +STEPS_THAT_ONLY_SUPPORT_CAN_ACT_ON = ( + ImportStep.EXPERIMENT_CREATION, + ImportStep.ARTIFACT_UPLOAD, + ImportStep.REGISTRATION, + ImportStep.MODEL_DOCUMENT, +) + + +class FakeSinks: + """The observability surface the channel writes to, recorded rather than performed.""" + + def __init__(self) -> None: + self.logged: list[tuple[str, dict]] = [] + self.metrics: list[tuple[Any, dict]] = [] + self.notifications: list[dict] = [] + + def error(self, message: str, metadata: dict | None = None) -> None: + self.logged.append((message, metadata or {})) + + def get_core_labels(self, metadata: dict | None = None, operation_type: str = '-') -> dict: + return { + 'model_name': (metadata or {}).get('model_name', '-'), + 'operation_type': operation_type, + } + + async def emit_metric( + self, + metric_object: Any, + tags: dict[str, Any], + method: str = 'inc', + value: float | None = None, + ) -> None: + self.metrics.append((metric_object, tags)) + + async def send_notification_async( + self, + metadata: dict, + notification_id: str, + message: str, + block: str, + level: str = 'INFO', + attachment_content: str | None = None, + ) -> None: + self.notifications.append( + { + 'metadata': metadata, + 'notification_id': notification_id, + 'message': message, + 'block': block, + 'level': level, + } + ) + + +# ------------------------------------------------------------------------- the catalog + + +def test_every_failing_step_has_exactly_one_entry(): + """A step added without a sentence fails here rather than reaching a user as `None`.""" + expected = set(ImportStep) - STEPS_WITHOUT_A_USER_FACING_REASON + assert set(IMPORT_FAILURE_CATALOG) == expected + + +def test_cleanup_is_the_only_step_without_a_sentence(): + """Cleanup failing is a log line on a successful import, never the reported failure.""" + assert STEPS_WITHOUT_A_USER_FACING_REASON == frozenset({ImportStep.CLEANUP}) + assert ImportStep.CLEANUP not in IMPORT_FAILURE_CATALOG + + +def test_every_gate_maps_to_a_step_that_has_a_sentence(): + for step in GATE_TO_STEP.values(): + assert step in IMPORT_FAILURE_CATALOG + + +def test_each_step_has_its_own_code(): + codes = [code for code, _ in IMPORT_FAILURE_CATALOG.values()] + assert len(codes) == len(set(codes)) + + +def test_the_codes_that_never_reach_a_record_are_absent_from_the_catalog(): + catalogued = {code for code, _ in IMPORT_FAILURE_CATALOG.values()} + for code in ( + ImportErrorCode.LOG_ROW_NOT_CLAIMABLE, + ImportErrorCode.STATUS_NOT_RECORDED, + ImportErrorCode.CLEANUP_INCOMPLETE, + ): + assert code not in catalogued + + +@mark.parametrize('step', list(IMPORT_FAILURE_CATALOG)) +def test_a_reason_never_carries_the_exception(step: ImportStep): + failure = classify_import_failure(step, RuntimeError(LEAKY_MESSAGE)) + + for leak in LEAKS_THAT_MUST_NOT_SURFACE: + assert leak not in failure.reason + assert failure.reason == IMPORT_FAILURE_CATALOG[step][1] + + +@mark.parametrize('step', list(IMPORT_FAILURE_CATALOG)) +def test_the_classification_is_a_function_of_the_step_alone(step: ImportStep): + """Which is what makes a reworded sentence keep its code: neither is read from the exception.""" + first = classify_import_failure(step, RuntimeError(LEAKY_MESSAGE)) + second = classify_import_failure(step, KeyError('target_variable')) + third = classify_import_failure(step, None) + + assert first == second == third + assert first.code is IMPORT_FAILURE_CATALOG[step][0] + + +def test_the_decryption_reason_does_not_say_which_of_the_two_it_was(): + reason = IMPORT_FAILURE_CATALOG[ImportStep.DECRYPTION][1] + + # Both causes are named as possibilities; neither is asserted, because the AEAD cannot tell them + # apart and guessing would be a hint about somebody else's password. + assert 'password is wrong' in reason + assert 'damaged' in reason + assert 'altered' in reason + + +@mark.parametrize('step', STEPS_THAT_ONLY_SUPPORT_CAN_ACT_ON) +def test_a_reason_the_user_cannot_act_on_names_the_import_identifier(step: ImportStep): + assert "Contact support with this import's identifier." in IMPORT_FAILURE_CATALOG[step][1] + + +def test_a_failure_with_no_step_is_unclassified(): + failure = classify_import_failure(None, RuntimeError('boom')) + + assert failure.step is None + assert failure.code is ImportErrorCode.UNEXPECTED_ERROR + assert failure.reason == UNCLASSIFIED_FAILURE[1] + + +def test_every_reason_is_long_enough_for_the_column(): + """`error_message` carries a `LENGTH(...) >= 3` CHECK; every sentence here clears it.""" + for _, reason in IMPORT_FAILURE_CATALOG.values(): + assert len(reason) >= 3 + + +# ----------------------------------------------------------------- the typed failures + + +def test_the_two_unrepeatable_failures_are_non_retryable(): + assert ImportLogRowNotClaimableError('x').non_retryable is True + assert ImportStatusRejectedError('x').non_retryable is True + assert ImportInputError('x').non_retryable is True + + +def test_a_gate_rejection_carries_its_step_and_gate_across_the_boundary(): + """An exception loses its Python type crossing an activity boundary but keeps its details.""" + error = ImportGateRejectedError('rejected', step=ImportStep.ARCHIVE_INSPECTION, gate=4) + + assert failure_location(error) == ('archive_inspection', 4) + + +def test_an_input_error_reports_the_received_step(): + assert failure_location(ImportInputError('missing object_key')) == ('received', None) + + +def test_a_located_failure_is_found_through_the_activity_wrapper(): + wrapper = RuntimeError('activity failed') + wrapper.__cause__ = ImportGateRejectedError('nope', step=ImportStep.EXTRACTION, gate=5) + + assert failure_location(wrapper) == ('extraction', 5) + + +def test_an_unlocated_failure_says_nothing_rather_than_guessing(): + assert failure_location(RuntimeError('boom')) == (None, None) + assert failure_location(None) == (None, None) + + +def test_a_cause_chain_cannot_loop_forever(): + first = RuntimeError('a') + second = RuntimeError('b') + first.__cause__ = second + second.__cause__ = first + + assert failure_location(first) == (None, None) + + +# ------------------------------------------------------------------------- the channel + + +@mark.asyncio +async def test_the_channel_hits_every_sink_once(): + sinks = FakeSinks() + + failure = await report_import_failure( + sinks, + ImportStep.REGISTRATION, + RuntimeError(LEAKY_MESSAGE), + import_run_id=77, + workflow_id='import_model-77-abc', + metadata={'model_name': 'imported_model'}, + ) + + assert failure.code is ImportErrorCode.MODEL_NOT_REGISTERED + assert len(sinks.logged) == 1 + assert len(sinks.metrics) == 1 + assert len(sinks.notifications) == 1 + assert sinks.metrics[0][0] is metrics.MODEL_IMPORT_ERROR_COUNT + assert sinks.metrics[0][1]['operation_type'] == ImportErrorCode.MODEL_NOT_REGISTERED.value + + +@mark.asyncio +async def test_the_exception_text_reaches_the_log_and_nothing_else(): + sinks = FakeSinks() + try: + raise RuntimeError(LEAKY_MESSAGE) + except RuntimeError as raised: + caught = raised + + await report_import_failure( + sinks, + ImportStep.MODEL_DOCUMENT, + caught, + import_run_id=77, + workflow_id='import_model-77-abc', + ) + + logged, _ = sinks.logged[0] + assert 'KeyError' in logged + assert 'public.experiment_run' in logged + assert 'Traceback' in logged + notification = sinks.notifications[0] + assert notification['message'] == IMPORT_FAILURE_CATALOG[ImportStep.MODEL_DOCUMENT][1] + assert 'KeyError' not in notification['message'] + assert 'KeyError' not in str(sinks.metrics[0][1]) + + +@mark.asyncio +async def test_the_log_line_correlates_the_record_and_the_workflow(): + sinks = FakeSinks() + + await report_import_failure( + sinks, + ImportStep.DOWNLOAD, + RuntimeError('boom'), + import_run_id=77, + workflow_id='import_model-77-abc', + ) + + logged, _ = sinks.logged[0] + assert 'import_run_id=77' in logged + assert 'workflow_id=import_model-77-abc' in logged + assert 'step=download' in logged + + +@mark.asyncio +async def test_a_failure_that_is_not_a_step_reports_under_its_marker(): + sinks = FakeSinks() + + failure = await report_import_failure( + sinks, + None, + None, + import_run_id=77, + workflow_id='wf', + code=ImportErrorCode.STATUS_NOT_RECORDED, + marker='status_write', + verdict='succeeded=True step=model_document gate=None', + detail_text='OperationalError: connection refused', + ) + + logged, _ = sinks.logged[0] + assert failure.code is ImportErrorCode.STATUS_NOT_RECORDED + assert 'step=status_write' in logged + assert 'verdict_not_recorded=succeeded=True step=model_document gate=None' in logged + assert 'connection refused' in logged + assert sinks.notifications[0]['block'] == 'status_write' + + +@mark.asyncio +async def test_a_report_without_an_exception_object_says_so(): + sinks = FakeSinks() + + await report_import_failure(sinks, None, None, import_run_id=77, workflow_id='wf') + + assert 'no exception object available' in sinks.logged[0][0] + + +@mark.asyncio +async def test_an_explicit_code_keeps_the_step_s_sentence(): + """The cleanup case: its own code, and a sentence a person can still read.""" + sinks = FakeSinks() + + failure = await report_import_failure( + sinks, + ImportStep.CLEANUP, + OSError('device busy'), + import_run_id=77, + workflow_id='wf', + code=ImportErrorCode.CLEANUP_INCOMPLETE, + marker=ImportStep.CLEANUP.value, + ) + + assert failure.code is ImportErrorCode.CLEANUP_INCOMPLETE + assert failure.reason == UNCLASSIFIED_FAILURE[1] + assert sinks.notifications[0]['level'] == 'ERROR' + + +# ------------------------------------------------- the channel is the only reporter + + +def _reporting_calls(path: Path) -> list[str]: + """Every call in `path` that would log an error or send a notification.""" + forbidden = { + 'error', + 'custom_error', + 'critical', + 'custom_critical', + 'send_notification', + 'send_notification_async', + 'publish_notification', + } + tree = ast.parse(path.read_text()) + found = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + target = node.func + name = target.attr if isinstance(target, ast.Attribute) else getattr(target, 'id', '') + if name in forbidden: + found.append(f'{path.name}:{node.lineno}:{name}') + return found + + +@mark.parametrize('module_path', IMPORT_MODULES, ids=lambda path: path.name) +def test_no_import_module_reports_a_failure_of_its_own(module_path: Path): + """So no failure is reported twice, or half-reported by whoever noticed it first.""" + assert _reporting_calls(module_path) == [] + + +def test_the_channel_itself_is_where_the_reporting_lives(): + channel = REPOSITORY_ROOT / 'laborious/activities/model_import_errors.py' + + calls = _reporting_calls(channel) + + assert [call.split(':')[2] for call in calls] == ['error', 'send_notification_async'] + + +def test_the_reader_neither_logs_nor_notifies(): + """The bundle reader raises; it is the activity layer that decides what a failure means.""" + for module in ('laborious/utils/bundle/reader.py', 'laborious/utils/bundle/format.py'): + assert _reporting_calls(REPOSITORY_ROOT / module) == [] + + +def test_an_unclassified_step_cannot_be_recorded_as_a_known_one(): + with raises(ValueError, match='not a valid ImportStep'): + ImportStep('provisioning') diff --git a/tests/laborious/activities/test_model_import_provisioning.py b/tests/laborious/activities/test_model_import_provisioning.py new file mode 100644 index 0000000..65012f4 --- /dev/null +++ b/tests/laborious/activities/test_model_import_provisioning.py @@ -0,0 +1,656 @@ +"""Provisioning: the experiment, the run, the registered version, the document, the cleanup. + +The boundary this file defends is where the import stops. A finished import leaves a version whose +stage was never touched and a document that is inert — promoting and activating are an operator's +decisions, owned by another card — so the tests assert not only what is written but what is *not*: +no stage transition, no alias, no `active: true`, no update of a document already there. + +The rest is the model listing's arithmetic (`max(int(id)) + 1`, which is not `count` and not a +lexicographic sort) and the two operations that must never fail an import that otherwise worked: +removing the scratch directory and putting an expiry rule on the bundle prefix. +""" + +import json +import shutil +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import mlflow as mlflow_lib +from pytest import fixture, mark, raises + +from laborious.activities.model_import import ModelImport +from laborious.utils.repository.model_repository import MLFlowRepository +from tests.helpers.status_engine import FakeStatusEngine + +METADATA = { + 'model_id': '77', + 'model_name': 'factory_model', + 'workflow_name': 'import_model', + 'schedule_name': '-', +} +WORKFLOW_ID = 'import_model-77-abc' +RUN_ID = '0123456789abcdef0123456789abcdef' +# What a tracking server with proxied artifacts reports as a run's artifact root, and therefore what +# the version's `source` is built from. The run id sits at index 2 of this URI — the whole reason the +# registration cannot store a literal `runs:/…` source. +ARTIFACT_URI = f'mlflow-artifacts:/17/{RUN_ID}/artifacts' +RESOLVED_SOURCE = f'{ARTIFACT_URI}/prediction_model' + +BUNDLE_PARAMETERS = { + 'model_name': 'factory_model', + 'target_variable': 'SE', + 'input_variables': "['feature_a', 'feature_b']", + 'retrain': 'False', +} +BUNDLE_METADATA = { + 'model_name': 'factory_model', + 'experiment_name': 'factory_experiment', + 'run_id': 'aa11bb22cc33dd44ee55ff6677889900', + 'model_version': '2', + 'model_project': 'factory_project', + 'export_timestamp': '2026-08-25T12:00:00+00:00', +} + +# The whole vocabulary of promotion. None of it may be reached by this change. +PROMOTION_CALLS = ( + 'transition_model_version_stage', + 'set_registered_model_alias', + 'set_model_version_tag', + 'update_model_version', +) + + +@fixture +def mlflow_client() -> MagicMock: + client = MagicMock() + client.create_run.return_value.info.run_id = RUN_ID + client.get_run.return_value.info.artifact_uri = ARTIFACT_URI + client.create_model_version.return_value.version = 4 + return client + + +@fixture +def mongo() -> AsyncMock: + repository = AsyncMock() + repository.find_one.return_value = None + repository.distinct.return_value = [] + return repository + + +@fixture +def import_activity(tmp_path: Path, mlflow_client: MagicMock, mongo: AsyncMock) -> ModelImport: + minio_repository = MagicMock() + mlflow_repository = MagicMock() + mlflow_repository.client = mlflow_client + with patch('laborious.activities.model_import.create_engine', return_value=FakeStatusEngine()): + activity = ModelImport( + status_config={ + 'host': 'h', + 'port': 5432, + 'dbname': 'sientia-core-mlops-bff', + 'user': 'u', + 'password': 'p', + 'schema': 'public', + }, + import_config={ + 'password_key': None, + 'bucket': 'sientia', + 'prefix': 'imported_models/', + 'work_dir': str(tmp_path), + 'max_object_bytes': 1024, + 'max_entries': 10, + 'max_uncompressed_bytes': 4096, + 'max_compression_ratio': 200.0, + 'models_collection': 'models', + 'retention_days': 7, + }, + minio_repository=minio_repository, + mlflow_repository=mlflow_repository, + mongo_repository=mongo, + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=AsyncMock(), + ) + activity.emit_metric = AsyncMock() + activity.send_notification_async = AsyncMock() + return activity + + +def base(**overrides) -> dict: + return { + 'metadata': METADATA, + 'import_run_id': 77, + 'workflow_id': WORKFLOW_ID, + **overrides, + } + + +# ---------------------------------------------------------------------- the experiment + + +@mark.asyncio +async def test_the_experiment_keeps_the_origin_s_name(import_activity: ModelImport): + repository = import_activity.import_mlflow_repository + repository.get_experiment.return_value = MagicMock(experiment_id='17') + + result = await import_activity.create_import_experiment( + base(experiment_name='factory_experiment') + ) + + repository.get_experiment.assert_called_once_with( + 'factory_experiment', create_if_not_exists=True + ) + assert result == {'experiment_id': '17', 'experiment_name': 'factory_experiment'} + + +@mark.asyncio +async def test_a_missing_mlflow_repository_is_a_configuration_error(import_activity: ModelImport): + import_activity.import_mlflow_repository = None + + with raises(ValueError, match='MLflow repository not initialized'): + await import_activity.create_import_experiment(base(experiment_name='e')) + + +# ------------------------------------------------------------------- the run and its tree + + +def upload_input(extracted_dir: Path, **overrides) -> dict: + return base( + experiment_id='17', + run_name='import-factory_model-v2', + extracted_dir=str(extracted_dir), + parameters=BUNDLE_PARAMETERS, + source=BUNDLE_METADATA, + **overrides, + ) + + +@mark.asyncio +async def test_every_import_creates_a_new_run_with_the_recorded_name( + import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path +): + """The name is passed in, never recomputed, so the record and the run cannot diverge.""" + result = await import_activity.upload_import_artifacts(upload_input(tmp_path / 'extract')) + + mlflow_client.create_run.assert_called_once_with( + experiment_id='17', run_name='import-factory_model-v2' + ) + assert result == {'run_id': RUN_ID, 'run_name': 'import-factory_model-v2'} + + +@mark.asyncio +async def test_the_artifact_tree_is_logged_from_the_bundle_s_artifacts_directory( + import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path +): + extracted = tmp_path / 'extract' + + await import_activity.upload_import_artifacts(upload_input(extracted)) + + mlflow_client.log_artifacts.assert_called_once_with(RUN_ID, str(extracted / 'artifacts')) + + +@mark.asyncio +async def test_the_bundle_s_parameters_are_logged_verbatim( + import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path +): + await import_activity.upload_import_artifacts(upload_input(tmp_path / 'extract')) + + logged = { + call.args[1]: call.args[2] + for call in mlflow_client.log_param.call_args_list + if not call.args[1].startswith('source.') + } + assert logged == BUNDLE_PARAMETERS + assert logged['target_variable'] == 'SE' + + +@mark.asyncio +async def test_the_origin_is_logged_as_source_parameters( + import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path +): + await import_activity.upload_import_artifacts(upload_input(tmp_path / 'extract')) + + logged = { + call.args[1]: call.args[2] + for call in mlflow_client.log_param.call_args_list + if call.args[1].startswith('source.') + } + assert logged == {f'source.{key}': value for key, value in BUNDLE_METADATA.items()} + # A record of where the model came from, not a link: the origin run id is never dereferenced. + assert logged['source.run_id'] == BUNDLE_METADATA['run_id'] + mlflow_client.get_run.assert_not_called() + mlflow_client.download_artifacts.assert_not_called() + + +@mark.asyncio +async def test_the_run_is_closed_when_its_tree_is_in( + import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path +): + await import_activity.upload_import_artifacts(upload_input(tmp_path / 'extract')) + + mlflow_client.set_terminated.assert_called_once_with(RUN_ID, 'FINISHED') + + +# ------------------------------------------------------------------------ registration + + +@mark.asyncio +async def test_the_version_source_is_the_shape_the_legacy_loader_parses( + import_activity: ModelImport, mlflow_client: MagicMock +): + result = await import_activity.register_import_model_version( + base(model_name='factory_model', run_id=RUN_ID) + ) + + assert result == {'model_name': 'factory_model', 'version': '4', 'source': RESOLVED_SOURCE} + mlflow_client.create_model_version.assert_called_once_with( + name='factory_model', source=RESOLVED_SOURCE, run_id=RUN_ID + ) + # The one property the legacy loader depends on, asserted directly. + assert RESOLVED_SOURCE.split('/')[2] == RUN_ID + + +@mark.asyncio +async def test_a_literal_runs_uri_would_not_be_parseable_by_the_legacy_loader(): + """The trap this registration avoids: `create_model_version` stores `source` verbatim, and a + `runs:/` URI has only two slashes, so the loader's index 2 is the artifact path.""" + assert f'runs:/{RUN_ID}/prediction_model'.split('/')[2] == 'prediction_model' + assert 'models:/m-9f1c'.split('/')[1:] == ['m-9f1c'] + + +@mark.asyncio +async def test_a_registered_model_is_created_only_when_it_is_absent( + import_activity: ModelImport, mlflow_client: MagicMock +): + mlflow_client.get_registered_model.side_effect = RuntimeError('not found') + + await import_activity.register_import_model_version(base(model_name='new_model', run_id=RUN_ID)) + + mlflow_client.create_registered_model.assert_called_once_with('new_model') + + +@mark.asyncio +async def test_a_second_import_reuses_the_registered_model( + import_activity: ModelImport, mlflow_client: MagicMock +): + mlflow_client.get_registered_model.return_value = MagicMock(name='factory_model') + + await import_activity.register_import_model_version( + base(model_name='factory_model', run_id=RUN_ID) + ) + + mlflow_client.create_registered_model.assert_not_called() + + +@mark.asyncio +async def test_the_import_never_promotes_anything( + import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path +): + """A `COMPLETED` import is a registered model, not a servable one.""" + await import_activity.create_import_experiment(base(experiment_name='factory_experiment')) + await import_activity.upload_import_artifacts(upload_input(tmp_path / 'extract')) + await import_activity.register_import_model_version( + base(model_name='factory_model', run_id=RUN_ID) + ) + + for call_name in PROMOTION_CALLS: + getattr(mlflow_client, call_name).assert_not_called() + + +@mark.asyncio +async def test_no_provisioning_call_passes_a_stage( + import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path +): + await import_activity.upload_import_artifacts(upload_input(tmp_path / 'extract')) + result = await import_activity.register_import_model_version( + base(model_name='factory_model', run_id=RUN_ID) + ) + + for call in mlflow_client.method_calls: + assert 'stage' not in call.kwargs + assert 'alias' not in call.kwargs + assert 'stage' not in result + + +# ------------------------------------------------- the legacy loader reads what we wrote + + +def legacy_repository() -> MLFlowRepository: + """A real `MLFlowRepository`, with the process-wide side effects of its constructor stubbed.""" + with ( + patch.object(mlflow_lib, 'set_tracking_uri'), + patch.object(mlflow_lib.tracking, 'MlflowClient'), + ): + return MLFlowRepository( + host='http://localhost:5000', + username='admin', + password='admin', + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=AsyncMock(), + ) + + +def registry_holding(source: str) -> MagicMock: + client = MagicMock() + client.search_registered_models.return_value = [MagicMock()] + version = MagicMock(version='4', source=source) + version.current_stage = 'Production' + client.search_model_versions.return_value = [version] + return client + + +def test_the_legacy_loader_resolves_the_run_the_import_created(): + """The contract the source shape exists for: the real production code, fed a version this + import would create, returns this import's run id.""" + repository = legacy_repository() + repository.client = registry_holding(RESOLVED_SOURCE) + + assert repository.get_model_run_id('factory_model') == RUN_ID + + +def test_the_legacy_loader_would_read_a_runs_uri_as_the_artifact_path(): + """Left as an executable warning: registering with the unresolved URI silently breaks loading, + because nothing raises — the loader just returns the wrong string.""" + repository = legacy_repository() + repository.client = registry_holding(f'runs:/{RUN_ID}/prediction_model') + + assert repository.get_model_run_id('factory_model') == 'prediction_model' + + +def test_the_legacy_loader_cannot_read_a_logged_model_uri(): + repository = legacy_repository() + repository.client = registry_holding('models:/m-9f1c') + + with raises(mlflow_lib.exceptions.MlflowException, match='invalid source URI'): + repository.get_model_run_id('factory_model') + + +# ------------------------------------------------------------------- the model document + + +@mark.asyncio +async def test_the_document_id_continues_the_collection_across_gaps( + import_activity: ModelImport, mongo: AsyncMock +): + mongo.distinct.return_value = [str(value) for value in [*range(1, 19), *range(20, 25), 30]] + + result = await import_activity.write_import_model_document( + base(model_name='factory_model', target='SE') + ) + + assert result['id'] == '31' + + +@mark.asyncio +async def test_the_document_id_is_numeric_not_lexicographic( + import_activity: ModelImport, mongo: AsyncMock +): + mongo.distinct.return_value = ['9', '30'] + + result = await import_activity.write_import_model_document( + base(model_name='factory_model', target='SE') + ) + + assert result['id'] == '31' + + +@mark.asyncio +async def test_an_empty_collection_starts_at_one(import_activity: ModelImport, mongo: AsyncMock): + result = await import_activity.write_import_model_document( + base(model_name='factory_model', target='SE') + ) + + assert result['id'] == '1' + + +@mark.asyncio +async def test_the_document_is_exactly_the_contracted_shape( + import_activity: ModelImport, mongo: AsyncMock +): + mongo.distinct.return_value = ['7'] + + await import_activity.write_import_model_document(base(model_name='factory_model', target='SE')) + + collection, document, metadata = mongo.insert.await_args.args + assert collection == 'models' + assert metadata == METADATA + assert document == { + 'id': '8', + 'name': 'factory_model', + 'active': False, + 'model_config': { + 'transform_flavor': 'sklearn', + 'predict_flavor': 'joblib', + 'retention_minutes': 60, + 'target': 'SE', + }, + } + + +@mark.asyncio +async def test_the_document_carries_no_field_the_runtime_does_not_read( + import_activity: ModelImport, mongo: AsyncMock +): + await import_activity.write_import_model_document(base(model_name='factory_model', target='SE')) + + _, document, _ = mongo.insert.await_args.args + for absent in ('is_compressed', 'retention_target', 'stage', 'source', 'project'): + assert absent not in document + assert absent not in document['model_config'] + + +@mark.asyncio +async def test_a_name_already_in_the_collection_is_left_exactly_as_it_is( + import_activity: ModelImport, mongo: AsyncMock +): + """`active`, `target` and the flavors of a model already on the platform are operator-owned.""" + mongo.find_one.return_value = {'id': '12', 'name': 'factory_model', 'active': True} + + result = await import_activity.write_import_model_document( + base(model_name='factory_model', target='SE') + ) + + assert result == {'written': False, 'skipped': True, 'id': '12'} + mongo.insert.assert_not_awaited() + mongo.distinct.assert_not_awaited() + + +@mark.asyncio +async def test_unusable_ids_are_ignored_and_counted(import_activity: ModelImport, mongo: AsyncMock): + mongo.distinct.return_value = ['1', 'legacy-model', None, '3'] + + result = await import_activity.write_import_model_document( + base(model_name='factory_model', target='SE') + ) + + assert result['id'] == '4' + assert result['ignored_ids'] == 2 + assert any( + 'ignored 2 non-numeric' in str(call) + for call in import_activity.logger.custom_info.call_args_list + ) + + +@mark.asyncio +async def test_a_duplicate_id_fails_the_step_without_recomputing_it( + import_activity: ModelImport, mongo: AsyncMock +): + """No retry and no second guess: the unique index says the collection moved under us.""" + mongo.distinct.return_value = ['1'] + mongo.insert.side_effect = RuntimeError('E11000 duplicate key error: id_1') + + with raises(RuntimeError, match='duplicate key'): + await import_activity.write_import_model_document( + base(model_name='factory_model', target='SE') + ) + + assert mongo.distinct.await_count == 1 + assert mongo.insert.await_count == 1 + + +@mark.asyncio +async def test_the_import_only_reads_and_inserts_documents( + import_activity: ModelImport, mongo: AsyncMock +): + """No update, no replace, no upsert: an import cannot change a model already on the platform.""" + await import_activity.write_import_model_document(base(model_name='factory_model', target='SE')) + + used = {name for name, _, _ in mongo.mock_calls} + assert used <= {'find_one', 'distinct', 'insert'} + + +@mark.asyncio +async def test_a_missing_mongo_repository_is_a_configuration_error( + import_activity: ModelImport, +): + import_activity.mongo_repository = None + + with raises(ValueError, match='MongoDB repository not initialized'): + await import_activity.write_import_model_document(base(model_name='m', target='SE')) + + +def test_no_import_module_writes_an_active_document(): + """Asserted from the source too, so `active: false` cannot be flipped by a later edit.""" + source = Path('laborious/activities/model_import.py').read_text() + + assert "'active': False" in source + assert "'active': True" not in source + for forbidden in ('update_one', 'replace_one', 'update_many', 'upsert'): + assert forbidden not in source + + +# ------------------------------------------------------------------ cleanup and retention + + +@mark.asyncio +async def test_cleanup_removes_this_import_s_scratch_directory( + import_activity: ModelImport, tmp_path: Path +): + work_dir = tmp_path / WORKFLOW_ID + (work_dir / 'extract' / 'artifacts').mkdir(parents=True) + (work_dir / 'bundle.sientia').write_bytes(b'downloaded') + + result = await import_activity.cleanup_import_files(base()) + + assert result == {'cleaned': True, 'existed': True} + assert not work_dir.exists() + + +@mark.asyncio +async def test_cleanup_of_a_run_that_never_downloaded_anything_is_not_a_failure( + import_activity: ModelImport, +): + result = await import_activity.cleanup_import_files(base()) + + assert result == {'cleaned': True, 'existed': False} + + +@mark.asyncio +async def test_cleanup_that_fails_is_reported_and_swallowed( + import_activity: ModelImport, tmp_path: Path +): + """The user's model was imported; a leaked directory is a log line, not an ERROR.""" + (tmp_path / WORKFLOW_ID).mkdir() + + with patch.object(shutil, 'rmtree', side_effect=OSError('device or resource busy')): + result = await import_activity.cleanup_import_files(base()) + + assert result['cleaned'] is False + assert result['reason'] == 'OSError' + logged = import_activity.logger.custom_error.call_args_list[0].args[0] + assert 'IMPORT_CLEANUP_INCOMPLETE' in logged + import_activity.send_notification_async.assert_awaited_once() + + +@mark.asyncio +async def test_cleanup_never_touches_the_uploaded_object( + import_activity: ModelImport, tmp_path: Path +): + (tmp_path / WORKFLOW_ID).mkdir() + + await import_activity.cleanup_import_files(base()) + + client = import_activity.import_minio_repository.minio_client + client.remove_object.assert_not_called() + client.remove_objects.assert_not_called() + + +@mark.asyncio +async def test_the_expiry_rule_is_applied_to_the_import_prefix(import_activity: ModelImport): + client = import_activity.import_minio_repository.minio_client + client.get_bucket_lifecycle.return_value = None + + result = await import_activity.ensure_import_bundle_retention(base()) + + assert result == {'bucket': 'sientia', 'prefix': 'imported_models/', 'days': 7, 'rules': 1} + config = client.set_bucket_lifecycle.call_args.args[1] + rule = config.rules[0] + assert rule.rule_id == 'sientia-import-bundle-expiry' + assert rule.expiration.days == 7 + assert rule.rule_filter.prefix == 'imported_models/' + + +@mark.asyncio +async def test_applying_the_rule_twice_leaves_one_rule(import_activity: ModelImport): + client = import_activity.import_minio_repository.minio_client + client.get_bucket_lifecycle.return_value = None + + await import_activity.ensure_import_bundle_retention(base()) + applied = client.set_bucket_lifecycle.call_args.args[1] + client.get_bucket_lifecycle.return_value = applied + await import_activity.ensure_import_bundle_retention(base()) + + config = client.set_bucket_lifecycle.call_args.args[1] + assert [rule.rule_id for rule in config.rules] == ['sientia-import-bundle-expiry'] + + +@mark.asyncio +async def test_a_rule_that_belongs_to_somebody_else_is_kept(import_activity: ModelImport): + client = import_activity.import_minio_repository.minio_client + existing = MagicMock() + existing.rules = [MagicMock(rule_id='predictions-offload-expiry')] + client.get_bucket_lifecycle.return_value = existing + + await import_activity.ensure_import_bundle_retention(base()) + + config = client.set_bucket_lifecycle.call_args.args[1] + assert [rule.rule_id for rule in config.rules] == [ + 'predictions-offload-expiry', + 'sientia-import-bundle-expiry', + ] + + +@mark.asyncio +async def test_a_bucket_with_no_lifecycle_configuration_at_all_is_handled( + import_activity: ModelImport, +): + """Some deployments answer with an error, others with `None`; both mean "no rule yet".""" + client = import_activity.import_minio_repository.minio_client + client.get_bucket_lifecycle.side_effect = RuntimeError('NoSuchLifecycleConfiguration') + + result = await import_activity.ensure_import_bundle_retention(base()) + + assert result['rules'] == 1 + + +@mark.asyncio +async def test_the_retention_rule_can_be_applied_to_a_requested_bucket( + import_activity: ModelImport, +): + client = import_activity.import_minio_repository.minio_client + client.get_bucket_lifecycle.return_value = None + + result = await import_activity.ensure_import_bundle_retention(base(bucket='other-bucket')) + + assert result['bucket'] == 'other-bucket' + assert client.set_bucket_lifecycle.call_args.args[0] == 'other-bucket' + + +def test_the_model_document_shape_is_documented_in_one_place(): + """A guard against the shape drifting between the code and what the runtime reads.""" + source = Path('laborious/activities/model_import.py').read_text() + + for key in ('transform_flavor', 'predict_flavor', 'retention_minutes', 'target'): + assert f"'{key}'" in source + assert json.dumps({'transform_flavor': 'sklearn'}) # sanity: the flavors are plain strings diff --git a/tests/laborious/activities/test_model_import_status.py b/tests/laborious/activities/test_model_import_status.py new file mode 100644 index 0000000..b568f59 --- /dev/null +++ b/tests/laborious/activities/test_model_import_status.py @@ -0,0 +1,754 @@ +"""The import log row: the claim, the hints, the names and the one authoritative write. + +What these tests defend, in the order the workflow does it: + +- the **state mapping** is total — every `ImportStep` has a status, so a new step cannot be added + without deciding what the record says when it fails; +- the **claim** is the only statement that checks ownership, and a claim that matches nothing writes + nothing anywhere and fails loudly everywhere else; +- every **later** write keys on the id alone, never rewrites a column the inserting side owns, and + never drops the workflow identifier out of the JSON detail; +- the **terminal** write is the single authority: one classification, `NULL` (never `''`) on success, + and a failure of its own that neither invents a failed import nor masks a real one. +""" + +import json +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock, patch + +from pytest import fixture, mark, raises +from sqlalchemy.exc import DataError, IntegrityError, OperationalError, ProgrammingError + +from laborious import metrics +from laborious.activities.model_import import ( + IMPORT_LOG_TABLE, + IMPORT_STATUS_HINT_RETRY_POLICY, + IMPORT_STATUS_RETRY_POLICY, + RUN_NAME_MAX_LENGTH, + ExperimentRunStatus, + ImportState, + ModelImport, + build_import_detail, + derive_run_name, + map_state_to_status, +) +from laborious.activities.model_import_errors import ( + ImportErrorCode, + ImportLogRowNotClaimableError, + ImportStatusRejectedError, +) +from laborious.utils.bundle.steps import ImportStep +from tests.helpers.status_engine import FakeStatusEngine + +STATUS_CONFIG = { + 'host': 'bff-db.internal', + 'port': 5432, + 'dbname': 'sientia-core-mlops-bff', + 'user': 'mlops_bff_user', + 'password': 'bff-password', + 'schema': 'public', +} + +IMPORT_CONFIG = { + 'password_key': None, + 'bucket': 'sientia', + 'prefix': 'imported_models/', + 'work_dir': '/tmp/sientia-import', # noqa: S108 - never written to in this module's tests + 'max_object_bytes': 1024, + 'max_entries': 10, + 'max_uncompressed_bytes': 4096, + 'max_compression_ratio': 200.0, + 'models_collection': 'models', + 'retention_days': 7, +} + +METADATA = { + 'model_id': '77', + 'model_name': 'imported_model', + 'workflow_name': 'import_model', + 'schedule_name': '-', +} + +# The columns the inserting side owns. No write in this module may ever assign one of them: the +# record is shared with the training flow and with whoever created the row. +BFF_OWNED_COLUMNS = frozenset( + {'username', 'request_data', 'bucket_name', 'file_name', 'run_type', 'created_at'} +) + +# What the record's status must be for a failure at each step. Written out rather than derived, so a +# new step forces a decision here instead of silently inheriting one. +EXPECTED_ERROR_STATUS = { + ImportStep.RECEIVED: ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR, + ImportStep.DOWNLOAD: ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR, + ImportStep.DECRYPTION: ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR, + ImportStep.ARCHIVE_INSPECTION: ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR, + ImportStep.EXTRACTION: ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR, + ImportStep.STRUCTURE_VALIDATION: ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR, + ImportStep.CONTENT_POLICY: ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR, + ImportStep.EXPERIMENT_CREATION: ExperimentRunStatus.IMPORT_ERROR, + ImportStep.ARTIFACT_UPLOAD: ExperimentRunStatus.IMPORT_ERROR, + ImportStep.REGISTRATION: ExperimentRunStatus.IMPORT_ERROR, + ImportStep.MODEL_DOCUMENT: ExperimentRunStatus.IMPORT_ERROR, + ImportStep.CLEANUP: ExperimentRunStatus.IMPORT_ERROR, +} + + +def build_activity(engine: FakeStatusEngine) -> ModelImport: + """A `ModelImport` whose only connection is the recording engine double.""" + with patch('laborious.activities.model_import.create_engine', return_value=engine): + activity = ModelImport( + status_config=STATUS_CONFIG, + import_config=IMPORT_CONFIG, + minio_repository=MagicMock(), + mlflow_repository=MagicMock(), + mongo_repository=AsyncMock(), + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=AsyncMock(), + ) + activity.emit_metric = AsyncMock() + activity.send_notification_async = AsyncMock() + return activity + + +@fixture +def engine() -> FakeStatusEngine: + return FakeStatusEngine() + + +@fixture +def import_activity(engine: FakeStatusEngine) -> ModelImport: + return build_activity(engine) + + +def detail_of(statement) -> dict: + """The `import` object a write put into `orchestrator_response_data`.""" + return json.loads(statement.parameters['detail'])['import'] + + +def claim_input(**overrides) -> dict: + return { + 'metadata': METADATA, + 'import_run_id': 77, + 'workflow_id': 'import_model-77-abc', + **overrides, + } + + +# --------------------------------------------------------------------- the state mapping + + +def test_running_is_the_orchestrator_holding_the_row(): + assert map_state_to_status(ImportState.RUNNING) == ExperimentRunStatus.ORCHESTRATOR_WAITING_PROC + # The step must not change a running row's status: the row is running whatever it is doing. + for step in ImportStep: + assert ( + map_state_to_status(ImportState.RUNNING, step) + == ExperimentRunStatus.ORCHESTRATOR_WAITING_PROC + ) + + +def test_completed_is_import_success(): + assert map_state_to_status(ImportState.COMPLETED) == ExperimentRunStatus.IMPORT_SUCCESS + assert ( + map_state_to_status(ImportState.COMPLETED, ImportStep.MODEL_DOCUMENT) + == ExperimentRunStatus.IMPORT_SUCCESS + ) + + +@mark.parametrize('step', list(ImportStep)) +def test_every_step_has_an_error_status(step: ImportStep): + assert map_state_to_status(ImportState.ERROR, step) == EXPECTED_ERROR_STATUS[step] + # The value form is what the workflow actually passes across the activity boundary. + assert map_state_to_status(ImportState.ERROR, step.value) == EXPECTED_ERROR_STATUS[step] + + +def test_the_error_mapping_covers_the_whole_enum(): + """A step added without a decision here fails this test rather than defaulting quietly.""" + assert set(EXPECTED_ERROR_STATUS) == set(ImportStep) + + +def test_an_unclassified_error_is_a_platform_error(): + assert map_state_to_status(ImportState.ERROR) == ExperimentRunStatus.IMPORT_ERROR + + +# ------------------------------------------------------------------------- the run name + + +def test_run_name_has_the_documented_shape(): + assert derive_run_name('sales_forecast', '3') == 'import-sales_forecast-v3' + + +def test_run_name_truncates_inside_the_name_segment(): + name = derive_run_name('a' * 120, '12') + assert name.startswith('import-') + assert name.endswith('-v12') + assert len(name) == RUN_NAME_MAX_LENGTH + + +def test_run_name_survives_a_version_that_leaves_no_room(): + name = derive_run_name('model', 'v' * 80) + assert name.startswith('import-') + assert len(name) <= RUN_NAME_MAX_LENGTH + + +def test_run_name_is_deterministic(): + assert derive_run_name('model', '4') == derive_run_name('model', '4') + + +def test_run_name_is_never_shorter_than_the_column_floor(): + # `error_message` is not the only column with a length CHECK; the prefix is what guarantees it. + assert len(derive_run_name('', '')) >= 3 + + +# --------------------------------------------------------------------- the JSON detail + + +def test_the_detail_always_carries_the_workflow_id(): + detail = json.loads(build_import_detail(workflow_id='wf-1'))['import'] + assert detail['workflow_id'] == 'wf-1' + + +def test_the_detail_omits_a_code_when_there_is_none(): + detail = json.loads(build_import_detail(workflow_id='wf-1', step=ImportStep.RECEIVED))['import'] + assert 'code' not in detail + assert detail['step'] == 'received' + + +def test_the_detail_records_a_cleanup_failure_without_a_code(): + detail = json.loads(build_import_detail(workflow_id='wf-1', cleanup_failed=True))['import'] + assert detail['cleanup_failed'] is True + assert 'code' not in detail + + +# ------------------------------------------------------------------------------ the claim + + +@mark.asyncio +async def test_claim_writes_the_running_status_and_the_workflow_id( + import_activity: ModelImport, engine: FakeStatusEngine +): + engine.rows = [{'run_type': 'IMPORT', 'bucket_name': 'sientia', 'file_name': None}] + + result = await import_activity.claim_import_status(claim_input()) + + assert result['claimed'] is True + write = engine.writes[0] + assert write.table == IMPORT_LOG_TABLE + assert write.parameters['status'] == ExperimentRunStatus.ORCHESTRATOR_WAITING_PROC.value + assert detail_of(write) == { + 'workflow_id': 'import_model-77-abc', + 'gate': None, + 'step': 'received', + } + + +@mark.asyncio +async def test_claim_is_the_only_statement_that_guards_ownership( + import_activity: ModelImport, engine: FakeStatusEngine +): + engine.rows = [{'run_type': 'IMPORT'}] + + await import_activity.claim_import_status(claim_input()) + + sql = engine.writes[0].sql + assert "run_type = 'IMPORT'" in sql + assert "status = 'PENDING'" in sql + # The idempotency disjunct: a re-execution of the same workflow re-claims its own row instead of + # colliding with itself. It reads the identifier out of the JSON detail because the change adds + # no column for it. + assert "orchestrator_response_data -> 'import' ->> 'workflow_id' = :workflow_id" in sql + + +@mark.asyncio +async def test_claim_returns_the_values_the_input_guards_are_checked_against( + import_activity: ModelImport, engine: FakeStatusEngine +): + engine.rows = [ + { + 'run_type': 'IMPORT', + 'bucket_name': 'sientia', + 'file_name': 'imported_models/model.sientia', + 'request_data': {'expected_digest': 'AB' * 32}, + } + ] + + result = await import_activity.claim_import_status(claim_input()) + + assert result['bucket_name'] == 'sientia' + assert result['file_name'] == 'imported_models/model.sientia' + assert result['request_digest'] == 'ab' * 32 + + +@mark.asyncio +@mark.parametrize('key', ['expected_digest', 'digest', 'sha256', 'file_digest']) +async def test_claim_finds_the_digest_wherever_the_bff_recorded_it(key: str): + engine = FakeStatusEngine(rows=[{'run_type': 'IMPORT', 'request_data': {key: 'f' * 64}}]) + activity = build_activity(engine) + + result = await activity.claim_import_status(claim_input()) + + assert result['request_digest'] == 'f' * 64 + + +@mark.asyncio +@mark.parametrize('request_data', [None, 'not-a-dict', {}, {'expected_digest': ''}]) +async def test_claim_treats_an_absent_digest_as_absent(request_data): + engine = FakeStatusEngine(rows=[{'run_type': 'IMPORT', 'request_data': request_data}]) + activity = build_activity(engine) + + result = await activity.claim_import_status(claim_input()) + + assert result['request_digest'] is None + + +@mark.asyncio +async def test_claim_reclaimed_by_the_same_workflow_succeeds( + import_activity: ModelImport, engine: FakeStatusEngine +): + engine.rows = [ + {'run_type': 'IMPORT', 'status': 'ORCHESTRATOR_WAITING_PROC', 'claimed_by': 'wf-mine'} + ] + + result = await import_activity.claim_import_status(claim_input(workflow_id='wf-mine')) + + assert result['claimed'] is True + + +@mark.asyncio +async def test_claim_opens_one_transaction_and_leaves_none_open( + import_activity: ModelImport, engine: FakeStatusEngine +): + engine.rows = [{'run_type': 'IMPORT'}] + + await import_activity.claim_import_status(claim_input()) + + assert engine.transactions == 1 + assert engine.open_scopes == 0 + + +@mark.asyncio +async def test_the_claim_writes_before_it_reads( + import_activity: ModelImport, engine: FakeStatusEngine +): + """A read-then-write claim would race two workflows into the same row; this one cannot.""" + engine.rows = [{'run_type': 'IMPORT'}] + + await import_activity.claim_import_status(claim_input()) + + assert engine.statements[0].is_write is True + assert engine.statements[1].verb == 'SELECT' + + +@mark.asyncio +@mark.parametrize( + ('row', 'expected'), + [ + (None, 'no such import log row'), + ({'run_type': 'TRAIN', 'status': 'PENDING'}, 'is a TRAIN run'), + ( + { + 'run_type': 'IMPORT', + 'status': 'ORCHESTRATOR_WAITING_PROC', + 'claimed_by': 'another-workflow', + }, + 'already claimed by workflow another-workflow', + ), + ({'run_type': 'IMPORT', 'status': 'IMPORT_SUCCESS'}, 'in status IMPORT_SUCCESS'), + ], +) +async def test_an_unclaimable_row_names_its_cause(row, expected: str): + engine = FakeStatusEngine(rowcounts=[0], rows=[row]) + activity = build_activity(engine) + + with raises(ImportLogRowNotClaimableError) as failure: + await activity.claim_import_status(claim_input()) + + assert expected in str(failure.value) + assert failure.value.import_run_id == 77 + + +@mark.asyncio +async def test_an_unclaimable_row_is_not_retried(): + engine = FakeStatusEngine(rowcounts=[0], rows=[None]) + activity = build_activity(engine) + + with raises(ImportLogRowNotClaimableError) as failure: + await activity.claim_import_status(claim_input()) + + assert failure.value.non_retryable is True + assert failure.value.type == 'ImportLogRowNotClaimableError' + # And the policy that governs the claim agrees, so the ten attempts are not spent on a verdict + # that cannot change. + assert 'ImportLogRowNotClaimableError' in IMPORT_STATUS_RETRY_POLICY.non_retryable_error_types + assert ( + 'ImportLogRowNotClaimableError' in IMPORT_STATUS_HINT_RETRY_POLICY.non_retryable_error_types + ) + + +@mark.asyncio +async def test_an_unclaimable_row_is_never_written_to(): + """The only statement executed is the guarded claim that matched nothing, and no other write.""" + engine = FakeStatusEngine(rowcounts=[0], rows=[{'run_type': 'TRAIN'}]) + activity = build_activity(engine) + + with raises(ImportLogRowNotClaimableError): + await activity.claim_import_status(claim_input()) + + assert len(engine.writes) == 1 + assert engine.writes[0].sql == engine.statements[0].sql + assert "run_type = 'IMPORT'" in engine.writes[0].sql + assert [statement.verb for statement in engine.statements[1:]] == ['SELECT'] + assert engine.tables == {IMPORT_LOG_TABLE} + + +@mark.asyncio +async def test_an_unclaimable_row_is_reported_through_every_sink(): + engine = FakeStatusEngine(rowcounts=[0], rows=[None]) + activity = build_activity(engine) + + with raises(ImportLogRowNotClaimableError): + await activity.claim_import_status(claim_input()) + + logged = activity.logger.custom_error.call_args_list[0].args[0] + assert ImportErrorCode.LOG_ROW_NOT_CLAIMABLE.value in logged + assert 'not_claimable' in logged + metric = activity.emit_metric.await_args.kwargs + assert metric['metric_object'] is metrics.MODEL_IMPORT_ERROR_COUNT + assert metric['tags']['operation_type'] == ImportErrorCode.LOG_ROW_NOT_CLAIMABLE.value + notification = activity.send_notification_async.await_args.kwargs + assert notification['notification_id'] == ImportErrorCode.LOG_ROW_NOT_CLAIMABLE.value + assert notification['level'] == 'ERROR' + + +@mark.asyncio +async def test_a_claimed_row_counts_as_an_import_started( + import_activity: ModelImport, engine: FakeStatusEngine +): + engine.rows = [{'run_type': 'IMPORT'}] + + await import_activity.claim_import_status(claim_input()) + + metric = import_activity.emit_metric.await_args.kwargs + assert metric['metric_object'] is metrics.MODEL_IMPORT_STARTED_COUNT + + +# ------------------------------------------------------------------- progress and names + + +@mark.asyncio +async def test_a_progress_write_keys_on_the_id_alone( + import_activity: ModelImport, engine: FakeStatusEngine +): + await import_activity.record_import_status({**claim_input(), 'step': ImportStep.DOWNLOAD.value}) + + write = engine.writes[0] + assert 'WHERE id = :import_run_id' in write.sql + assert 'run_type' not in write.sql + assert write.parameters['status'] == ExperimentRunStatus.ORCHESTRATOR_WAITING_PROC.value + assert detail_of(write)['step'] == 'download' + + +@mark.asyncio +async def test_a_lost_progress_write_is_not_a_failure( + import_activity: ModelImport, engine: FakeStatusEngine +): + engine.rowcounts = [0] + + result = await import_activity.record_import_status( + {**claim_input(), 'step': ImportStep.DOWNLOAD.value} + ) + + assert result == {'recorded': False, 'rows': 0} + + +@mark.asyncio +async def test_the_names_write_touches_only_the_two_name_columns( + import_activity: ModelImport, engine: FakeStatusEngine +): + await import_activity.record_import_names( + { + **claim_input(), + 'experiment_name': 'sales_forecast', + 'run_name': 'import-sales_forecast-v3', + } + ) + + write = engine.writes[0] + assert write.assignments == { + 'experiment_name', + 'run_name', + 'updated_at', + 'orchestrator_response_data', + } + # Deliberately no `status`: the names are written mid-flight and must not disturb the verdict. + assert 'status' not in write.assignments + assert write.parameters['experiment_name'] == 'sales_forecast' + assert write.parameters['run_name'] == 'import-sales_forecast-v3' + + +@mark.asyncio +async def test_the_names_write_preserves_the_workflow_id( + import_activity: ModelImport, engine: FakeStatusEngine +): + await import_activity.record_import_names( + {**claim_input(), 'experiment_name': 'e', 'run_name': 'import-e-v1'} + ) + + assert detail_of(engine.writes[0])['workflow_id'] == 'import_model-77-abc' + + +# ------------------------------------------------------------------- the terminal write + + +@mark.asyncio +async def test_a_successful_import_writes_a_null_reason_and_no_code( + import_activity: ModelImport, engine: FakeStatusEngine +): + verdict = await import_activity.record_import_terminal_status( + {**claim_input(), 'succeeded': True, 'step': ImportStep.MODEL_DOCUMENT.value} + ) + + write = engine.writes[0] + assert write.parameters['status'] == ExperimentRunStatus.IMPORT_SUCCESS.value + # `''` would violate the column's `LENGTH(error_message) >= 3` CHECK. + assert write.parameters['error_message'] is None + assert 'code' not in detail_of(write) + assert verdict['status'] == ExperimentRunStatus.IMPORT_SUCCESS.value + assert verdict['code'] is None + + +@mark.asyncio +@mark.parametrize( + ('step', 'status', 'code'), + [ + ( + ImportStep.ARCHIVE_INSPECTION, + ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR, + ImportErrorCode.BUNDLE_REJECTED, + ), + ( + ImportStep.STRUCTURE_VALIDATION, + ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR, + ImportErrorCode.BUNDLE_INCOMPLETE, + ), + ( + ImportStep.REGISTRATION, + ExperimentRunStatus.IMPORT_ERROR, + ImportErrorCode.MODEL_NOT_REGISTERED, + ), + ( + ImportStep.MODEL_DOCUMENT, + ExperimentRunStatus.IMPORT_ERROR, + ImportErrorCode.MODEL_SETTINGS_NOT_SAVED, + ), + ], +) +async def test_a_failure_is_recorded_with_its_step_code_and_reason( + step: ImportStep, status: ExperimentRunStatus, code: ImportErrorCode +): + engine = FakeStatusEngine() + activity = build_activity(engine) + + verdict = await activity.record_import_terminal_status( + {**claim_input(), 'succeeded': False, 'step': step.value, 'gate': 4} + ) + + write = engine.writes[0] + assert write.parameters['status'] == status.value + assert write.parameters['error_message'] == verdict['reason'] + detail = detail_of(write) + assert detail['step'] == step.value + assert detail['code'] == code.value + assert detail['gate'] == 4 + + +@mark.asyncio +async def test_a_cleanup_failure_does_not_turn_a_success_into_an_error( + import_activity: ModelImport, engine: FakeStatusEngine +): + verdict = await import_activity.record_import_terminal_status( + {**claim_input(), 'succeeded': True, 'cleanup_failed': True} + ) + + write = engine.writes[0] + assert write.parameters['status'] == ExperimentRunStatus.IMPORT_SUCCESS.value + assert write.parameters['error_message'] is None + assert detail_of(write)['cleanup_failed'] is True + assert verdict['code'] is None + + +@mark.asyncio +async def test_the_terminal_write_returns_the_verdict_it_wrote( + import_activity: ModelImport, +): + verdict = await import_activity.record_import_terminal_status( + {**claim_input(), 'succeeded': False, 'step': ImportStep.DECRYPTION.value} + ) + + assert verdict == { + 'status': ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR.value, + 'code': ImportErrorCode.BUNDLE_CANNOT_BE_OPENED.value, + 'reason': verdict['reason'], + 'step': ImportStep.DECRYPTION.value, + } + assert 'password' in verdict['reason'] + + +@mark.asyncio +async def test_executing_the_terminal_write_twice_leaves_the_row_the_same( + import_activity: ModelImport, engine: FakeStatusEngine +): + """What retries must be idempotent: two executions of the same verdict, one row state.""" + payload = {**claim_input(), 'succeeded': True, 'step': ImportStep.MODEL_DOCUMENT.value} + + await import_activity.record_import_terminal_status(payload) + await import_activity.record_import_terminal_status(payload) + + first, second = engine.writes + assert first.sql == second.sql + assert {k: v for k, v in first.parameters.items() if k != 'updated_at'} == { + k: v for k, v in second.parameters.items() if k != 'updated_at' + } + + +@mark.asyncio +async def test_every_write_advances_updated_at( + import_activity: ModelImport, engine: FakeStatusEngine +): + stamps = [datetime(2026, 8, 25, 12, 0, 0), datetime(2026, 8, 25, 12, 0, 5)] + with patch('laborious.activities.model_import.now', side_effect=stamps): + await import_activity.record_import_status( + {**claim_input(), 'step': ImportStep.DOWNLOAD.value} + ) + await import_activity.record_import_terminal_status( + {**claim_input(), 'succeeded': True, 'step': ImportStep.MODEL_DOCUMENT.value} + ) + + assert engine.writes[0].parameters['updated_at'] < engine.writes[1].parameters['updated_at'] + + +@mark.asyncio +async def test_no_status_write_touches_a_column_the_inserting_side_owns( + import_activity: ModelImport, engine: FakeStatusEngine +): + engine.rows = [{'run_type': 'IMPORT'}] + + await import_activity.claim_import_status(claim_input()) + await import_activity.record_import_status({**claim_input(), 'step': ImportStep.DOWNLOAD.value}) + await import_activity.record_import_names( + {**claim_input(), 'experiment_name': 'e', 'run_name': 'import-e-v1'} + ) + await import_activity.record_import_terminal_status({**claim_input(), 'succeeded': True}) + + assert len(engine.writes) == 4 + for write in engine.writes: + assert write.assignments & BFF_OWNED_COLUMNS == set() + assert write.table == IMPORT_LOG_TABLE + # One transaction per write: no status write shares a transaction with anything else. + assert engine.transactions == 4 + + +@mark.asyncio +async def test_every_post_claim_write_keys_on_the_id_alone( + import_activity: ModelImport, engine: FakeStatusEngine +): + """Ownership was settled by the claim. Re-checking it later can only lose a real verdict.""" + await import_activity.record_import_status({**claim_input(), 'step': ImportStep.DOWNLOAD.value}) + await import_activity.record_import_names( + {**claim_input(), 'experiment_name': 'e', 'run_name': 'import-e-v1'} + ) + await import_activity.record_import_terminal_status({**claim_input(), 'succeeded': True}) + + for write in engine.writes: + where = write.sql[write.sql.index('WHERE') :] + assert where == 'WHERE id = :import_run_id' + + +@mark.asyncio +async def test_a_row_edited_by_hand_after_the_claim_still_receives_the_verdict( + import_activity: ModelImport, engine: FakeStatusEngine +): + """The predicate carries no status, so an operator's edit cannot orphan the record.""" + engine.rowcounts = [1] + + verdict = await import_activity.record_import_terminal_status( + {**claim_input(), 'succeeded': False, 'step': ImportStep.REGISTRATION.value} + ) + + assert verdict['status'] == ExperimentRunStatus.IMPORT_ERROR.value + assert 'status' not in engine.writes[0].sql[engine.writes[0].sql.index('WHERE') :] + + +@mark.asyncio +@mark.parametrize('error_class', [IntegrityError, ProgrammingError, DataError]) +async def test_a_refused_statement_fails_on_the_first_attempt(error_class): + """A missing `V12` is this: the connection worked and the statement was refused.""" + engine = FakeStatusEngine( + write_errors=[error_class('UPDATE ...', {}, Exception('refused'))], + ) + activity = build_activity(engine) + + with raises(ImportStatusRejectedError) as failure: + await activity.record_import_terminal_status({**claim_input(), 'succeeded': True}) + + assert failure.value.non_retryable is True + assert failure.value.type == 'ImportStatusRejectedError' + assert 'ImportStatusRejectedError' in IMPORT_STATUS_RETRY_POLICY.non_retryable_error_types + + +@mark.asyncio +async def test_a_connection_failure_is_left_to_the_retry(): + """Repeating it is exactly what fixes it, so it must not be swallowed or reclassified.""" + engine = FakeStatusEngine(write_errors=[OperationalError('UPDATE ...', {}, OSError('down'))]) + activity = build_activity(engine) + + with raises(OperationalError): + await activity.record_import_terminal_status({**claim_input(), 'succeeded': True}) + + +@mark.asyncio +async def test_a_status_write_that_fails_then_succeeds_records_the_right_verdict(): + engine = FakeStatusEngine( + write_errors=[OperationalError('UPDATE ...', {}, OSError('down')), None] + ) + activity = build_activity(engine) + payload = {**claim_input(), 'succeeded': False, 'step': ImportStep.REGISTRATION.value} + + with raises(OperationalError): + await activity.record_import_terminal_status(payload) + verdict = await activity.record_import_terminal_status(payload) + + assert verdict['status'] == ExperimentRunStatus.IMPORT_ERROR.value + assert verdict['code'] == ImportErrorCode.MODEL_NOT_REGISTERED.value + assert engine.writes[-1].parameters['status'] == ExperimentRunStatus.IMPORT_ERROR.value + + +@mark.asyncio +async def test_a_verdict_that_cannot_be_written_is_reported_with_the_verdict(): + engine = FakeStatusEngine() + activity = build_activity(engine) + + result = await activity.report_import_status_write_failure( + { + **claim_input(), + 'detail': 'OperationalError: connection refused', + 'verdict': 'succeeded=True step=model_document gate=None', + } + ) + + assert result == {'reported': True} + # The one failure with no row to land on: nothing is written anywhere. + assert engine.statements == [] + logged = activity.logger.custom_error.call_args_list[0].args[0] + assert ImportErrorCode.STATUS_NOT_RECORDED.value in logged + assert 'status_write' in logged + assert 'verdict_not_recorded=succeeded=True step=model_document' in logged + assert 'connection refused' in logged + + +@mark.asyncio +async def test_the_engine_is_disposed_on_shutdown(import_activity: ModelImport, engine): + import_activity.close() + + assert engine.disposed is True diff --git a/tests/laborious/test_import_absence.py b/tests/laborious/test_import_absence.py new file mode 100644 index 0000000..b3a6dce --- /dev/null +++ b/tests/laborious/test_import_absence.py @@ -0,0 +1,264 @@ +"""What the import deliberately does **not** do. + +Every absence here was a decision, and every one of them is the kind of thing a later well-meaning +edit adds back: a rollback for a failed import, a resume for a half-finished one, a signal to retry, +a `joblib.load` to "check the model is valid", a reason string built from the exception. None of +those are missing features — a failed import stays failed, and trying again is a new upload — so the +tests are written against the source itself, because there is no behaviour to observe when the +behaviour is the point of not existing. +""" + +import ast +from pathlib import Path + +from pytest import mark + +from laborious.workflows.import_model import ImportModel + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] + +WORKFLOW_MODULE = REPOSITORY_ROOT / 'laborious/workflows/import_model.py' +ACTIVITIES_MODULE = REPOSITORY_ROOT / 'laborious/activities/model_import.py' +ERRORS_MODULE = REPOSITORY_ROOT / 'laborious/activities/model_import_errors.py' +READER_MODULE = REPOSITORY_ROOT / 'laborious/utils/bundle/reader.py' +FORMAT_MODULE = REPOSITORY_ROOT / 'laborious/utils/bundle/format.py' + +IMPORT_PATH_MODULES = ( + WORKFLOW_MODULE, + ACTIVITIES_MODULE, + ERRORS_MODULE, + READER_MODULE, + FORMAT_MODULE, +) + +# The vocabulary of recovery. A partial import is not repaired; it is reported and abandoned. +RECOVERY_WORDS = ('rollback', 'compensate', 'resume', 'reconcile', 'undo') + +# Loading a model is the *execution* side, owned by another card. The importer moves bytes and +# writes metadata; it never unpickles what it just accepted from a user. +MODEL_LOADING_CALLS = ( + 'mlflow.pyfunc.load_model', + 'mlflow.sklearn.load_model', + 'joblib.load', + 'pickle.load', + 'pickle.loads', +) + +# Decryption lives in the activity that uses the password, never in the workflow, which would put +# the plaintext into the event history. +DECRYPTION_PRIMITIVES = ( + 'AESGCM', + 'crypto_pwhash', + 'crypto_secretstream', + 'decrypt_password_envelope', + 'load_envelope_key', + 'IMPORT_PASSWORD_KEY', + 'Argon2', +) + + +def parse(path: Path) -> ast.Module: + return ast.parse(path.read_text()) + + +def called_names(tree: ast.Module) -> list[str]: + """Every called name, as written: `a.b.c(...)` becomes `a.b.c`.""" + names = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + names.append(ast.unparse(node.func)) + return names + + +def defined_names(tree: ast.Module) -> list[str]: + return [ + node.name + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef) + ] + + +# ------------------------------------------------------------- no retry beyond the record + + +def test_the_workflow_holds_no_recovery_routine(): + tree = parse(WORKFLOW_MODULE) + + for name in defined_names(tree) + called_names(tree): + for word in RECOVERY_WORDS: + assert word not in name.lower(), f'{name} looks like recovery' + + +def test_the_activities_hold_no_recovery_routine(): + tree = parse(ACTIVITIES_MODULE) + + for name in defined_names(tree) + called_names(tree): + for word in RECOVERY_WORDS: + assert word not in name.lower(), f'{name} looks like recovery' + + +def test_the_workflow_never_continues_as_new(): + """A single import is one workflow execution, start to finish.""" + assert not [name for name in called_names(parse(WORKFLOW_MODULE)) if 'continue_as_new' in name] + + +# ------------------------------------------------------------------ no handlers, no waiting + + +def test_the_workflow_declares_no_signal_or_query_handler(): + """Nothing outside can steer an import once it starts: no retry signal, no progress query.""" + for name in dir(ImportModel): + member = getattr(ImportModel, name, None) + for marker in ( + '__temporal_signal_definition', + '__temporal_query_definition', + '__temporal_update_definition', + ): + assert not hasattr(member, marker), f'{name} declares {marker}' + + +def test_the_workflow_module_carries_no_handler_decorator(): + tree = parse(WORKFLOW_MODULE) + decorators = [ + ast.unparse(decorator) + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) + for decorator in node.decorator_list + ] + + assert set(decorators) == {'workflow.run', 'staticmethod'} + assert decorators.count('workflow.run') == 1 + + +def test_the_workflow_never_waits_for_a_condition(): + """No `wait_condition`: there is no external event an import waits on.""" + assert not [name for name in called_names(parse(WORKFLOW_MODULE)) if 'wait_condition' in name] + + +# --------------------------------------------------------------------- nothing is loaded + + +@mark.parametrize('module_path', IMPORT_PATH_MODULES, ids=lambda path: path.name) +def test_no_module_in_the_import_path_loads_a_model(module_path: Path): + source = module_path.read_text() + + for call in MODEL_LOADING_CALLS: + assert call not in source + + +def test_the_reader_knows_nothing_about_mlflow(): + """It stays liftable into a shared package: the reader is about the file, not the platform.""" + source = READER_MODULE.read_text() + + assert 'mlflow' not in source + assert 'temporalio' not in source + + +# ----------------------------------------------------------- the workflow holds no secret + + +@mark.parametrize('primitive', DECRYPTION_PRIMITIVES) +def test_the_workflow_module_holds_no_decryption_primitive(primitive: str): + assert primitive not in WORKFLOW_MODULE.read_text() + + +def test_the_workflow_only_ever_holds_the_envelope(): + """The envelope crosses the workflow; the password it wraps never does.""" + source = WORKFLOW_MODULE.read_text() + + assert 'password_envelope' in source + # The only other mention of a password is the input field the workflow refuses to accept. + assert source.count("'password'") == 1 + + +def test_the_plaintext_password_field_is_refused_by_name(): + from laborious.workflows.import_model import REJECTED_INPUT_FIELDS + + assert 'password' in REJECTED_INPUT_FIELDS + + +# ------------------------------------------------------- every reason comes from the catalog + + +def _error_message_values(tree: ast.Module) -> list[ast.expr]: + """Every expression written into the record's `error_message`.""" + values = [] + for node in ast.walk(tree): + if isinstance(node, ast.Dict): + for key, value in zip(node.keys, node.values, strict=True): + if isinstance(key, ast.Constant) and key.value == 'error_message': + values.append(value) + if isinstance(node, ast.keyword) and node.arg == 'error_message': + values.append(node.value) + return values + + +def test_no_recorded_reason_is_built_from_an_exception(): + """The catalog is the only source of a sentence a user reads.""" + values = _error_message_values(parse(ACTIVITIES_MODULE)) + + assert values, 'no error_message write found — has the terminal statement moved?' + for value in values: + assert isinstance(value, ast.Name | ast.Constant), ast.unparse(value) + assert not isinstance(value, ast.JoinedStr | ast.BinOp) + + +def test_the_reason_written_is_the_classified_one(): + values = _error_message_values(parse(ACTIVITIES_MODULE)) + + assert {ast.unparse(value) for value in values} == {'reason'} + + +def test_the_classification_ignores_the_exception_by_construction(): + """`classify_import_failure` deletes its exception argument; nothing can leak from it.""" + source = ERRORS_MODULE.read_text() + + assert 'del exception' in source + + +@mark.parametrize('module_path', (WORKFLOW_MODULE, ACTIVITIES_MODULE), ids=lambda p: p.name) +def test_no_status_detail_interpolates_an_exception(module_path: Path): + """The JSON detail is built from a code and a step, both of them enum values.""" + tree = parse(module_path) + + for node in ast.walk(tree): + if not isinstance(node, ast.Call) or ast.unparse(node.func) != 'build_import_detail': + continue + for keyword in node.keywords: + rendered = ast.unparse(keyword.value) + assert 'error' not in rendered + assert 'exception' not in rendered + assert not isinstance(keyword.value, ast.JoinedStr) + + +# ------------------------------------------------------------------ no relational target + + +def test_the_import_writes_to_one_table_and_names_it_once(): + """D22: no project table, no model table, no second database. One `UPDATE`, one relation.""" + statements = [] + for node in ast.walk(parse(ACTIVITIES_MODULE)): + if not isinstance(node, ast.Call) or ast.unparse(node.func) != 'text': + continue + argument = node.args[0] + if isinstance(argument, ast.Constant): + statements.append(argument.value) + + assert statements, 'no SQL statement found in the import activities' + for statement in statements: + words = statement.split() + relations = { + words[index + 1] + for index, word in enumerate(words) + if word.upper() in ('UPDATE', 'FROM', 'INTO', 'JOIN') + } + assert relations == {'public.experiment_run'} + + +def test_the_import_never_inserts_a_relational_row(): + source = ACTIVITIES_MODULE.read_text().upper() + + assert 'INSERT INTO' not in source + assert 'DELETE FROM' not in source + assert 'CREATE TABLE' not in source diff --git a/tests/laborious/worker/test_runtime_task_queues.py b/tests/laborious/worker/test_runtime_task_queues.py index 8b40d4a..37a46cd 100644 --- a/tests/laborious/worker/test_runtime_task_queues.py +++ b/tests/laborious/worker/test_runtime_task_queues.py @@ -1,10 +1,27 @@ +"""Queue names and how many workers a runtime starts. + +Every queue is runtime-scoped, so two runtimes never poll each other's work. The import worker is +the only conditional one: unset, the four default workers are exactly what they were, and no import +queue is polled at all. +""" + +import ast +from pathlib import Path + +from pytest import mark from sientia_do.temporal.worker.prepare_worker import build_queue_name +from laborious.worker.worker import import_worker_enabled from laborious.workflows.drift import Drift +from laborious.workflows.import_model import ImportModel from laborious.workflows.minimal_retrain import MinimalRetrain from laborious.workflows.predictions_batch import PredictionsBatch from laborious.workflows.simple_metrics import SimpleMetrics +WORKER_MODULE = Path(__file__).resolve().parents[3] / 'laborious/worker/worker.py' + +DEFAULT_WORKFLOWS = (PredictionsBatch, MinimalRetrain, Drift, SimpleMetrics) + def test_runtime_scoped_queue_names(): runtime = 'prod-a' @@ -12,3 +29,86 @@ def test_runtime_scoped_queue_names(): assert build_queue_name(MinimalRetrain.__name__, runtime) == 'minimal_retrain-prod-a-queue' assert build_queue_name(Drift.__name__, runtime) == 'drift-prod-a-queue' assert build_queue_name(SimpleMetrics.__name__, runtime) == 'simple_metrics-prod-a-queue' + + +def test_the_import_queue_shares_the_runtime_suffix(): + assert build_queue_name(ImportModel.__name__, 'prod-a') == 'import_model-prod-a-queue' + + +def test_every_queue_carries_the_same_runtime(): + runtime = 'prod-a' + queues = [ + build_queue_name(workflow.__name__, runtime) + for workflow in (*DEFAULT_WORKFLOWS, ImportModel) + ] + + assert all(queue.endswith(f'-{runtime}-queue') for queue in queues) + assert len(set(queues)) == 5 + + +@mark.parametrize('value', ['1', 'true', 'TRUE', 'yes', 'on', ' true ']) +def test_the_import_worker_is_registered_when_the_flag_says_so(monkeypatch, value: str): + monkeypatch.setenv('IMPORT_MODEL_ENABLED', value) + + assert import_worker_enabled() is True + + +@mark.parametrize('value', ['', '0', 'false', 'no', 'off', 'maybe']) +def test_the_import_worker_stays_out_by_default(monkeypatch, value: str): + monkeypatch.setenv('IMPORT_MODEL_ENABLED', value) + + assert import_worker_enabled() is False + + +def test_an_unset_flag_leaves_the_import_worker_out(monkeypatch): + monkeypatch.delenv('IMPORT_MODEL_ENABLED', raising=False) + + assert import_worker_enabled() is False + + +def _prepare_worker_calls(tree: ast.Module) -> tuple[int, int]: + """How many workers are built unconditionally, and how many behind the flag.""" + conditional = 0 + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + if 'import_worker_enabled' not in ast.unparse(node.test): + continue + conditional += sum( + 1 + for inner in ast.walk(node) + if isinstance(inner, ast.Call) and ast.unparse(inner.func) == 'prepare_worker' + ) + total = sum( + 1 + for node in ast.walk(tree) + if isinstance(node, ast.Call) and ast.unparse(node.func) == 'prepare_worker' + ) + return total - conditional, conditional + + +def test_four_workers_by_default_and_five_with_the_flag(): + unconditional, conditional = _prepare_worker_calls(ast.parse(WORKER_MODULE.read_text())) + + assert unconditional == 4 + assert conditional == 1 + + +def test_the_import_worker_registers_only_the_import_activities(): + """The containment boundary: the import runtime polls one queue and holds one workflow.""" + tree = ast.parse(WORKER_MODULE.read_text()) + activities: list[str] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.If) or 'import_worker_enabled' not in ast.unparse(node.test): + continue + for inner in ast.walk(node): + if isinstance(inner, ast.keyword) and inner.arg == 'activities': + activities = [ast.unparse(element) for element in inner.value.elts] + if isinstance(inner, ast.keyword) and inner.arg == 'main_workflow': + assert ast.unparse(inner.value) == 'ImportModel' + if isinstance(inner, ast.keyword) and inner.arg == 'other_workflows': + assert ast.unparse(inner.value) == '[]' + + assert len(activities) == 13 + assert all(name.startswith('activities.') for name in activities) + assert all('import' in name for name in activities) diff --git a/tests/laborious/workflows/test_import_model.py b/tests/laborious/workflows/test_import_model.py new file mode 100644 index 0000000..0f426d3 --- /dev/null +++ b/tests/laborious/workflows/test_import_model.py @@ -0,0 +1,781 @@ +"""The `import_model` workflow: order, fail-fast, and which calls are allowed to retry. + +The workflow itself holds no logic worth testing in isolation except the two things that can only be +wrong here: **the order** — the claim before anything else, the names before MLflow, the document +last, cleanup and then one terminal write in the `finally` — and **the retry split**, where a single +misplaced policy either replays a registry call or spends 151 seconds on a hint nobody reads. + +Everything is asserted through the recorded activity calls, because that is exactly what the +workflow is: a sequence of activity invocations with policies attached. +""" + +from datetime import timedelta +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from pytest import fixture, mark, raises +from sientia_do.temporal.policies import retry_policy + +from laborious.activities.activities import Activities +from laborious.activities.model_import import ( + IMPORT_STATUS_HINT_RETRY_POLICY, + IMPORT_STATUS_RETRY_POLICY, +) +from laborious.activities.model_import_errors import ( + ImportGateRejectedError, + ImportInputError, + ImportLogRowNotClaimableError, +) +from laborious.utils.bundle.steps import ImportStep +from laborious.workflows.import_model import ( + REJECTED_INPUT_FIELDS, + STATUS_WRITE_ENVELOPE, + STATUS_WRITE_TIMEOUT, + ImportModel, +) + +WORKFLOW_ID = 'import_model-77-abc' +RUN_ID = '0123456789abcdef0123456789abcdef' +DIGEST = 'a' * 64 +OBJECT_KEY = 'imported_models/factory_model_v2.sientia' + +BUNDLE_METADATA = { + 'model_name': 'factory_model', + 'experiment_name': 'factory_experiment', + 'run_id': 'aa11bb22cc33dd44ee55ff6677889900', + 'model_version': '2', + 'model_project': 'factory_project', + 'export_timestamp': '2026-08-25T12:00:00+00:00', +} +BUNDLE_PARAMETERS = {'model_name': 'factory_model', 'target_variable': 'SE'} + +# The activities that write the import log row. These four are the only calls allowed to retry. +STATUS_ACTIVITIES = ( + Activities.claim_import_status, + Activities.record_import_status, + Activities.record_import_names, + Activities.record_import_terminal_status, +) +CLAIM_AND_TERMINAL = (Activities.claim_import_status, Activities.record_import_terminal_status) +HINTS = (Activities.record_import_status, Activities.record_import_names) + +# What a correct import does, in order. +EXPECTED_ORDER = [ + Activities.claim_import_status, + Activities.record_import_status, + Activities.ensure_import_bundle_retention, + Activities.download_import_bundle, + Activities.record_import_status, + Activities.open_import_bundle, + Activities.record_import_names, + Activities.record_import_status, + Activities.create_import_experiment, + Activities.record_import_status, + Activities.upload_import_artifacts, + Activities.record_import_status, + Activities.register_import_model_version, + Activities.record_import_status, + Activities.write_import_model_document, + Activities.cleanup_import_files, + Activities.record_import_terminal_status, +] + +PROVISIONING = ( + Activities.create_import_experiment, + Activities.upload_import_artifacts, + Activities.register_import_model_version, + Activities.write_import_model_document, +) + +HAPPY_RESULTS = { + Activities.claim_import_status: { + 'claimed': True, + 'bucket_name': None, + 'file_name': None, + 'request_digest': None, + }, + Activities.download_import_bundle: { + 'path': '/work/bundle.sientia', + 'size': 4096, + 'digest': DIGEST, + }, + Activities.open_import_bundle: { + 'extracted_dir': '/work/extract', + 'metadata': BUNDLE_METADATA, + 'parameters': BUNDLE_PARAMETERS, + 'digest': DIGEST, + }, + Activities.create_import_experiment: {'experiment_id': '17'}, + Activities.upload_import_artifacts: {'run_id': RUN_ID, 'run_name': 'import-factory_model-v2'}, + Activities.register_import_model_version: {'model_name': 'factory_model', 'version': '4'}, + Activities.write_import_model_document: {'written': True, 'id': '8'}, + Activities.cleanup_import_files: {'cleaned': True, 'existed': True}, + Activities.record_import_terminal_status: {'status': 'IMPORT_SUCCESS'}, +} + + +class Recorder: + """Stands in for `workflow.execute_activity_method`, recording every invocation.""" + + def __init__(self, failures: dict | None = None, results: dict | None = None) -> None: + self.results = {**HAPPY_RESULTS, **(results or {})} + self.failures = dict(failures or {}) + self.calls: list[SimpleNamespace] = [] + + async def __call__(self, method, payload, **kwargs): + self.calls.append(SimpleNamespace(method=method, payload=payload, kwargs=kwargs)) + if method in self.failures: + raise self.failures[method] + return self.results.get(method, {}) + + @property + def order(self) -> list: + return [call.method for call in self.calls] + + def payload_of(self, method): + return next(call.payload for call in self.calls if call.method is method) + + def last_payload_of(self, method): + return [call.payload for call in self.calls if call.method is method][-1] + + def calls_to(self, method) -> list[SimpleNamespace]: + return [call for call in self.calls if call.method is method] + + +def valid_input(**overrides) -> dict: + return { + 'import_run_id': 77, + 'object_key': OBJECT_KEY, + 'expected_digest': DIGEST, + 'password_envelope': 'ZW52ZWxvcGU=', + **overrides, + } + + +@fixture +def import_workflow() -> ImportModel: + return ImportModel() + + +def run_with(recorder: Recorder): + """Patch the workflow module's `workflow` object and hand back the patch context.""" + patcher = patch('laborious.workflows.import_model.workflow') + workflow_mock = patcher.start() + workflow_mock.info.return_value = MagicMock(workflow_id=WORKFLOW_ID) + # Assigned directly rather than wrapped in an `AsyncMock`: a mock whose side effect is an + # instance with an `async def __call__` hands back the coroutine instead of awaiting it. + workflow_mock.execute_activity_method = recorder + return patcher, workflow_mock + + +@fixture +def recorder(): + return Recorder() + + +@fixture +def workflow_env(recorder: Recorder): + patcher, workflow_mock = run_with(recorder) + yield SimpleNamespace(recorder=recorder, workflow=workflow_mock) + patcher.stop() + + +# ------------------------------------------------------------------------------- order + + +@mark.asyncio +async def test_a_successful_import_calls_its_activities_in_order( + import_workflow: ImportModel, workflow_env +): + result = await import_workflow.run(valid_input()) + + assert workflow_env.recorder.order == EXPECTED_ORDER + assert result == { + 'import_run_id': 77, + 'run_id': RUN_ID, + 'model_name': 'factory_model', + 'version': '4', + 'document_id': '8', + } + + +@mark.asyncio +async def test_the_claim_is_the_first_thing_that_happens( + import_workflow: ImportModel, workflow_env +): + await import_workflow.run(valid_input()) + + assert workflow_env.recorder.order[0] is Activities.claim_import_status + + +@mark.asyncio +async def test_the_names_are_recorded_before_mlflow_is_touched( + import_workflow: ImportModel, workflow_env +): + """So a person watching the import list sees which model is being imported, while it runs.""" + await import_workflow.run(valid_input()) + + order = workflow_env.recorder.order + assert order.index(Activities.record_import_names) < order.index( + Activities.create_import_experiment + ) + names = workflow_env.recorder.payload_of(Activities.record_import_names) + assert names['experiment_name'] == 'factory_experiment' + assert names['run_name'] == 'import-factory_model-v2' + + +@mark.asyncio +async def test_the_run_is_created_with_exactly_the_recorded_name( + import_workflow: ImportModel, workflow_env +): + await import_workflow.run(valid_input()) + + recorded = workflow_env.recorder.payload_of(Activities.record_import_names)['run_name'] + used = workflow_env.recorder.payload_of(Activities.upload_import_artifacts)['run_name'] + assert used == recorded + + +@mark.asyncio +async def test_the_model_document_is_the_last_thing_provisioned( + import_workflow: ImportModel, workflow_env +): + await import_workflow.run(valid_input()) + + order = workflow_env.recorder.order + for step in PROVISIONING[:-1]: + assert order.index(step) < order.index(Activities.write_import_model_document) + assert order[-2:] == [Activities.cleanup_import_files, Activities.record_import_terminal_status] + + +@mark.asyncio +async def test_the_document_target_comes_from_the_bundle_s_parameters( + import_workflow: ImportModel, workflow_env +): + await import_workflow.run(valid_input()) + + assert ( + workflow_env.recorder.payload_of(Activities.write_import_model_document)['target'] == 'SE' + ) + + +@mark.asyncio +async def test_the_terminal_write_reports_a_success_with_its_last_step( + import_workflow: ImportModel, workflow_env +): + await import_workflow.run(valid_input()) + + terminal = workflow_env.recorder.payload_of(Activities.record_import_terminal_status) + assert terminal['succeeded'] is True + assert terminal['step'] == ImportStep.MODEL_DOCUMENT.value + assert terminal['cleanup_failed'] is False + assert terminal['source'] == BUNDLE_METADATA + + +@mark.asyncio +async def test_the_download_is_handed_the_digest_the_request_carried( + import_workflow: ImportModel, workflow_env +): + await import_workflow.run(valid_input()) + + payload = workflow_env.recorder.payload_of(Activities.download_import_bundle) + assert payload['expected_digest'] == DIGEST + assert payload['object_key'] == OBJECT_KEY + + +# --------------------------------------------------------------------------- fail-fast + + +@mark.asyncio +@mark.parametrize( + ('failing', 'step'), + [ + (Activities.download_import_bundle, ImportStep.DOWNLOAD), + (Activities.open_import_bundle, ImportStep.DECRYPTION), + (Activities.create_import_experiment, ImportStep.EXPERIMENT_CREATION), + (Activities.upload_import_artifacts, ImportStep.ARTIFACT_UPLOAD), + (Activities.register_import_model_version, ImportStep.REGISTRATION), + (Activities.write_import_model_document, ImportStep.MODEL_DOCUMENT), + ], + ids=lambda value: getattr(value, '__name__', str(value)), +) +async def test_a_failing_step_is_recorded_as_that_step_and_stops_the_import( + import_workflow: ImportModel, failing, step: ImportStep +): + recorder = Recorder(failures={failing: RuntimeError('dependency unavailable')}) + patcher, _ = run_with(recorder) + try: + with raises(RuntimeError, match='dependency unavailable'): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + terminal = recorder.payload_of(Activities.record_import_terminal_status) + assert terminal['succeeded'] is False + assert terminal['step'] == step.value + # Nothing after the failing step was attempted, and cleanup still ran before the verdict. + order = recorder.order + assert order[-2:] == [Activities.cleanup_import_files, Activities.record_import_terminal_status] + later = EXPECTED_ORDER[EXPECTED_ORDER.index(failing) + 1 :] + for method in later: + if method in (Activities.cleanup_import_files, Activities.record_import_terminal_status): + continue + if method in STATUS_ACTIVITIES: + continue + assert method not in order + + +@mark.asyncio +async def test_nothing_is_provisioned_before_the_gates_pass(import_workflow: ImportModel): + recorder = Recorder( + failures={ + Activities.open_import_bundle: ImportGateRejectedError( + 'rejected', step=ImportStep.ARCHIVE_INSPECTION, gate=4 + ) + } + ) + patcher, _ = run_with(recorder) + try: + with raises(ImportGateRejectedError): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + for method in PROVISIONING: + assert method not in recorder.order + + +@mark.asyncio +async def test_a_gate_rejection_is_recorded_with_the_gate_that_rejected_it( + import_workflow: ImportModel, +): + """Gates 2 to 7 share one activity, so the step travels in the error's details, not in the + step the workflow was attempting.""" + recorder = Recorder( + failures={ + Activities.open_import_bundle: ImportGateRejectedError( + 'rejected', step=ImportStep.CONTENT_POLICY, gate=7 + ) + } + ) + patcher, _ = run_with(recorder) + try: + with raises(ImportGateRejectedError): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + terminal = recorder.payload_of(Activities.record_import_terminal_status) + assert terminal['step'] == ImportStep.CONTENT_POLICY.value + assert terminal['gate'] == 7 + + +@mark.asyncio +async def test_an_ambiguous_decryption_failure_keeps_its_step(import_workflow: ImportModel): + recorder = Recorder( + failures={ + Activities.open_import_bundle: ImportGateRejectedError( + 'the bundle could not be opened', step=ImportStep.DECRYPTION, gate=3 + ) + } + ) + patcher, _ = run_with(recorder) + try: + with raises(ImportGateRejectedError): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + terminal = recorder.payload_of(Activities.record_import_terminal_status) + assert terminal['step'] == ImportStep.DECRYPTION.value + assert terminal['gate'] == 3 + + +# ------------------------------------------------------------------- the input contract + + +@mark.asyncio +@mark.parametrize('field', list(REJECTED_INPUT_FIELDS)) +async def test_an_unknown_field_is_rejected_rather_than_ignored( + import_workflow: ImportModel, field: str +): + """A caller that believes it recorded a project link must be told it did not.""" + recorder = Recorder() + patcher, _ = run_with(recorder) + try: + with raises(ImportInputError, match='does not accept'): + await import_workflow.run(valid_input(**{field: 'whatever'})) + finally: + patcher.stop() + + # After the claim, so the refusal lands on the record instead of only in the logs. + assert recorder.order[0] is Activities.claim_import_status + assert Activities.download_import_bundle not in recorder.order + terminal = recorder.payload_of(Activities.record_import_terminal_status) + assert terminal['step'] == ImportStep.RECEIVED.value + assert terminal['succeeded'] is False + + +@mark.asyncio +@mark.parametrize( + 'digest', + [None, '', 'not-a-digest', 'a' * 63, 'a' * 65, 'g' * 64], + ids=['missing', 'empty', 'short-text', 'too-short', 'too-long', 'not-hex'], +) +async def test_a_missing_or_malformed_digest_fails_before_anything_is_downloaded( + import_workflow: ImportModel, digest +): + recorder = Recorder() + patcher, _ = run_with(recorder) + try: + with raises(ImportInputError): + await import_workflow.run(valid_input(expected_digest=digest)) + finally: + patcher.stop() + + assert Activities.download_import_bundle not in recorder.order + assert ( + recorder.payload_of(Activities.record_import_terminal_status)['step'] + == ImportStep.RECEIVED.value + ) + + +@mark.asyncio +@mark.parametrize('missing', ['object_key', 'password_envelope']) +async def test_a_request_missing_a_required_field_fails_at_received( + import_workflow: ImportModel, missing: str +): + recorder = Recorder() + patcher, _ = run_with(recorder) + payload = valid_input() + payload.pop(missing) + try: + with raises(ImportInputError, match=missing): + await import_workflow.run(payload) + finally: + patcher.stop() + + assert Activities.download_import_bundle not in recorder.order + + +@mark.asyncio +async def test_a_record_naming_another_object_is_a_received_failure( + import_workflow: ImportModel, +): + recorder = Recorder( + results={ + Activities.claim_import_status: { + 'claimed': True, + 'file_name': 'imported_models/somebody_elses.sientia', + 'request_digest': None, + } + } + ) + patcher, _ = run_with(recorder) + try: + with raises(ImportInputError, match='different object'): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + assert Activities.download_import_bundle not in recorder.order + + +@mark.asyncio +async def test_a_record_naming_another_digest_is_a_received_failure( + import_workflow: ImportModel, +): + recorder = Recorder( + results={ + Activities.claim_import_status: { + 'claimed': True, + 'file_name': OBJECT_KEY, + 'request_digest': 'b' * 64, + } + } + ) + patcher, _ = run_with(recorder) + try: + with raises(ImportInputError, match='different digest'): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + assert Activities.download_import_bundle not in recorder.order + + +@mark.asyncio +@mark.parametrize('value', [None, 0, -1, '77', 7.0, True], ids=lambda v: f'{type(v).__name__}-{v}') +async def test_an_unusable_record_id_fails_before_any_activity(import_workflow: ImportModel, value): + """There is no record to claim and none to write: the run ends FAILED and nothing is touched.""" + recorder = Recorder() + patcher, _ = run_with(recorder) + try: + with raises(ImportInputError, match='positive integer'): + await import_workflow.run(valid_input(import_run_id=value)) + finally: + patcher.stop() + + assert recorder.calls == [] + + +@mark.asyncio +async def test_a_claim_that_matches_nothing_writes_nothing_at_all(import_workflow: ImportModel): + recorder = Recorder( + failures={ + Activities.claim_import_status: ImportLogRowNotClaimableError( + 'import log row 77 could not be claimed: no such import log row' + ) + } + ) + patcher, _ = run_with(recorder) + try: + with raises(ImportLogRowNotClaimableError): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + # The claim is outside the try/finally on purpose: no row is owned, so there is nothing to clean + # up and nothing to record. + assert recorder.order == [Activities.claim_import_status] + + +# ----------------------------------------------------------- failures that must not fail + + +@mark.asyncio +async def test_a_lost_hint_does_not_fail_a_working_import(import_workflow: ImportModel): + recorder = Recorder(failures={Activities.record_import_status: RuntimeError('db down')}) + patcher, _ = run_with(recorder) + try: + result = await import_workflow.run(valid_input()) + finally: + patcher.stop() + + assert result['run_id'] == RUN_ID + assert Activities.report_import_status_write_failure in recorder.order + assert recorder.payload_of(Activities.record_import_terminal_status)['succeeded'] is True + + +@mark.asyncio +async def test_a_lost_names_write_does_not_fail_a_working_import(import_workflow: ImportModel): + recorder = Recorder(failures={Activities.record_import_names: RuntimeError('db down')}) + patcher, _ = run_with(recorder) + try: + result = await import_workflow.run(valid_input()) + finally: + patcher.stop() + + assert result['version'] == '4' + assert Activities.upload_import_artifacts in recorder.order + + +@mark.asyncio +async def test_a_retention_rule_that_cannot_be_set_does_not_fail_an_import( + import_workflow: ImportModel, +): + recorder = Recorder( + failures={Activities.ensure_import_bundle_retention: RuntimeError('access denied')} + ) + patcher, _ = run_with(recorder) + try: + result = await import_workflow.run(valid_input()) + finally: + patcher.stop() + + assert result['run_id'] == RUN_ID + + +@mark.asyncio +async def test_a_cleanup_failure_is_carried_into_the_verdict_not_raised( + import_workflow: ImportModel, +): + recorder = Recorder(failures={Activities.cleanup_import_files: RuntimeError('busy')}) + patcher, _ = run_with(recorder) + try: + result = await import_workflow.run(valid_input()) + finally: + patcher.stop() + + terminal = recorder.payload_of(Activities.record_import_terminal_status) + assert result['run_id'] == RUN_ID + assert terminal['succeeded'] is True + assert terminal['cleanup_failed'] is True + + +@mark.asyncio +async def test_a_terminal_write_that_cannot_land_does_not_invent_a_failed_import( + import_workflow: ImportModel, +): + recorder = Recorder( + failures={Activities.record_import_terminal_status: RuntimeError('still down')} + ) + patcher, _ = run_with(recorder) + try: + result = await import_workflow.run(valid_input()) + finally: + patcher.stop() + + assert result['run_id'] == RUN_ID + reported = recorder.payload_of(Activities.report_import_status_write_failure) + assert reported['verdict'] == 'succeeded=True step=model_document gate=None' + assert 'still down' in reported['detail'] + + +@mark.asyncio +async def test_a_terminal_write_failure_does_not_mask_a_real_failure( + import_workflow: ImportModel, +): + recorder = Recorder( + failures={ + Activities.register_import_model_version: RuntimeError('registry refused'), + Activities.record_import_terminal_status: RuntimeError('status db down'), + } + ) + patcher, _ = run_with(recorder) + try: + with raises(RuntimeError, match='registry refused'): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + reported = recorder.payload_of(Activities.report_import_status_write_failure) + assert reported['verdict'] == 'succeeded=False step=registration gate=None' + + +@mark.asyncio +async def test_reporting_the_report_is_where_it_stops(import_workflow: ImportModel): + """A failure in the failure channel cannot become the import's outcome.""" + recorder = Recorder( + failures={ + Activities.record_import_terminal_status: RuntimeError('down'), + Activities.report_import_status_write_failure: RuntimeError('also down'), + } + ) + patcher, _ = run_with(recorder) + try: + result = await import_workflow.run(valid_input()) + finally: + patcher.stop() + + assert result['run_id'] == RUN_ID + + +# ------------------------------------------------------------------- the retry split + + +@mark.asyncio +async def test_only_the_status_writes_retry(import_workflow: ImportModel, workflow_env): + await import_workflow.run(valid_input()) + + for call in workflow_env.recorder.calls: + if call.method in STATUS_ACTIVITIES: + continue + assert call.kwargs['retry_policy'] is retry_policy + assert retry_policy.maximum_attempts == 1 + + +@mark.asyncio +async def test_the_claim_and_the_terminal_write_use_the_long_envelope( + import_workflow: ImportModel, workflow_env +): + await import_workflow.run(valid_input()) + + for method in CLAIM_AND_TERMINAL: + for call in workflow_env.recorder.calls_to(method): + assert call.kwargs['retry_policy'] is IMPORT_STATUS_RETRY_POLICY + assert call.kwargs['start_to_close_timeout'] == STATUS_WRITE_TIMEOUT + assert call.kwargs['schedule_to_close_timeout'] == STATUS_WRITE_ENVELOPE + + +@mark.asyncio +async def test_the_hints_use_the_short_policy(import_workflow: ImportModel, workflow_env): + await import_workflow.run(valid_input()) + + for method in HINTS: + for call in workflow_env.recorder.calls_to(method): + assert call.kwargs['retry_policy'] is IMPORT_STATUS_HINT_RETRY_POLICY + # No long envelope: a hint that cannot land must not hold the import open. + assert 'schedule_to_close_timeout' not in call.kwargs + + +@mark.asyncio +async def test_no_provisioning_call_references_a_retrying_policy( + import_workflow: ImportModel, workflow_env +): + """A repeated registry call or Mongo insert writes twice; that is why they cannot retry.""" + await import_workflow.run(valid_input()) + + for call in workflow_env.recorder.calls: + if call.method in PROVISIONING or call.method is Activities.open_import_bundle: + assert call.kwargs['retry_policy'] not in ( + IMPORT_STATUS_RETRY_POLICY, + IMPORT_STATUS_HINT_RETRY_POLICY, + ) + + +@mark.asyncio +async def test_the_only_retrying_calls_are_the_four_status_writes( + import_workflow: ImportModel, workflow_env +): + await import_workflow.run(valid_input()) + + retrying = { + call.method + for call in workflow_env.recorder.calls + if call.kwargs['retry_policy'] + in (IMPORT_STATUS_RETRY_POLICY, IMPORT_STATUS_HINT_RETRY_POLICY) + } + assert retrying == set(STATUS_ACTIVITIES) + + +def test_the_retry_policies_are_the_values_the_design_chose(): + assert IMPORT_STATUS_RETRY_POLICY.maximum_attempts == 10 + assert IMPORT_STATUS_RETRY_POLICY.initial_interval == timedelta(seconds=1) + assert IMPORT_STATUS_RETRY_POLICY.backoff_coefficient == 2.0 + assert IMPORT_STATUS_RETRY_POLICY.maximum_interval == timedelta(seconds=30) + assert IMPORT_STATUS_HINT_RETRY_POLICY.maximum_attempts == 3 + assert IMPORT_STATUS_HINT_RETRY_POLICY.maximum_interval == timedelta(seconds=2) + + +def test_the_terminal_write_s_envelope_outlasts_its_own_retries(): + """Set this smaller and Temporal cuts the retries short, silently undoing the retry split.""" + backoff = sum( + min( + IMPORT_STATUS_RETRY_POLICY.initial_interval.total_seconds() + * IMPORT_STATUS_RETRY_POLICY.backoff_coefficient**attempt, + IMPORT_STATUS_RETRY_POLICY.maximum_interval.total_seconds(), + ) + for attempt in range(IMPORT_STATUS_RETRY_POLICY.maximum_attempts - 1) + ) + + assert backoff == 151 + assert STATUS_WRITE_ENVELOPE.total_seconds() > backoff + + +def test_a_hint_gives_up_inside_a_short_bound(): + """Three attempts capped at two seconds: a lost hint costs the import seconds, not minutes.""" + backoff = sum( + min( + IMPORT_STATUS_HINT_RETRY_POLICY.initial_interval.total_seconds() + * IMPORT_STATUS_HINT_RETRY_POLICY.backoff_coefficient**attempt, + IMPORT_STATUS_HINT_RETRY_POLICY.maximum_interval.total_seconds(), + ) + for attempt in range(IMPORT_STATUS_HINT_RETRY_POLICY.maximum_attempts - 1) + ) + + assert backoff <= 3 + + +@mark.asyncio +async def test_no_retrying_call_carries_a_payload_that_would_insert( + import_workflow: ImportModel, workflow_env +): + """What retries must be idempotent, so the payload may not name a row to create.""" + await import_workflow.run(valid_input()) + + for call in workflow_env.recorder.calls: + if call.method not in STATUS_ACTIVITIES: + continue + assert call.payload['import_run_id'] == 77 + for owned in ('username', 'request_data', 'bucket_name', 'file_name', 'run_type'): + assert owned not in call.payload diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scripts/test_import_model_manual_run.py b/tests/scripts/test_import_model_manual_run.py new file mode 100644 index 0000000..2d61784 --- /dev/null +++ b/tests/scripts/test_import_model_manual_run.py @@ -0,0 +1,217 @@ +"""The anti-rot guard for `scripts/import_model_manual_run.py`. + +The script itself is only ever smoke-run by a person (task 13.6 of `import-model-e2e-suite`), so +nothing else would notice the three things most likely to be quietly broken in it: + +1. the `# %%` cell layout — the whole contract is "cell N does step N", and a lost or reordered + marker turns a ten-step procedure into an unreadable one; +2. the boundary of design E15 — the script stands in for the frontend, so borrowing + `laborious.activities` or `laborious.workflows` would make it exercise the implementation instead + of the contract, and that is exactly the shortcut somebody reaches for when a cell needs one more + value; +3. the configuration rule — no host, user, password or key literal in the file, every target from + the environment, and every variable it reads documented in `.env.example`. + +The wire shapes the script produces (the digest, the envelope, the object key, the row) are asserted +by the e2e suite against the real workflow, so they are deliberately not re-asserted here. This file +is about the script staying the *kind of thing* it is. +""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT_PATH = REPO_ROOT / 'scripts' / 'import_model_manual_run.py' +ENV_EXAMPLE_PATH = REPO_ROOT / '.env.example' + +CELL_MARKER = '# %%' +EXPECTED_CELLS = 10 + +FORBIDDEN_IMPORT_ROOTS = ('laborious.activities', 'laborious.workflows') +ALLOWED_LABORIOUS_IMPORT = 'laborious.utils.bundle.format' + +# Substrings that would mean a target or a credential was written into the file instead of read +# from the environment. `mongodb://` and `postgresql` are schemes, not hosts, so they are not here. +FORBIDDEN_LITERAL_SUBSTRINGS = ( + 'localhost', + '127.0.0.1', + '0.0.0.0', # noqa: S104 - a denylist entry, not an interface this test binds + '::1', + '.svc.cluster.local', + 'minioadmin', + 'mlops_bff_user', +) + +# A dotted host with a port (`db.example.com:5432`), an IPv4 address, or inline URL credentials. +HOSTLIKE_PATTERNS = ( + re.compile(r'[A-Za-z0-9][A-Za-z0-9.-]*\.[A-Za-z]{2,}:\d{2,5}'), + re.compile(r'\b\d{1,3}(?:\.\d{1,3}){3}\b'), + re.compile(r'//[^/\s]+:[^/\s]+@'), +) + + +@pytest.fixture(scope='module') +def source() -> str: + """The script's source text.""" + return SCRIPT_PATH.read_text(encoding='utf-8') + + +@pytest.fixture(scope='module') +def tree(source: str) -> ast.Module: + """The script parsed. A syntax error here is itself the finding — the module must be runnable.""" + return ast.parse(source, filename=str(SCRIPT_PATH)) + + +def _string_literals(tree: ast.Module) -> list[str]: + """Every string constant in the module, comments and docstrings excluded.""" + docstrings = set() + for node in ast.walk(tree): + if isinstance(node, ast.Module | ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): + body = getattr(node, 'body', []) + if ( + body + and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant) + and isinstance(body[0].value.value, str) + ): + docstrings.add(id(body[0].value)) + return [ + node.value + for node in ast.walk(tree) + if isinstance(node, ast.Constant) + and isinstance(node.value, str) + and id(node) not in docstrings + ] + + +def _env_names(tree: ast.Module) -> set[str]: + """Every environment variable the script reads, by name.""" + names: set[str] = set() + readers = {'env', 'env_required', 'getenv'} + for node in ast.walk(tree): + if not isinstance(node, ast.Call) or not node.args: + continue + target = node.func + called = ( + target.attr + if isinstance(target, ast.Attribute) + else target.id + if isinstance(target, ast.Name) + else '' + ) + first = node.args[0] + if called in readers and isinstance(first, ast.Constant) and isinstance(first.value, str): + names.add(first.value) + return names + + +class TestCellLayout: + """The `# %%` contract of design E15.""" + + def test_script_exists(self) -> None: + assert SCRIPT_PATH.is_file(), f'{SCRIPT_PATH} is missing' + + def test_every_cell_marker_is_present(self, source: str) -> None: + markers = [line for line in source.splitlines() if line.strip() == CELL_MARKER] + assert len(markers) == EXPECTED_CELLS, ( + f'expected {EXPECTED_CELLS} `{CELL_MARKER}` cell markers, found {len(markers)}' + ) + + def test_cells_are_numbered_in_order(self, source: str) -> None: + """Each cell opens with `# Cell N — ...`, and N counts 1..10 without a gap.""" + lines = source.splitlines() + numbers: list[int] = [] + for index, line in enumerate(lines): + if line.strip() != CELL_MARKER: + continue + following = lines[index + 1] if index + 1 < len(lines) else '' + match = re.match(r'#\s*Cell\s+(\d+)\b', following.strip()) + assert match, f'the line after the marker on line {index + 1} does not name its cell' + numbers.append(int(match.group(1))) + assert numbers == list(range(1, EXPECTED_CELLS + 1)) + + def test_the_marker_precedes_the_first_statement(self, source: str) -> None: + """The module docstring comes first, then cell 1 — nothing executable before a marker.""" + body = source.split('"""', 2)[-1] + assert body.lstrip().startswith(CELL_MARKER) + + +class TestBoundary: + """Design E15: the script stands in for the frontend, so it borrows no consumer code.""" + + def test_no_import_from_activities_or_workflows(self, tree: ast.Module) -> None: + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + assert not alias.name.startswith(FORBIDDEN_IMPORT_ROOTS), ( + f'the script imports {alias.name}' + ) + elif isinstance(node, ast.ImportFrom) and node.module: + assert not node.module.startswith(FORBIDDEN_IMPORT_ROOTS), ( + f'the script imports from {node.module}' + ) + + def test_the_only_laborious_import_is_the_format_helper(self, tree: ast.Module) -> None: + modules: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + modules.update(a.name for a in node.names if a.name.startswith('laborious')) + elif ( + isinstance(node, ast.ImportFrom) + and node.module + and node.module.startswith('laborious') + ): + modules.add(node.module) + assert modules <= {ALLOWED_LABORIOUS_IMPORT}, ( + f'unexpected laborious imports: {sorted(modules - {ALLOWED_LABORIOUS_IMPORT})}' + ) + + +class TestConfiguration: + """No credential and no target literal in the file; every variable documented.""" + + @pytest.mark.parametrize('forbidden', FORBIDDEN_LITERAL_SUBSTRINGS) + def test_no_forbidden_substring(self, tree: ast.Module, forbidden: str) -> None: + offenders = [text for text in _string_literals(tree) if forbidden in text] + assert not offenders, f'{forbidden!r} appears in a string literal: {offenders}' + + def test_no_hostlike_literal(self, tree: ast.Module) -> None: + for text in _string_literals(tree): + for pattern in HOSTLIKE_PATTERNS: + assert not pattern.search(text), ( + f'{text!r} looks like a host, an address or inline credentials' + ) + + def test_every_variable_it_reads_is_documented(self, tree: ast.Module) -> None: + documented = { + line.split('=', 1)[0].strip() + for line in ENV_EXAMPLE_PATH.read_text(encoding='utf-8').splitlines() + if '=' in line and not line.lstrip().startswith('#') + } + undocumented = sorted( + _env_names(tree) - documented - {'MLFLOW_TRACKING_USERNAME', 'MLFLOW_TRACKING_PASSWORD'} + ) + assert not undocumented, f'.env.example does not document: {undocumented}' + + def test_it_reads_the_variables_the_contract_names(self, tree: ast.Module) -> None: + names = _env_names(tree) + required = { + 'IMPORT_PASSWORD_KEY', + 'IMPORT_BUNDLE_BUCKET', + 'IMPORT_BUNDLE_PREFIX', + 'IMPORT_STATUS_DB_HOST', + 'IMPORT_STATUS_DB_PORT', + 'IMPORT_STATUS_DB_NAME', + 'IMPORT_STATUS_DB_USER', + 'IMPORT_STATUS_DB_PASSWORD', + 'IMPORT_SCRIPT_BUNDLE_PATH', + 'IMPORT_SCRIPT_BUNDLE_PASSWORD', + 'IMPORT_SCRIPT_USERNAME', + 'RUNTIME', + } + assert required <= names, f'the script no longer reads: {sorted(required - names)}' From e95e65b72b5e6e5c1755b109de85d0babb70c01e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 26 Aug 2026 15:45:33 -0300 Subject: [PATCH 27/64] QTZPOC-20 QTZPOC-20: Implement explicit joblib model flavor and enhance model import workflow with extensive E2E tests. This commit introduces a first-class joblib model flavor for direct loading and saving of Python pickle artifacts, including custom code within the artifact's code/ directory, bypassing MLflow's flavor loaders. The model import workflow's status database configuration is now streamlined to default to the POSTGRES_* server, with explicit IMPORT_STATUS_DB_* overrides. Comprehensive end-to-end tests have been added for the model import workflow, covering success paths, gate rejections, digest enforcement, provisioning failures, status record integrity, cleanup, and retry/recovery scenarios using a real-clock environment. New E2 --- .env.example | 14 +- README.md | 74 +- docs/model-flavors.md | 112 +++ docs/model-import.md | 59 +- e2e/conftest.py | 182 +++-- e2e/import_harness.py | 401 ++++++++++- e2e/scenarios.md | 642 ++++++++++++++++- e2e/test_import_model_absence.py | 184 +++++ e2e/test_import_model_cleanup.py | 247 +++++++ e2e/test_import_model_gates.py | 4 + ...test_import_model_provisioning_failures.py | 108 +-- e2e/test_import_model_retry_recovery.py | 514 ++++++++++++++ e2e/test_import_model_second_import.py | 270 ++++++++ e2e/test_import_model_status_record.py | 596 ++++++++++++++++ ...> test_predictions_batch_joblib_flavor.py} | 112 ++- e2e/test_retrain_joblib_flavor.py | 126 ++++ laborious/activities/activities.py | 4 +- laborious/activities/model_import.py | 6 +- laborious/utils/connectors_config.py | 33 +- .../utils/repository/model_repository.py | 379 +++++----- laborious/worker/worker.py | 4 +- pyproject.toml | 11 + requirements-dev.txt | 5 +- scripts/encode.sh | 20 - scripts/encrypt.py | 113 --- scripts/import_model_manual_run.py | 19 +- scripts/inter_arrival.py | 27 - scripts/model_convert.ipynb | 107 --- .../utils/repository/test_model_repository.py | 647 ++++++++++++------ .../laborious/utils/test_connectors_config.py | 50 +- tests/scripts/__init__.py | 0 tests/scripts/test_import_model_manual_run.py | 217 ------ 32 files changed, 4247 insertions(+), 1040 deletions(-) create mode 100644 docs/model-flavors.md create mode 100644 e2e/test_import_model_absence.py create mode 100644 e2e/test_import_model_cleanup.py create mode 100644 e2e/test_import_model_retry_recovery.py create mode 100644 e2e/test_import_model_second_import.py create mode 100644 e2e/test_import_model_status_record.py rename e2e/{test_predictions_batch_joblib_fallback.py => test_predictions_batch_joblib_flavor.py} (56%) create mode 100644 e2e/test_retrain_joblib_flavor.py delete mode 100755 scripts/encode.sh delete mode 100644 scripts/encrypt.py delete mode 100644 scripts/inter_arrival.py delete mode 100644 scripts/model_convert.ipynb delete mode 100644 tests/scripts/__init__.py delete mode 100644 tests/scripts/test_import_model_manual_run.py diff --git a/.env.example b/.env.example index d13ef47..0f631da 100644 --- a/.env.example +++ b/.env.example @@ -55,14 +55,12 @@ IMPORT_BUNDLE_PREFIX="imported_models/" # MongoDB collection holding the model listing the import writes its document into. IMPORT_MODELS_COLLECTION="models" -# The import log lives in the Spring Boot BFF database, a second Postgres -# connection independent of POSTGRES_* above. The schema (public) and the table -# (experiment_run) are constants of the code, not configuration. -IMPORT_STATUS_DB_HOST="localhost" -IMPORT_STATUS_DB_PORT="5432" +# The import log lives in the Spring Boot BFF database on the POSTGRES_* server +# above: same host, port, user and password, another database name. Only that +# name is configuration here; the schema (public) and the table (experiment_run) +# are constants of the code. Set IMPORT_STATUS_DB_{HOST,PORT,USER,PASSWORD} only +# if the log really is on a different server than POSTGRES_* — they override it. IMPORT_STATUS_DB_NAME="sientia-core-mlops-bff" -IMPORT_STATUS_DB_USER="mlops_bff_user" -IMPORT_STATUS_DB_PASSWORD="bff_password" # --- scripts/import_model_manual_run.py only --------------------------------- # Inputs for the manual run that stands in for the frontend. The script writes @@ -70,6 +68,6 @@ IMPORT_STATUS_DB_PASSWORD="bff_password" IMPORT_SCRIPT_BUNDLE_PATH="tests/fixtures/bundle/golden_model_v3.sientia" IMPORT_SCRIPT_BUNDLE_PASSWORD="the-password-the-bundle-was-written-with" IMPORT_SCRIPT_USERNAME="manual.run@example.com" -# Cell 2's guard: restate IMPORT_STATUS_DB_HOST here (or in the cell) after reading what cell 1 +# Cell 2's guard: restate POSTGRES_HOST here (or in the cell) after reading what cell 1 # printed. Left empty on purpose — the script refuses to write until it matches. IMPORT_SCRIPT_CONFIRM_TARGET="" \ No newline at end of file diff --git a/README.md b/README.md index c818329..48c75d6 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ A comprehensive, Temporal-based ML orchestration system for industrial data proc - [Testing](#testing) - [Test Structure](#test-structure) - [Test Execution](#test-execution) + - [End-to-End Tests](#end-to-end-tests-e2e) - [Monitoring and Metrics](#monitoring-and-metrics) - [Application Health Metrics](#application-health-metrics) - [Prediction Operation Metrics](#prediction-operation-metrics) @@ -881,6 +882,57 @@ pytest tests/laborious/activities/test_gates.py pytest tests/laborious/workflows/test_predictions_batch.py ``` +### End-to-End Tests (`e2e/`) + +The `e2e/` suite runs against real dependencies in Docker containers via +[testcontainers](https://testcontainers.com/): PostgreSQL, MinIO, MongoDB, and — for the model-import +suite — an `mlflow server` subprocess started from this repository's venv. It lives outside +`testpaths`, so it is never collected by the unit gate. + +**One-time setup, required before running anything under `e2e/`:** + +```bash +echo "ryuk.disabled=true" >> ~/.testcontainers.properties +``` + +Without it, **every** e2e test fails at setup with a Docker mount error and never reaches an +assertion: + +``` +docker.errors.APIError: 500 Server Error ... /containers//start: +Mounts denied: The path /socket_mnt/home//.docker/desktop/docker.sock +is not shared from the host and is not known to Docker. +``` + +The cause is not the tests. Before starting any test container, testcontainers starts a reaper +container ("Ryuk") that bind-mounts the Docker socket so it can clean up if the test process dies +without teardown. Under Docker Desktop on Linux that socket is `~/.docker/desktop/docker.sock`, a path +Docker Desktop's File Sharing does not expose, so the reaper's own container cannot start. Note the +symptom is `ERROR`, not `FAILED` — the failure is in fixture setup. + +Disabling it costs nothing in a normal run: every fixture stops its own containers in teardown. Ryuk +only matters when the process is killed outright (`kill -9`, a crash, a hard stop from the IDE). After +one of those, clean up with: + +```bash +docker ps -a --filter "label=org.testcontainers=true" -q | xargs -r docker rm -f +``` + +**Running it:** + +```bash +source ./venv/bin/activate + +# the predictions and OPC suites +pytest e2e -m "not import_e2e" + +# the .sientia model-import suite (opt-in; see e2e/scenarios.md § 4) +pytest e2e -m import_e2e +``` + +Full scenario catalogue and the import suite's fidelity contract: +[`e2e/scenarios.md`](e2e/scenarios.md). + ## 📊 Monitoring and Metrics The Laborious system exposes comprehensive Prometheus metrics for operational visibility and performance monitoring: @@ -950,11 +1002,11 @@ See [OPC UA Communication](#opc-ua-communication) for semantics, concurrency, an | `IMPORT_MAX_UNCOMPRESSED_BYTES` | Gate 4 uncompressed size ceiling | `4294967296` | No | | `IMPORT_MAX_COMPRESSION_RATIO` | Gate 4 compression ratio ceiling | `200` | No | | `IMPORT_MODELS_COLLECTION` | MongoDB collection holding the model listing | `models` | No | -| `IMPORT_STATUS_DB_HOST` | Import log database host (the BFF's) | `localhost` | If importing | -| `IMPORT_STATUS_DB_PORT` | Import log database port | `5432` | No | -| `IMPORT_STATUS_DB_NAME` | Import log database name | `sientia-core-mlops-bff` | No | -| `IMPORT_STATUS_DB_USER` | Import log database user | `mlops_bff_user` | If importing | -| `IMPORT_STATUS_DB_PASSWORD` | Import log database password | _(empty)_ | If importing | +| `IMPORT_STATUS_DB_NAME` | Import log database name (the BFF's, on the `POSTGRES_*` server) | `sientia-core-mlops-bff` | If importing | +| `IMPORT_STATUS_DB_HOST` | Override: import log host, when it is not the `POSTGRES_*` server | `POSTGRES_HOST` | No | +| `IMPORT_STATUS_DB_PORT` | Override: import log port | `POSTGRES_PORT` | No | +| `IMPORT_STATUS_DB_USER` | Override: import log user | `POSTGRES_USER` | No | +| `IMPORT_STATUS_DB_PASSWORD` | Override: import log password | `POSTGRES_PASSWORD` | No | | `OPC_CONFIG` | OPC server configuration (JSON) | `{}` | No | | `OPC_ID` | OPC server identifier | `1` | No | | `OPC_URL` | OPC server URL | `opc.tcp://localhost:4840` | No | @@ -1055,6 +1107,18 @@ Legacy MinIO object layout (relative key): `training_datasets/{model_name}/{object_prefix}_{timestamp}.parquet` where `object_prefix` is sanitized (slashes replaced by underscores) to keep a stable model-level directory. +## Model Flavors + +`model_config.predict_flavor` / `model_config.transform_flavor` accept `sklearn`, `pyfunc`, +`pytorch` and `joblib`. The flavor is configuration, never discovered from the artifact, and there is +no fallback between flavors: a failing loader propagates. + +`joblib` reads the artifact directly — download, first top-level `.pkl`, `joblib.load` — with the +artifact's `code/` put on `sys.path` first, and it writes on retrain (`model.pkl` + `code/`, no +`MLmodel`, so only this runtime reads it back). + +Full reference, including how to switch an existing model over: **[docs/model-flavors.md](docs/model-flavors.md)**. + ## OPC UA Communication Full reference: **[docs/opc-communication.md](docs/opc-communication.md)** (connection lifecycle, Tier-1 `Bad*` reconnect, connection lock / session readiness, metrics, PostgreSQL confidence **12** vs **14**, tests). diff --git a/docs/model-flavors.md b/docs/model-flavors.md new file mode 100644 index 0000000..a8db934 --- /dev/null +++ b/docs/model-flavors.md @@ -0,0 +1,112 @@ +# Model flavors + +The *flavor* is how this runtime is told to deserialize a model. It is **configuration, never +discovery**: the value comes from the model document in the `models` collection and the code never +reads the artifact's `MLmodel` manifest to guess it. + +```json +{ + "model_config": { + "transform_flavor": "sklearn", + "predict_flavor": "joblib", + "retention_minutes": 60, + "target": "SE" + } +} +``` + +`predict_flavor` drives `load_predict_model` (artifact `prediction_model`); `transform_flavor` drives +`load_transform_model` (artifact `data_model`). Both default to `sklearn` when the key is absent. + +## Accepted values + +| Flavor | Read | Write (retrain) | How it loads | +|---|---|---|---| +| `sklearn` | yes | yes | `mlflow.sklearn.load_model()` | +| `pyfunc` | yes | yes | `mlflow.pyfunc.load_model()`; the wrapper path also unwraps `._model_impl.python_model` | +| `pytorch` | yes | yes | `mlflow.pytorch.load_model()` | +| `joblib` | yes | yes | downloads the artifact and `joblib.load`s its pickle — **no MLflow flavor loader involved** | + +Anything else raises `ValueError` with `INVALID_FLAVOR_MESSAGE`, **before** any artifact is resolved +or downloaded. The same message and the same four values govern reading and writing. + +**There is no fallback between flavors.** A failing loader propagates: one attempt, one cause. A +model configured with the wrong flavor fails visibly instead of being rescued silently. + +## The `joblib` flavor + +Some models the pipeline produces do not load through an MLflow flavor loader even though the +artifact is intact: the `.pkl` is in the compressed form `joblib.dump` writes, which MLflow's +`pickle.load` does not read. Declaring `joblib` is how such a model is served. + +**Reading.** Resolve the Production run, `client.download_artifacts(run_id, )`, take the +**first top-level `.pkl` in name order**, and `joblib.load` it. The listing does not recurse, so a +nested wrapper artifact (`artifacts/*.pkl`) is never deserialized in place of the model. No +`MLmodel` is parsed and no `models://production` URI is built. + +**The artifact's `code/` goes on `sys.path` first.** A model whose class lives inside the artifact +would otherwise raise `ModuleNotFoundError` at unpickle time. Before `joblib.load`, the flavor +prepends `/code` to `sys.path` when that directory exists — the directory itself, matching +MLflow's own convention where `code/utils/…` imports as the package `utils`. The entry is not +duplicated and is **not** removed after the load, because the deserialized object may import +lazily. A missing `code/` is not an error, and no extra download is performed: `download_artifacts` +already brings `code/` along. + +> `sys.path` is process-global. Two different models that embed a package with the same name +> (`utils/`, typically) and are served by the same worker resolve to whichever entered first — the +> second silently runs the first one's code. This is pre-existing behaviour of MLflow's own pyfunc +> path with the same artifact layout; the mitigation, if it ever bites, is one worker per model. + +**Writing / retraining.** `joblib` is a write flavor too, so a joblib model retrains. `log_model` +writes `joblib.dump(model, 'model.pkl', compress=3)` under the artifact path and re-logs the source +artifact's `code/` to `/code`, so the retrained model stays deserializable. Registration +and promotion to `Production` are unchanged. + +**The written artifact has no `MLmodel` manifest.** Only this runtime's `joblib` flavor reads it: +`mlflow.sklearn.load_model` and `models://production` both fail on it. A model retrained in +joblib **stays** joblib — `predict_flavor` needs no edit after a retrain, and no consumer outside +this runtime (the MLflow UI, MLflow serving, another service) recognizes that artifact as a model. + +**What `joblib` is for.** A pickled object exposing `predict(data)` — and `fit(data)` if it is +retrained. It is **not** for a bare pyfunc `PythonModel` wrapper: `get_prediction_data` and +`get_cached_operation` call `predict(data)` for every non-pyfunc flavor, so a wrapper expecting +`predict(context, model_input)` raises `TypeError` at predict and retrain time. + +**Observability.** A joblib load is an ordinary load: `MODEL_READ_LAG` / `MODEL_READ_COUNT` on +success and `MODEL_READ_ERROR_COUNT` on failure, under the base `operation_type` +(`load_predict_model` / `load_transform_model`). A joblib write emits the usual `MODEL_WRITE_*` +under `log_model`. No suffixed operation type, no new metric, no new label. + +## Switching a model to `joblib` + +The runtime does not migrate documents. Editing the `models` collection is an operator step, and the +order matters: + +1. **Inventory first, before the deploy.** Collect the models that fail in their configured flavor + loader — run predict/transform and note which ones raise. There is no automatic discovery + afterwards: once deployed, such a model simply fails, and the failure names the flavor loader, + not the fix. +2. **Edit the document.** Set `model_config.predict_flavor: 'joblib'` and/or + `model_config.transform_flavor: 'joblib'` for each model in the inventory. Nothing else changes; + models already on a working flavor need no edit and behave identically. +3. **Imported models arrive ready.** The `import_model` workflow writes + `predict_flavor: 'joblib'` itself — see [`model-import.md`](model-import.md) § 12. +4. **Check the first retrain.** In MLflow, confirm the new run carries + `prediction_model/model.pkl` **and** `prediction_model/code/` when the source model had a + `code/` directory, and that the promoted version loads on the next predict. +5. **Rolling back is not just a revert.** Reverting the code leaves any artifact a joblib retrain + already wrote in MLflow: that Production version stops being loadable, so the rollback also means + promoting the previous version back in the registry, and setting the documents' flavors back — + manual, in reverse order. + +## Known gap: how the run id is resolved + +`get_model_run_id` resolves the Production run with `source.split('/')[2]` on the registered +version's `source`. **This has not been confirmed against the real tracking server.** No artifact +scheme reproduced locally puts the run id in that position (a file store yields `''`, S3 yields the +bucket, `mlflow-artifacts:/` yields the experiment id), yet predict works in production — so the +real `source` has a shape none of the local probes produced. + +This is not specific to `joblib`: every flavor that resolves a run goes through the same line, and +the joblib read path depends on it. Confirm it with a **read-only** query against the tracking +server before relying on it, ideally before the next deploy. diff --git a/docs/model-import.md b/docs/model-import.md index 125f36a..51ca951 100644 --- a/docs/model-import.md +++ b/docs/model-import.md @@ -88,14 +88,18 @@ is at fault. | `IMPORT_MAX_UNCOMPRESSED_BYTES` | 4 GiB | gate 4 | | `IMPORT_MAX_COMPRESSION_RATIO` | `200` | gate 4 | | `IMPORT_MODELS_COLLECTION` | `models` | the MongoDB collection that *is* the model listing | -| `IMPORT_STATUS_DB_HOST` | `localhost` | the BFF database holding the import log | -| `IMPORT_STATUS_DB_PORT` | `5432` | | -| `IMPORT_STATUS_DB_NAME` | `sientia-core-mlops-bff` | | -| `IMPORT_STATUS_DB_USER` | `mlops_bff_user` | | -| `IMPORT_STATUS_DB_PASSWORD` | empty | | +| `IMPORT_STATUS_DB_NAME` | `sientia-core-mlops-bff` | the BFF database holding the import log | +| `IMPORT_STATUS_DB_HOST` | `POSTGRES_HOST` | override: the log is on another server | +| `IMPORT_STATUS_DB_PORT` | `POSTGRES_PORT` | override | +| `IMPORT_STATUS_DB_USER` | `POSTGRES_USER` | override | +| `IMPORT_STATUS_DB_PASSWORD` | `POSTGRES_PASSWORD` | override | -`IMPORT_STATUS_DB_*` is a **second** Postgres connection, independent of the one Laborious already -holds for predictions. The import writes to that database and no other, and to one table in it. +The import log is a **second connection but the same server**: host, port, user and password come +from `POSTGRES_*`, and only the database name is import configuration, because the log lives in the +BFF's database rather than in the one Laborious holds for predictions. The import writes to that +database and no other, and to one table in it. The four `IMPORT_STATUS_DB_*` connection variables +are overrides for a deployment whose log genuinely sits elsewhere — the e2e suite, which runs the +two databases as two containers, is exactly that case. A missing or malformed `IMPORT_PASSWORD_KEY` is an import failure at the decryption step, not a boot failure: the worker starts either way. @@ -324,10 +328,11 @@ A `COMPLETED` import is **not** a servable model. Stated plainly: So an operator must promote the version to `Production` and set `active: true`. This workflow never promotes and never activates — that runbook is QTZPOC-21's. -One more thing to know before serving an imported model: `predict_flavor: 'joblib'` is a **read-only** -flavor. It is what the artifact actually is, and it is enough to load and predict, but nothing in this -platform writes that flavor — so an imported model cannot be retrained until a write flavor is -configured for it. +One more thing to know before serving an imported model: `predict_flavor: 'joblib'` is what the +artifact actually is, and it loads, predicts **and** retrains — `log_model` writes that flavor too, +as `model.pkl` + `code/`. The artifact it writes carries no `MLmodel` manifest, so an imported model +stays readable only by this runtime, before and after a retrain. Full contract: +[`model-flavors.md`](model-flavors.md). ## 14. Retry, and what does not retry @@ -347,3 +352,35 @@ change. There is no rollback, no compensation, no resume and no reconciliation. A failed import stays failed; trying again is a new upload and a new row. + +## 15. How this contract is verified + +Two suites, and they prove different things. + +**Unit** — `pytest tests/`, part of the standard gate (`validate --project-name=laborious`). Covers +the bundle reader against committed golden bytes, every gate, the failure catalog, the status SQL and +the workflow's ordering, all against fakes. + +**End to end** — `pytest e2e -m import_e2e`, opt-in and outside `testpaths`. Runs the real workflow on +a real worker against real MinIO, a real Postgres holding this document's `public.experiment_run` +(with and without `V12`), a real MongoDB with the unique index on `models.id`, and a real +`mlflow server`. Nothing on the import path is mocked. This is what proves the claims a fake cannot: +that the `CHECK` constraints of § 6 accept every status written, that the registry source of § 11 +resolves through the existing loader, that the `V12` failure shape of § 7 is what it says, and that no +plaintext password reaches the Temporal event history. + +Requires Docker plus one one-time setup per machine: + +```bash +echo "ryuk.disabled=true" >> ~/.testcontainers.properties +``` + +Without it every e2e test fails in fixture setup with a Docker mount error — see +[`README.md` § End-to-End Tests](../README.md#end-to-end-tests-e2e) for why and what it costs. + +Scenario catalogue: [`e2e/scenarios.md`](../e2e/scenarios.md) § 4. +Requirement-to-test mapping: `openspec/changes/import-model-e2e-suite/traceability.md`. + +Doing one import by hand, standing in for the frontend that does not exist yet: +[`scripts/import_model_manual_run.py`](../scripts/import_model_manual_run.py) — see +[`README.md` § Running One Model Import by Hand](../README.md#running-one-model-import-by-hand). diff --git a/e2e/conftest.py b/e2e/conftest.py index 878a114..51af260 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -2,9 +2,19 @@ Pytest configuration and fixtures for E2E tests. """ +import importlib import sys from unittest.mock import AsyncMock, MagicMock, patch +# Load every BSON codec now, for its side effect only. `pymongo` imports `bson.objectid`, +# `bson.timestamp`, `bson.decimal128` and a dozen more lazily, and the first time is while an +# activity is running — after Temporal's workflow sandbox has snapshotted `sys.modules`. The sandbox +# then warns once per submodule ("Module bson.timestamp was imported after initial workflow load"), +# which was ~20 lines of noise per test that touches MongoDB, enough to bury a warning worth reading. +# This has to happen **before** the workflow modules are imported below, which is why it is here and +# not in `import_harness.py` — pytest loads `pytest_plugins` only after this module's body has run. +import bson.json_util # noqa: F401 + # E2E workflows under test do not run ModelAnalysis; stub before Activities import. _model_analysis_module = MagicMock() _model_analysis_module.ModelAnalysis = MagicMock @@ -18,14 +28,11 @@ import pytest import pytest_asyncio from sientia_do.notifications.handlers import CoreNotificationHandler from sientia_do.observability.metrics_controller import MetricsController -from sklearn.compose import ColumnTransformer -from sklearn.linear_model import LinearRegression -from sklearn.pipeline import Pipeline from sqlalchemy import create_engine, text from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker -from testcontainers.minio import MinioContainer -from testcontainers.postgres import PostgresContainer +from testcontainers.community.minio import MinioContainer +from testcontainers.community.postgres import PostgresContainer from e2e.opc_test_server import OpcE2ETestServer from laborious.activities.activities import Activities @@ -42,9 +49,56 @@ pytest_plugins = ['e2e.import_harness'] TEST_MONGODB_CONNECTION_STRING = 'mongodb://localhost:27017' TEST_DATABASE_NAME = 'test_db' -# Dedicated model whose MLflow flavor loaders fail, so the joblib fallback is exercised. -JOBLIB_FALLBACK_MODEL_NAME = 'joblib_fallback_model' -JOBLIB_FALLBACK_FLAVOR_ERROR = 'flavor loader cannot deserialize this artifact' +# Dedicated model served through the explicit `joblib` flavor, so no MLflow flavor loader runs. +JOBLIB_FLAVOR_MODEL_NAME = 'joblib_flavor_model' + +# Module carrying the classes of the joblib artifacts' models. It ships inside the artifact's +# `code/` directory and is importable only while that directory is on `sys.path` — which is what +# the joblib flavor arranges before deserializing. +JOBLIB_FLAVOR_CODE_MODULE = 'joblib_flavor_probe' + +JOBLIB_FLAVOR_WEIGHTS = {'feature_1': 2.0, 'feature_2': 3.0} + +JOBLIB_FLAVOR_CODE_SOURCE = '''"""Model classes that exist only inside the artifact's `code/` directory.""" + +import pandas as pd + +FEATURES = ('feature_1', 'feature_2') + + +class JoblibFlavorPredictor: + """Prediction model: a weighted sum of the transform output columns.""" + + def __init__(self, weights): + self.weights = weights + + def fit(self, data): + return self + + def predict(self, data): + frame = pd.DataFrame(data) + return [ + sum(row[name] * self.weights[name] for name in FEATURES) + for _, row in frame.iterrows() + ] + + +class JoblibFlavorTransformer: + """Transform model: emits the fixed feature frame the prediction model expects.""" + + target_variable = 'target' + + def fit(self, data): + return self.predict(data) + + def predict(self, data): + frame = pd.DataFrame(data) + result = pd.DataFrame( + {'feature_1': [0.234] * len(frame), 'feature_2': [0.783] * len(frame)} + ) + result.index = frame.index + return result +''' @pytest_asyncio.fixture(scope='session') @@ -389,61 +443,78 @@ def mock_mlflow_models(): } -def _build_joblib_fallback_estimator(): - """ - Build a real sklearn estimator for the joblib fallback artifact. +def _write_artifact_code_dir(artifact_dir): + """Write the artifact's `code/` directory, carrying the module with the model classes.""" + code_dir = artifact_dir / 'code' + code_dir.mkdir() + (code_dir / f'{JOBLIB_FLAVOR_CODE_MODULE}.py').write_text(JOBLIB_FLAVOR_CODE_SOURCE) + return code_dir - `request_predict` appends a string `timestamp` column to the transformed - data, so the estimator selects the transform output columns explicitly and - drops the rest. + +def _build_joblib_flavor_artifact(artifact_dir, class_name, *args): """ - features = ['feature_1', 'feature_2'] - estimator = Pipeline( - steps=[ - ( - 'select', - ColumnTransformer([('features', 'passthrough', features)], remainder='drop'), - ), - ('regressor', LinearRegression()), - ] - ) - train = pd.DataFrame( - { - 'feature_1': [0.0, 1.0, 2.0, 3.0], - 'feature_2': [0.0, 1.0, 0.0, 1.0], - 'timestamp': ['2024-01-01 12:00:00'] * 4, - } - ) - estimator.fit(train, train['feature_1'] * 2 + train['feature_2'] * 3) - return estimator + Build one joblib artifact: `code/` with the class module, plus the compressed `model.pkl`. + + Instantiating **and** dumping happen while `code/` is on `sys.path` — `pickle` refuses to + serialize a class it cannot import. Both are undone on the way out, so from here the pickled + class is unreachable until something puts the artifact's `code/` back on the path. That + something is the flavor under test. + """ + code_dir = _write_artifact_code_dir(artifact_dir) + + sys.path.insert(0, str(code_dir)) + try: + module = importlib.import_module(JOBLIB_FLAVOR_CODE_MODULE) + model = getattr(module, class_name)(*args) + joblib.dump(model, artifact_dir / 'model.pkl', compress=3) + finally: + sys.path.remove(str(code_dir)) + sys.modules.pop(JOBLIB_FLAVOR_CODE_MODULE, None) + importlib.invalidate_caches() + + return {'path': str(artifact_dir), 'code_dir': str(code_dir), 'model': model} @pytest.fixture -def joblib_fallback_artifact(tmp_path): +def joblib_flavor_artifacts(tmp_path): """ - Real artifact directory holding a joblib-compressed sklearn estimator. + Real artifact directories for the `joblib` flavor, one per model type. - Mirrors what the joblib fallback finds after downloading `prediction_model`: - a top-level `model.pkl` written by `joblib.dump(..., compress=3)` plus an - `MLmodel` placeholder. + Each holds a top-level `model.pkl` written by `joblib.dump(..., compress=3)` and a real + `code/` directory with the module where the model's class lives. No `MLmodel` is written: + the artifact this flavor reads — and the one a joblib retrain writes — has none. Return: - dict with 'path' (the artifact directory) and 'estimator' (the object - that was serialized, so tests can compute the expected prediction). + dict keyed by artifact name ('prediction_model', 'data_model'), each value carrying + 'path', 'code_dir' and 'model' (the object serialized, so tests can compute the + expected prediction). """ - artifact_dir = tmp_path / 'prediction_model' - artifact_dir.mkdir() + original_sys_path = list(sys.path) - estimator = _build_joblib_fallback_estimator() - joblib.dump(estimator, artifact_dir / 'model.pkl', compress=3) - (artifact_dir / 'MLmodel').write_text('flavors:\n sklearn: {}\n') + builds = { + 'prediction_model': ('JoblibFlavorPredictor', (JOBLIB_FLAVOR_WEIGHTS,)), + 'data_model': ('JoblibFlavorTransformer', ()), + } - return {'path': str(artifact_dir), 'estimator': estimator} + artifacts = {} + + for artifact_name, (class_name, args) in builds.items(): + artifact_dir = tmp_path / artifact_name + artifact_dir.mkdir() + + artifacts[artifact_name] = _build_joblib_flavor_artifact(artifact_dir, class_name, *args) + + yield artifacts + + # The flavor prepends `code/` to `sys.path` and deliberately never removes it. + sys.path[:] = original_sys_path + sys.modules.pop(JOBLIB_FLAVOR_CODE_MODULE, None) + importlib.invalidate_caches() @pytest.fixture -def joblib_fallback_artifact_without_pickle(tmp_path): - """Artifact directory with no top-level `.pkl`, so the joblib fallback also fails.""" +def joblib_flavor_artifact_without_pickle(tmp_path): + """Artifact directory with no top-level `.pkl`, so the joblib flavor fails on it.""" artifact_dir = tmp_path / 'prediction_model_without_pickle' artifact_dir.mkdir() (artifact_dir / 'MLmodel').write_text('flavors:\n sklearn: {}\n') @@ -451,14 +522,12 @@ def joblib_fallback_artifact_without_pickle(tmp_path): @pytest_asyncio.fixture -def patch_mlflow(mock_mlflow_models, joblib_fallback_artifact): +def patch_mlflow(mock_mlflow_models, joblib_flavor_artifacts): """Patch mlflow module in repository with load_model mocks.""" mock_mlflow = MagicMock() # Mock sklearn.load_model def mock_sklearn_load_model(model_uri): - if JOBLIB_FALLBACK_MODEL_NAME in model_uri: - raise RuntimeError(JOBLIB_FALLBACK_FLAVOR_ERROR) if 'data_model' in model_uri or 'transform' in model_uri.lower(): return mock_mlflow_models['transform_model'] return mock_mlflow_models['predict_model'] @@ -468,8 +537,6 @@ def patch_mlflow(mock_mlflow_models, joblib_fallback_artifact): # Mock pyfunc.load_model def mock_pyfunc_load_model(model_uri): - if JOBLIB_FALLBACK_MODEL_NAME in model_uri: - raise RuntimeError(JOBLIB_FALLBACK_FLAVOR_ERROR) if 'artifacts' in model_uri or 'tmp' in model_uri: return mock_mlflow_models['pyfunc_model'] if 'data_model' in model_uri or 'transform' in model_uri.lower(): @@ -499,8 +566,13 @@ def patch_mlflow(mock_mlflow_models, joblib_fallback_artifact): ] ), ) - # Only the joblib fallback calls download_artifacts directly on the client. - mock_mlflow_client.download_artifacts = MagicMock(return_value=joblib_fallback_artifact['path']) + + # Only the joblib flavor calls download_artifacts directly on the client, one artifact + # per model type — a single return value would serve `data_model` from `prediction_model`. + def mock_download_artifacts(_run_id, artifact_name): + return joblib_flavor_artifacts[artifact_name]['path'] + + mock_mlflow_client.download_artifacts = MagicMock(side_effect=mock_download_artifacts) mock_mlflow.tracking.MlflowClient = MagicMock(return_value=mock_mlflow_client) with patch('laborious.utils.repository.model_repository.mlflow', new=mock_mlflow): diff --git a/e2e/import_harness.py b/e2e/import_harness.py index 4589317..517f802 100644 --- a/e2e/import_harness.py +++ b/e2e/import_harness.py @@ -41,10 +41,12 @@ model" unreadable. from __future__ import annotations +import asyncio import contextlib import hashlib import json import os +import re import signal import socket import subprocess @@ -52,7 +54,7 @@ import sys import time import uuid from collections.abc import Callable, Iterator -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any @@ -63,9 +65,11 @@ import requests from pymongo import MongoClient from sqlalchemy import Engine, create_engine, text from temporalio.client import WorkflowHandle +from temporalio.common import RetryPolicy from temporalio.worker import Worker -from testcontainers.mongodb import MongoDbContainer -from testcontainers.postgres import PostgresContainer +from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner, SandboxRestrictions +from testcontainers.community.mongodb import MongoDbContainer +from testcontainers.community.postgres import PostgresContainer from laborious.activities.activities import Activities from laborious.workflows.import_model import ImportModel @@ -106,6 +110,7 @@ SEEDED_MODEL_IDS = ('1', '2', '7') MLFLOW_HEALTH_TIMEOUT_SECONDS = 90.0 MLFLOW_HEALTH_POLL_SECONDS = 0.5 +MONGO_SERVER_SELECTION_MS = 5000 IMPORT_ACTIVITY_NAMES = ( 'claim_import_status', @@ -139,13 +144,22 @@ def import_bff_container() -> Iterator[PostgresContainer]: `log_statement=all` is what makes the absence assertions possible: the server log is the only vantage point that sees every client, including the psycopg pool `Storage` holds. + + The host port is **bound explicitly** rather than left ephemeral. Docker re-allocates an + ephemeral published port when a container is started again, so a stop/start would move the + database to a new address and the retry-recovery tests would be asserting that a *different* + server came up. A fixed binding is what makes "it came back" mean what it says. """ - container = PostgresContainer( - 'postgres:15', - username=BFF_USER, - password=BFF_PASSWORD, - dbname=BFF_DATABASE, - ).with_command('postgres -c log_statement=all') + container = ( + PostgresContainer( + 'postgres:15', + username=BFF_USER, + password=BFF_PASSWORD, + dbname=BFF_DATABASE, + ) + .with_command('postgres -c log_statement=all') + .with_bind_ports(5432, _free_port()) + ) container.start() try: yield container @@ -732,7 +746,13 @@ async def import_activities( logger=mock_logger, notification_handler=notification_handler, mongo_config={ - 'connection_string': import_mongo_container.get_connection_url(), + # A short server-selection timeout so the "MongoDB is down" case fails in seconds + # instead of spending pymongo's 30-second default. Everything else about the client is + # the production one. + 'connection_string': ( + f'{import_mongo_container.get_connection_url()}' + f'?serverSelectionTimeoutMS={MONGO_SERVER_SELECTION_MS}' + ), 'database_name': MONGO_DATABASE, }, import_config=build_import_config(), @@ -753,16 +773,208 @@ async def import_activities( @pytest_asyncio.fixture async def import_worker(temporal_test_env, import_activities: Any) -> Any: - """An `ImportModel` worker with the thirteen import activities, on the import queue.""" + """An `ImportModel` worker with the thirteen import activities, on the import queue. + + `bson` and `pymongo` are declared **passthrough** on the sandbox. Without that, the workflow + module's `imports_passed_through()` block pulls in `Activities` → `MongoDBRepository` → `pymongo` + during the first workflow activation, and `pymongo` loads its BSON codecs lazily + (`bson.objectid`, `bson.timestamp`, `bson.decimal128`, …). Each one is an import the sandbox has + not seen, so it warns "Module bson.X was imported after initial workflow load" — about twenty + lines per test that touches MongoDB. + + Declaring them passthrough is the supported fix and the accurate one: these modules are pure + serialisation code with no workflow-relevant global state, which is exactly what passthrough is + for. Eagerly importing them on the host does **not** help — the sandbox judges imports against + its own module table during an activation, not against the host's `sys.modules`. + """ async with Worker( temporal_test_env.client, task_queue=IMPORT_TASK_QUEUE, workflows=[ImportModel], activities=[getattr(import_activities, name) for name in IMPORT_ACTIVITY_NAMES], + workflow_runner=SandboxedWorkflowRunner( + restrictions=SandboxRestrictions.default.with_passthrough_modules('bson', 'pymongo') + ), ) as worker: yield worker +# --- the real-clock environment, for the retry-envelope tests ------------------------------------ +# +# Design E9: everything else in this suite runs on the shared time-skipping environment. Two claims +# cannot: "the database came back *during* the retry envelope" needs the restart to occupy real +# time, and "the failure landed on the *first* attempt" is an assertion about elapsed attempts. Both +# run here instead, and this is the one place the harness deliberately weakens fidelity — so it says +# so, rather than hiding it in a fixture. + + +@pytest.fixture +def import_status_retry_shortening() -> dict[str, Any]: + """How much to shorten the status retry policies for a real-clock test. + + The **real** values (10 attempts, 1 s doubling to 30 s, a 300 s envelope) stay asserted where + they belong — unit test 12.1 of `import-model-workflow` pins them. What the tests here prove is + the recovery *behaviour*, which is identical at any interval, so waiting 151 seconds to observe + it would be waiting for arithmetic. + + A constant half-second interval rather than a shortened doubling: it makes the envelope a number + a reader can multiply out (`attempts × interval`) instead of a geometric series. + """ + return { + 'initial_interval': 0.5, + 'backoff_coefficient': 1.0, + 'maximum_attempts': 24, # 12 s of retrying + 'hint_maximum_attempts': 2, + 'envelope_seconds': 25, + 'write_timeout_seconds': 5, + } + + +@pytest_asyncio.fixture +async def import_real_clock_env() -> Any: + """A `WorkflowEnvironment` on the real clock.""" + from temporalio.testing import WorkflowEnvironment + + env = await WorkflowEnvironment.start_local() + async with env: + yield env + + +@pytest_asyncio.fixture +async def import_worker_real_clock( + monkeypatch: pytest.MonkeyPatch, + import_real_clock_env: Any, + import_status_retry_shortening: dict[str, Any], + import_activities: Any, +) -> Any: + """An `ImportModel` worker on the real clock, with the status retry policies shortened. + + Two harness choices worth knowing about: + + - The policies are patched **on the workflow module** before the worker starts, because the + workflow reads them as module globals when it schedules an activity. + - The worker runs `UnsandboxedWorkflowRunner`. The sandbox imports the workflow module into a + namespace of its own, so a patch applied to the host module would simply not be seen and the + test would silently run the production 151-second envelope. Unsandboxed is what makes the + patch effective; nothing else about the workflow changes. + """ + from temporalio.worker import UnsandboxedWorkflowRunner + + from laborious.workflows import import_model as workflow_module + + shortening = import_status_retry_shortening + monkeypatch.setattr( + workflow_module, + 'IMPORT_STATUS_RETRY_POLICY', + RetryPolicy( + maximum_attempts=int(shortening['maximum_attempts']), + initial_interval=timedelta(seconds=shortening['initial_interval']), + backoff_coefficient=shortening['backoff_coefficient'], + maximum_interval=timedelta(seconds=shortening['initial_interval']), + non_retryable_error_types=[ + 'ImportLogRowNotClaimableError', + 'ImportStatusRejectedError', + ], + ), + ) + monkeypatch.setattr( + workflow_module, + 'IMPORT_STATUS_HINT_RETRY_POLICY', + RetryPolicy( + maximum_attempts=int(shortening['hint_maximum_attempts']), + initial_interval=timedelta(seconds=shortening['initial_interval']), + backoff_coefficient=1.0, + maximum_interval=timedelta(seconds=shortening['initial_interval']), + non_retryable_error_types=[ + 'ImportLogRowNotClaimableError', + 'ImportStatusRejectedError', + ], + ), + ) + monkeypatch.setattr( + workflow_module, + 'STATUS_WRITE_ENVELOPE', + timedelta(seconds=shortening['envelope_seconds']), + ) + monkeypatch.setattr( + workflow_module, + 'STATUS_WRITE_TIMEOUT', + timedelta(seconds=shortening['write_timeout_seconds']), + ) + + async with Worker( + import_real_clock_env.client, + task_queue=IMPORT_TASK_QUEUE, + workflows=[ImportModel], + activities=[getattr(import_activities, name) for name in IMPORT_ACTIVITY_NAMES], + workflow_runner=UnsandboxedWorkflowRunner(), + ) as worker: + yield worker + + +@pytest.fixture +def import_execute_real_clock(import_real_clock_env: Any, import_worker_real_clock: Any) -> Any: + """Start an import on the real-clock environment and hand back the handle.""" + + async def execute(payload: dict[str, Any], workflow_id: str | None = None) -> WorkflowHandle: + return await import_real_clock_env.client.start_workflow( + ImportModel.run, + payload, + id=workflow_id or f'import-e2e-rc-{uuid.uuid4().hex}', + task_queue=IMPORT_TASK_QUEUE, + ) + + return execute + + +@pytest.fixture +def import_bff_stopped(import_bff_container: PostgresContainer, import_bff_engine: Engine) -> Any: + """An **async** context manager that stops the status database and always brings it back. + + `docker stop` / `docker start` on the wrapped container, not the testcontainers `stop()`, which + removes the container. The published port is bound explicitly by `import_bff_container`, so the + database comes back at the address the worker is already configured with. + + Async, and every blocking call goes through `asyncio.to_thread`, for a reason that cost an + afternoon to find: the Temporal worker runs on this test's event loop. A synchronous + `docker stop` — or a plain `time.sleep` in the test body — starves the worker, so the workflow + makes no progress at all while the database is down and the test ends up asserting that a + *stalled* workflow behaved correctly. Nothing retries on a blocked loop. + + The restart is in a `finally` and the engine is disposed on the way in: pooled connections do not + survive the server going away, and a stale one would fail the next test rather than this one. + """ + + @contextlib.asynccontextmanager + async def stopped() -> Any: + wrapped = import_bff_container.get_wrapped_container() + await asyncio.to_thread(import_bff_engine.dispose) + await asyncio.to_thread(wrapped.stop, timeout=5) + try: + yield + finally: + await asyncio.to_thread(wrapped.start) + await asyncio.to_thread(_wait_for_postgres, import_bff_engine) + + return stopped + + +def _wait_for_postgres(engine: Engine) -> None: + """Block until the restarted server accepts a connection again.""" + deadline = time.monotonic() + 60 + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + with engine.connect() as connection: + connection.execute(text('SELECT 1')) + return + except Exception as error: # noqa: BLE001 - any failure here means "not up yet" + last_error = error + engine.dispose() + time.sleep(0.5) + raise RuntimeError(f'the status database did not come back: {last_error!r}') + + # --- running an import -------------------------------------------------------------------------- @@ -834,6 +1046,173 @@ def import_history_events() -> Callable[[WorkflowHandle], Any]: return fetch +# --- the statement-log window -------------------------------------------------------------------- +# +# "The import writes to one table in one database" cannot be asserted from the application side. +# A SQLAlchemy `before_cursor_execute` listener is blind to the psycopg pool `Storage` holds, which +# is exactly the client that would betray the claim; `pg_stat_statements` is aggregated, so it +# cannot be scoped to one import. The server's own log is the only vantage point that sees every +# client, which is why both containers run with `log_statement=all`. + +_STATEMENT_MARKERS = ('statement:', 'execute :', 'execute ') + +# A new log entry starts with the server's timestamp; anything else continues the previous one. +# Statements are logged across as many lines as they were written on, so without this a multi-line +# `UPDATE ... WHERE id = ...` would be judged on its first line alone. +_LOG_ENTRY_START = re.compile(r'^\d{4}-\d{2}-\d{2} ') + +# Statements that name no *user* relation and are therefore not evidence either way. +_RELATIONLESS_PREFIXES = ( + 'begin', + 'commit', + 'rollback', + 'set ', + 'show ', + 'discard', + 'select 1', + 'select current_database', + 'select current_schema', + 'select pg_backend_pid', + 'select version()', +) + +# System catalogs. Every driver reads these on connection setup — psycopg2 asks `pg_type` for its +# OID map before the application issues anything — so a statement whose only relations are catalogs +# is connection bookkeeping, not a table the feature chose to touch. +_CATALOG_IDENTIFIERS = ( + 'pg_catalog', + 'pg_type', + 'pg_namespace', + 'pg_range', + 'pg_class', + 'pg_attribute', + 'pg_proc', + 'pg_settings', + 'pg_database', + 'information_schema', + 'typarray', +) + + +def _container_log_text(container: Any) -> str: + """Everything the container has written to stdout and stderr so far, as one string.""" + streams = container.get_logs() + parts = streams if isinstance(streams, tuple) else (streams,) + return '\n'.join( + part.decode('utf-8', errors='replace') if isinstance(part, bytes) else str(part) + for part in parts + if part + ) + + +def _statements_between(log_text: str, begin: str, end: str) -> list[str]: + """Every logged statement between the two marker strings, in order. + + Anchored on markers rather than on timestamps: the log is shared with every other client, and a + timestamp window would either include their statements or race with the clock. + """ + try: + start_index = log_text.index(begin) + except ValueError as error: + raise AssertionError(f'the opening marker {begin!r} is not in the log') from error + try: + end_index = log_text.index(end, start_index) + except ValueError as error: + raise AssertionError(f'the closing marker {end!r} is not in the log') from error + + # Expand to whole lines. Slicing on the marker offsets themselves would cut the marker's own log + # line in half at both ends, and the tail half of the closing one reads like a real statement. + region_start = log_text.find('\n', start_index) + region_start = 0 if region_start == -1 else region_start + 1 + region_end = log_text.rfind('\n', start_index, end_index) + region = log_text[region_start:region_end] if region_end > region_start else '' + + statements: list[str] = [] + collecting = False + for line in region.splitlines(): + if _LOG_ENTRY_START.match(line): + collecting = False + for marker in _STATEMENT_MARKERS: + position = line.find(marker) + if position != -1: + statements.append(line[position + len(marker) :].strip()) + collecting = True + break + elif collecting and statements: + # A continuation line of the statement above it. + statements[-1] = f'{statements[-1]} {line.strip()}' + return [statement for statement in statements if statement and begin not in statement] + + +_SINGLE_QUOTED = re.compile(r"'(?:[^']|'')*'") + + +def strip_literals(statement: str) -> str: + """The statement with every single-quoted literal removed. + + Necessary for any "this relation name does not appear" assertion: the import writes the bundle's + own `model_project` into the JSON detail, so the *value* `factory_project` would otherwise read + as a reference to a `project` table. A relation name appears outside a literal or not at all. + """ + return _SINGLE_QUOTED.sub("''", statement) + + +def names_no_relation(statement: str) -> bool: + """True when the statement touches no *user* relation. + + Two kinds: transaction and session bookkeeping (`BEGIN`, `SET`, `SELECT 1`), and the system + catalogs every driver reads when it connects. Neither is a table the feature chose to write. + """ + lowered = statement.strip().lower() + if any(lowered.startswith(prefix) for prefix in _RELATIONLESS_PREFIXES): + return True + return any(identifier in lowered for identifier in _CATALOG_IDENTIFIERS) + + +@pytest.fixture +def import_log_window( + import_bff_engine: Engine, + import_bff_container: PostgresContainer, + postgres_engine: Any, + postgres_container: Any, +) -> Any: + """A context manager bracketing an import with markers on **both** Postgres servers. + + On exit it fills the dict it yielded with `bff` and `predictions` — the statements each server + logged between the markers. A marker is written to both servers so each window is bounded on the + server it is about, rather than one server's window being inferred from the other's clock. + """ + + @contextlib.contextmanager + def window() -> Iterator[dict[str, list[str]]]: + token = uuid.uuid4().hex[:12] + begin = f'e2e_window_begin_{token}' + end = f'e2e_window_end_{token}' + captured: dict[str, list[str]] = {} + + _emit_marker(import_bff_engine, begin) + _emit_marker(postgres_engine, begin) + try: + yield captured + finally: + _emit_marker(import_bff_engine, end) + _emit_marker(postgres_engine, end) + captured['bff'] = _statements_between( + _container_log_text(import_bff_container), begin, end + ) + captured['predictions'] = _statements_between( + _container_log_text(postgres_container), begin, end + ) + + return window + + +def _emit_marker(engine: Engine, marker: str) -> None: + """A statement whose only purpose is to be findable in the server's log.""" + with engine.begin() as connection: + connection.execute(text(f"SELECT '{marker}' AS marker")) # noqa: S608 - a generated uuid + + # --- reading a failure --------------------------------------------------------------------------- # # Temporal wraps what actually went wrong: `WorkflowFailureError` -> `ActivityError` -> diff --git a/e2e/scenarios.md b/e2e/scenarios.md index 8841424..71abd5c 100644 --- a/e2e/scenarios.md +++ b/e2e/scenarios.md @@ -4,12 +4,71 @@ This document describes all possible test scenarios for the `predictions_batch` ## Running automated E2E tests (`e2e/`) +### One-time setup: disable the testcontainers reaper + +Do this once per machine, before running anything under `e2e/`: + +```bash +echo "ryuk.disabled=true" >> ~/.testcontainers.properties +``` + +**Why.** Before starting any test container, testcontainers starts a reaper container ("Ryuk") whose +job is to clean up containers if the test process dies without running its teardown. Ryuk bind-mounts +the Docker socket. Under **Docker Desktop on Linux** the socket lives at +`~/.docker/desktop/docker.sock`, which Docker Desktop's File Sharing does not expose, so the mount is +denied and the reaper cannot start: + +``` +docker.errors.APIError: 500 Server Error ... /containers//start: +Mounts denied: The path /socket_mnt/home//.docker/desktop/docker.sock +is not shared from the host and is not known to Docker. +``` + +Every test then errors **at setup** — `ERROR`, not `FAILED`, and never inside an assertion. If you see +that, this is why. + +**What you give up.** Nothing in a normal run: every fixture in `conftest.py` and +`import_harness.py` stops its own containers in teardown. Ryuk only covers the case where the process +is killed without teardown (`kill -9`, a crash, a hard stop from the IDE). After one of those, clean +up by hand: + +```bash +docker ps -a --filter "label=org.testcontainers=true" -q | xargs -r docker rm -f +``` + +The setting is read by `testcontainers/core/config.py`, which accepts either `ryuk.disabled` in +`~/.testcontainers.properties` or the `TESTCONTAINERS_RYUK_DISABLED` environment variable. The +properties file is preferred here because it is per-machine and applies to every project and every +shell, so nobody has to remember a prefix on the command line. + +*Alternative, if you want to keep the reaper:* point Docker at the classic socket with +`export DOCKER_HOST=unix:///var/run/docker.sock`. That only works if a native Docker Engine is +running alongside Docker Desktop; if the Desktop daemon is the only one, disabling Ryuk is the way. + +### Requirements and invocation + - **Runtime**: Docker (or a Docker-compatible daemon) must be available so [testcontainers](https://testcontainers.com/) can start **PostgreSQL** and **MinIO** containers. -- **Dependencies**: install dev requirements (includes `testcontainers[postgres,minio]`). +- **Dependencies**: install dev requirements (includes `testcontainers[postgres,minio,mongodb]`). - **Invocation**: run only integration-marked tests, for example: `pytest e2e/ -m integration`. - **MinIO tests**: `e2e/test_minio_offload.py` exercises real S3 uploads; other E2E modules continue to mock MinIO on the worker used by most scenarios. - **OPC tests (real server)**: `e2e/test_opc_real_server.py` uses an in-process **asyncua** server and real `OpcRepository` (`test_activities_real_opc`). Scenarios 3.1.2, 3.2.2, 3.2.4, and 3.2.5 are covered there. Other E2E modules keep the OPC mock. - Run only OPC real-server tests: `pytest e2e/test_opc_real_server.py -m "integration and opc"`. +- **Model import tests**: the `import_model` suite is opt-in and has its own harness, containers and + fidelity contract — see [§4.0](#40-running-the-import-e2e-tests). Run it with + `pytest e2e -m import_e2e`. +- **joblib flavor tests**: `e2e/test_predictions_batch_joblib_flavor.py` covers reading a model + through the explicit `joblib` flavor (predict and transform), and + `e2e/test_retrain_joblib_flavor.py` covers writing one back on a retrain — see + [§5](#5-joblib-flavor-scenarios). Neither `joblib.dump` nor `joblib.load` is mocked in either + module: the artifact fixture holds a real `joblib.dump(..., compress=3)` pickle whose class ships + inside the artifact's `code/` directory, so only the flavor's own `sys.path` handling makes it + deserialize. Run them with + `pytest e2e/test_predictions_batch_joblib_flavor.py e2e/test_retrain_joblib_flavor.py -m integration`. +- **Nothing under `e2e/` is covered by `validate`**: that script lints `laborious/` and `tests/` + only, and its test step is `pytest tests/` (`testpaths = ["tests"]`). Lint these files by hand + (`ruff format --check e2e/ && ruff check e2e/`), and remember that `addopts` carries + `--strict-markers`: a test without `@pytest.mark.integration` is not collected, and a run that + collects nothing still exits 0. Check the collected count, not the exit code. ## Workflow Overview @@ -561,4 +620,583 @@ These paths do **not** rely on Temporal activity retries for export failures: th - Error notification sent - Workflow completes ---- \ No newline at end of file +--- +## 4. Model Import Workflow Scenarios (`import_model`) + +The `import_model` workflow turns an encrypted `.sientia` bundle already in MinIO into a registered +MLflow model plus one MongoDB `models` document — or into a durable error on the row the BFF +inserted. It shares no code path with the predictions workflows above and has its own harness. + +### 4.0 Running the import E2E tests + +- **Marker**: every test below carries `import_e2e`; the real-clock retry tests also carry `slow`. + `testpaths` stays `["tests"]`, so none of this is collected by the unit gate. +- **Invocation**: `pytest e2e -m import_e2e`. +- **Runtime**: Docker for three containers (the shared MinIO, a dedicated Postgres for + `sientia-core-mlops-bff`, MongoDB) **plus** an `mlflow server` subprocess started from this + repository's venv. Dev requirements install `testcontainers[postgres,minio,mongodb]`. +- **Fidelity**: on the import path nothing is mocked — real MinIO, real Postgres, real MongoDB with a + unique index, a real MLflow tracking server with a working registry, real Argon2id and real + `secretstream`. Only `CoreNotificationHandler`'s Mongo client stays mocked, because notifications + are a side channel this suite counts rather than a dependency it tests. +- **Fixtures**: `e2e/import_harness.py`, registered as a pytest plugin from `e2e/conftest.py`. Every + fixture is `import_`-prefixed. +- **Prerequisite**: the one-time reaper setup at the top of this document + ([Running automated E2E tests](#one-time-setup-disable-the-testcontainers-reaper)). Without it every + test here errors at setup with a Docker mount error and never reaches an assertion. + +### 4.1 Success Scenarios + +--- + +#### Scenario 4.1.1: Producer-Emitted Bundle Imports End to End +**Description**: The committed golden `.sientia` file — bytes the producer's own code wrote — is +imported at the producer's real Argon2id cost (256 MiB, 3 passes) + +**Input**: +- `tests/fixtures/bundle/golden_model_v3.sientia` uploaded under `imported_models/` +- One `PENDING` / `run_type='IMPORT'` row carrying the object key and the uploader's digest +- The workflow input: `import_run_id`, `bucket`, `object_key`, `expected_digest`, `password_envelope` + +**Expected Behavior**: +- The row is claimed, the seven gates pass, the experiment and a new run are created +- The artifact tree is logged with `prediction_model` and `data_model` at the run's root +- A registry version is created and left unstaged; the model document is written inactive +- One terminal write sets `IMPORT_SUCCESS` + +**Assertions**: +- Exactly one run named `import-fixture_model-v3`, status `FINISHED` +- `prediction_model` contains `MLmodel` and `model.pkl` +- Bundle parameters logged, origin logged under `source.*` +- Destination version is `1`, independent of the bundle's `model_version` of `3` +- Model document `id` is the seeded maximum plus one; `active` is `false` +- Row: `IMPORT_SUCCESS`, `error_message` null, both names filled + +--- + +#### Scenario 4.1.2: Factory-Built Bundle Imports End to End +**Description**: The same assertions over a synthesised tree with names unique to the test + +**Input**: A bundle from `tests/helpers/bundle_factory.py` at the cheap KDF cost + +**Expected Behavior**: As 4.1.1 + +**Assertions**: +- As 4.1.1, plus `model_config` holds exactly `transform_flavor`, `predict_flavor`, + `retention_minutes` and `target` +- `import.workflow_id` in the row's detail JSON matches the run + +--- + +#### Scenario 4.1.3: The Registry Source Resolves Through the Legacy Loader +**Description**: The `source` the importer writes must be one the *existing* loader can parse + +**Input**: A successful import's registered version + +**Expected Behavior**: `MLFlowRepository.get_model_run_id` extracts the run id from the source + +**Assertions**: +- The source ends in `/prediction_model` +- `get_model_run_id` returns the run the import created +- Both artifact paths are present under that run +- Called with the stage the version is actually in, because an import promotes nothing + +--- + +#### Scenario 4.1.4: Second Import of the Same Model +**Description**: The same model imported twice, with different declared `model_version`s + +**Input**: Two bundles declaring `model_version` `2` then `7`, same model and experiment names + +**Expected Behavior**: Additive — a new run and a new registry version, nothing promoted, the +existing model document untouched + +**Assertions**: +- Two versions (`1`, `2`) of one registered model; destination numbering ignores the bundle's +- Two runs in the same experiment, each logging its own `source.model_version` +- No version in `Production` +- The model document is byte-identical to what the first import wrote + +--- + +#### Scenario 4.1.5: A Model Name Already in the Collection +**Description**: An operator-owned document already carries the bundle's model name + +**Input**: A document with `active: True`, `predict_flavor: pyfunc` and a hand-set target + +**Expected Behavior**: The import notices the name and leaves the document alone; the import still +succeeds + +**Assertions**: +- The collection is unchanged, exactly one document carries that name, no duplicate +- Row: `IMPORT_SUCCESS` — a document that is already there is not a failure + +--- + +### 4.2 Gate Rejection Scenarios + +Every case asserts the same five things: terminal status `ORCHESTRATOR_VALIDATION_ERROR`, the +recorded step, the recorded code, `error_message` **equal** to the catalog sentence for that step, +and that no experiment, run, artifact, registry version, model document or extracted file remains. + +--- + +#### Scenario 4.2.1: Object Above the Size Ceiling (gate 1) +**Description**: An object larger than `IMPORT_MAX_OBJECT_BYTES`, rejected from `stat_object` before +its body is fetched + +**Input**: A factory bundle against a ceiling narrowed to 1 KiB for this test class + +**Expected Behavior**: Rejected at `download` + +**Assertions**: The five shared assertions + +--- + +#### Scenario 4.2.2: Malformed Header (gate 2) +**Description**: Five variants — wrong magic, unsupported format version, KDF memory above the +ceiling, KDF passes above the ceiling, a header digest that does not describe the chunk stream + +**Expected Behavior**: Each rejected at `download`, sharing gate 1's step + +**Assertions**: +- The five shared assertions +- The recorded sentence does **not** mention the password: gates 1 and 2 must not implicate a + password that is fine + +--- + +#### Scenario 4.2.3: Decryption Failures (gate 3) +**Description**: Three variants — a wrong bundle password inside a valid envelope, a tampered chunk +with the header digest repaired, and a stream whose every chunk is framed `MESSAGE` + +**Expected Behavior**: Each rejected at `decryption` + +**Assertions**: +- The five shared assertions +- The tampered case proves the file is self-consistent and only the AEAD can tell +- The framing case proves the digest cannot see an incomplete stream + +--- + +#### Scenario 4.2.4: Hostile Archive (gate 4) +**Description**: Four variants — an entry escaping the tree, an absolute entry path, a symbolic +link, and a declared expansion over the ceiling + +**Expected Behavior**: Each rejected at `archive_inspection`, from the central directory alone + +**Assertions**: +- The five shared assertions +- **Nothing extracted**: the worker work directory is empty, not merely free of provisioning + +--- + +#### Scenario 4.2.5: A Member Extraction Cannot Place (gate 5) +**Description**: An archive declaring the same member name twice — passes gate 4, which checks names +and modes but not duplicates + +**Expected Behavior**: Rejected at `extraction` on the second member + +**Assertions**: The five shared assertions + +--- + +#### Scenario 4.2.6: Incomplete Structure (gate 6) +**Description**: Five variants — no `metadata.json`, metadata missing `run_id`, parameters missing +`target_variable`, no `prediction_model` directory, and a 60-character experiment name + +**Expected Behavior**: Each rejected at `structure_validation` + +**Assertions**: +- The five shared assertions +- For the over-long name: the row's `experiment_name` is still the inserted placeholder, so no write + reached the `VARCHAR(50)` column, and that column is confirmed still 50 wide + +--- + +#### Scenario 4.2.7: Forbidden File Type (gate 7) +**Description**: An `artifacts/payload.exe` member with a suffix outside the allow-list + +**Expected Behavior**: Rejected at `content_policy` + +**Assertions**: The five shared assertions + +--- + +### 4.3 Digest Enforcement Scenarios + +--- + +#### Scenario 4.3.1: A Digest Computed Over Other Bytes +**Description**: A perfectly valid bundle whose `expected_digest` belongs to a different file + +**Expected Behavior**: Fails at `download`, before any header is parsed + +**Assertions**: +- Terminal status, step, code and catalog sentence +- The failure names the digest, not the magic or the version +- `run_name` is still null and `experiment_name` still the placeholder — the bundle was never opened + +--- + +#### Scenario 4.3.2: The Object Is Replaced After Upload +**Description**: The same key, different bytes — also a valid bundle, so nothing but the digest can +tell + +**Expected Behavior**: Fails at `download` + +**Assertions**: As 4.3.1; only a real object store can show this at all + +--- + +#### Scenario 4.3.3: No Digest at All +**Description**: `expected_digest` absent from the input + +**Expected Behavior**: Fails at `received`, before anything is downloaded + +**Assertions**: The request-invalid code, and the work directory was never created + +--- + +### 4.4 Provisioning Failure Scenarios + +Each asserts the platform terminal status `IMPORT_ERROR` — not the validation status, because a +provisioning failure is not the user's to fix — its own step and code, the workflow identifier in +the detail JSON, and `error_message` **equal** to the catalog sentence. + +--- + +#### Scenario 4.4.1: The Tracking Server Is Stopped +**Description**: The `mlflow server` process is genuinely terminated for the duration + +**Expected Behavior**: Fails at `experiment_creation` + +**Assertions**: +- The shared assertions +- The names are already on the row: the bundle was opened before MLflow was reached +- Nothing in Mongo; nothing provisioned + +--- + +#### Scenario 4.4.2: Artifact Upload Fails +**Description**: `log_artifacts` is patched — the one step where neither hostile input nor a broken +dependency can isolate the failure + +**Expected Behavior**: Fails at `artifact_upload` + +**Assertions**: +- The shared assertions +- The experiment exists (the previous step succeeded); no version, no document + +--- + +#### Scenario 4.4.3: Registration Fails +**Description**: `create_model_version` is patched. MLflow rejects only an empty registered-model +name, and one process serves both tracking and the registry, so nothing else isolates this step + +**Expected Behavior**: Fails at `registration` + +**Assertions**: +- The shared assertions +- The run and both artifact paths survive — "the files were stored, the model was not registered" + +--- + +#### Scenario 4.4.4: The Model Document Is Refused +**Description**: A real unique index on `model_config.target` refuses the activity's own insert. +(A duplicate `id` cannot be constructed: the activity derives `max(int(id)) + 1` from the same +collection, so the id it computes can never already exist) + +**Expected Behavior**: Fails at `model_document` + +**Assertions**: +- The shared assertions +- One registry version exists — registered, and unusable + +--- + +### 4.5 Status Record Scenarios + +--- + +#### Scenario 4.5.1: The Claim Precedes Every Gate +**Description**: The row is observed from inside `stat_object`, the first call gate 1 makes + +**Assertions**: Running status and `import.workflow_id` already present at that moment + +--- + +#### Scenario 4.5.2: The Names Are Visible Mid-Flight +**Description**: The row is observed from inside the first call experiment creation makes, during an +import that then succeeds + +**Assertions**: `experiment_name` and `run_name` already populated before anything exists in MLflow + +--- + +#### Scenario 4.5.3: The Inserting Side's Columns Are Never Rewritten +**Description**: After a success and after a failure + +**Assertions**: `username`, `request_data`, `bucket_name`, `file_name`, `run_type`, `created_at` +unchanged; `status` and `updated_at` did change + +--- + +#### Scenario 4.5.4: Unclaimable Rows +**Description**: Three variants — an unknown id, a `run_type='TRAIN'` row, and a row already held by +another workflow id + +**Expected Behavior**: The Temporal run ends failed with the not-claimable error type, and **nothing +is written to any row** + +**Assertions**: The error type from the history, the whole table byte-identical, nothing provisioned + +--- + +#### Scenario 4.5.5: The Row Names a Different Object +**Description**: `file_name` disagrees with the input's `object_key` + +**Expected Behavior**: Fails at `received` **with the error recorded on the row** — unlike 4.5.4, +this row was successfully claimed + +**Assertions**: The received code and sentence on the row; nothing provisioned + +--- + +#### Scenario 4.5.6: Constraint Conformance +**Description**: The real constraints, exercised directly + +**Assertions**: +- Every status a normal import writes is accepted on an `IMPORT` row +- All four terminal values are refused across run types, naming + `chk_experiment_run_terminal_status_run_type`; the shared stage values are accepted on both +- A training row still walks its own vocabulary +- The table has exactly the 13 columns `V1`…`V12` leave (no `ALTER TABLE` by this feature) + +--- + +### 4.6 Absence Scenarios + +--- + +#### Scenario 4.6.1: One Table, One Database +**Description**: A full import bracketed by marker statements on both Postgres servers, read from +`log_statement=all` + +**Assertions**: +- Every statement touching a user relation names `experiment_run` +- No `project`, `models`, `import_run`, `model_project` or `model_registry` reference (with quoted + literals stripped, so the bundle's own `model_project` *value* is not mistaken for a table) +- The predictions database logs **no** statement at all, though the import holds working credentials + for it +- A rejected import — which writes more status rows than a successful one — behaves the same + +--- + +### 4.7 Cleanup, Retention and History Scenarios + +--- + +#### Scenario 4.7.1: The Worker Filesystem Is Clean +**Description**: After a success and after rejections at two different steps (decryption and +structure validation, which unwind on different code paths) + +**Assertions**: No extracted tree, no downloaded object, no temporary archive + +--- + +#### Scenario 4.7.2: The Retention Rule Is Present +**Assertions**: `sientia-import-bundle-expiry` on the import bucket, enabled, filtered on +`imported_models/`, expiring in 7 days — and the uploaded object still there, because the import +never deletes it + +--- + +#### Scenario 4.7.3: The Record Outlives the Object +**Description**: The uploaded object is deleted, as retention will eventually do + +**Assertions**: The row still reads `IMPORT_SUCCESS` with its names + +--- + +#### Scenario 4.7.4: No Plaintext Secret in the Event History +**Assertions**: +- The bundle password, the base64 envelope key and the key's hex form appear nowhere in the history +- `password_envelope` **is** in the history, so the negative assertions above cannot pass vacuously + +--- + +### 4.8 Retry Envelope and Missing-`V12` Scenarios + +These run on a **real clock** (`WorkflowEnvironment.start_local()`) with the status retry intervals +shortened; the production values stay asserted by the unit suite. All are marked `slow`. + +--- + +#### Scenario 4.8.1: A Database That Comes Back +**Description**: The status database is stopped for two seconds after provisioning and restarted +inside the retry envelope + +**Assertions**: The terminal write lands `IMPORT_SUCCESS` with the correct workflow id + +--- + +#### Scenario 4.8.2: A Broken Progress Update +**Description**: A brief early outage that only the progress hints see + +**Assertions**: The import completes with the correct verdict and does not spend the terminal +write's envelope on a hint nobody reads + +--- + +#### Scenario 4.8.3: A Database That Stays Down After a Successful Import +**Description**: The database is taken away as soon as the row is claimed and never comes back + +**Expected Behavior**: The model is fully provisioned and the workflow **completes** — reporting +failure would tell somebody to re-import a model that is already on the platform + +**Assertions**: +- The run returns its result; the model, version and document all exist +- The failure channel carries `IMPORT_STATUS_NOT_RECORDED`, `step=status_write` and + `verdict_not_recorded=` +- The row is left in the running status + +--- + +#### Scenario 4.8.4: A Failed Import Whose Failure Cannot Be Written Either +**Description**: The import fails at `artifact_upload` and the terminal write is then refused + +**Assertions**: +- The reported cause is the artifact-upload failure, not the status write +- The status write reports itself only through the channel +- The row is left running + +--- + +#### Scenario 4.8.5: A Table Without `V12` +**Description**: The evidence for why the migration gates enabling the import worker + +**Expected Behavior**: The claim succeeds (`V11` gave the table `run_type`; `PENDING` and the running +status are pre-`V12` values), the model is fully provisioned, and only the terminal value is new — so +the last write of the import is the first thing that fails, non-retryably + +**Assertions**: +- The run completes; the model, version and document all exist +- The row is left in the running status, with both names filled +- The channel carries the not-recorded code and the verdict +- Elapsed time shows the refusal was **not** retried +- `pg_constraint` confirms the fixture's `chk_experiment_run_status` really does not allow + `IMPORT_SUCCESS`, so the test cannot pass vacuously + +--- + +## 5. joblib Flavor Scenarios + +`joblib` is an explicit model flavor, declared in `model_config.predict_flavor` / +`model_config.transform_flavor` like any other. It reads the artifact directly — download the +`.pkl`, put the artifact's `code/` on `sys.path`, `joblib.load` — and **never** calls an MLflow +flavor loader. There is no fallback in either direction: a failing loader propagates. + +Covered by `e2e/test_predictions_batch_joblib_flavor.py` (read) and +`e2e/test_retrain_joblib_flavor.py` (write). Both use the `joblib_flavor_artifacts` fixture, which +builds one artifact directory per model type holding a real compressed `model.pkl` plus a `code/` +directory carrying the module where the model's class lives. The fixture takes that directory off +`sys.path` again before yielding, so the pickle is **unloadable** until the flavor puts it back — +that is what makes these tests prove module resolution rather than assume it. The fixture also +restores `sys.path` in teardown, because the flavor deliberately never removes what it inserted. + +### 5.1 Read Scenarios + +--- + +#### Scenario 5.1.1: Predict Through the joblib Flavor +**Description**: `predict_flavor: 'joblib'` on a normal `predictions_batch` run + +**Input**: +- `model_config.predict_flavor = 'joblib'`, `transform_flavor = 'sklearn'` +- `client.download_artifacts` serving the real `prediction_model` artifact + +**Expected Behavior**: +- The artifact is downloaded once and deserialized with `joblib.load` +- No `models://production` URI is built, so no flavor loader ever sees this model +- The prediction the artifact's own object produces is what lands in Postgres + +**Assertions**: +- Exactly one prediction row, `prediction_status='Good'`, value matching the artifact's model +- `download_artifacts` called once, with `'prediction_model'` +- No `mlflow.{sklearn,pyfunc,pytorch}.load_model` call names this model + +--- + +#### Scenario 5.1.2: Transform Through the joblib Flavor +**Description**: `transform_flavor: 'joblib'`, which had no E2E coverage before + +**Input**: +- `model_config.transform_flavor = 'joblib'`, `predict_flavor = 'sklearn'` +- `client.download_artifacts` serving the real `data_model` artifact + +**Expected Behavior**: +- `data_model` is downloaded and deserialized; the prediction model stays on its MLflow flavor +- No artifact URI is resolved for the transform, so `mlflow.get_run` is never called + +**Assertions**: +- The prediction lands with the mocked prediction model's value +- `download_artifacts` called once, with `'data_model'` +- `mlflow.get_run` not called; no flavor loader was given a `/data_model` URI + +--- + +#### Scenario 5.1.3: An MLflow Flavor Is Unaffected +**Description**: A model declaring `sklearn` behaves exactly as before + +**Assertions**: +- The prediction lands with the mocked model's value +- `download_artifacts` NOT called — the joblib path is never entered as a safety net + +--- + +### 5.2 Error Scenarios + +--- + +#### Scenario 5.2.1: joblib Flavor Failure Has a Single Cause +**Description**: The downloaded artifact has no top-level `.pkl` + +**Expected Behavior**: The workflow stops. The reported cause is the missing pickle and nothing +else — with no fallback there is no second exception chained to a first + +**Assertions**: +- The run stops (`assert_stop`) +- The log carries `No .pkl file found in prediction_model artifact` +- The log does **not** carry `Original error`, the marker of a chained cause + +--- + +### 5.3 Write Scenarios + +--- + +#### Scenario 5.3.1: Retraining a joblib Model Round-Trips +**Description**: `retrain_model` on a model whose `predict_flavor` and `transform_flavor` are both +`joblib` — write and read in one test + +**Input**: +- Training frame indexed by `DATETIME_FORMAT_WITH_TZ` timestamps, carrying the target column +- MLflow's artifact I/O replaced by a fake store that copies into an inspectable run directory + +**Expected Behavior**: +- Both models load through the joblib flavor; no flavor loader runs +- `log_model` writes `prediction_model/model.pkl` (a `joblib.dump(..., compress=3)` pickle) and + re-logs `prediction_model/code/`, without an `MLmodel` manifest +- Loading the **written** artifact through the joblib flavor deserializes it and predicts, after + the module the first load imported has been dropped from `sys.modules` + +**Assertions**: +- `retrain_model` returns `success: True` +- `download_artifacts` was asked for both `data_model` and `prediction_model` +- `sklearn`/`pyfunc` loaders have zero calls +- The written run has `prediction_model/model.pkl`, `prediction_model/code/.py` and + `data_model/model.pkl`, and **no** `prediction_model/MLmodel` +- The reloaded model's prediction equals the original's, and the written `code/` is on `sys.path` + +--- diff --git a/e2e/test_import_model_absence.py b/e2e/test_import_model_absence.py new file mode 100644 index 0000000..98f164f --- /dev/null +++ b/e2e/test_import_model_absence.py @@ -0,0 +1,184 @@ +"""One table, one database — proven from the servers' own logs, not from the application's side. + +Decision D22 says an import writes to `public.experiment_run` and to nothing else relational: no +project table, no model table, no row in the predictions database. That claim is only worth making +if it is checked from somewhere that sees *every* client, and there is exactly one such place. + +Why not the obvious alternatives: + +- a SQLAlchemy `before_cursor_execute` listener sees the status engine and is blind to the psycopg + pool `Storage` holds — which is precisely the client that would betray the claim; +- `pg_stat_statements` is aggregated, so it cannot be scoped to the window of one import. + +So both containers run with `log_statement=all` and the window is bracketed by a marker statement on +each server. The assertions are about *which relation names appear*, not about exact statement text, +because the text is the driver's business and changes with it. + +The predictions database is deliberately real and reachable here: `import_activities` is built with +working credentials for it, so "no statement reached it" is a claim about the import's behaviour and +not about a missing connection. +""" + +from __future__ import annotations + +from typing import Any + +import pytest +from temporalio.client import WorkflowFailureError + +from e2e.import_harness import names_no_relation, strip_literals + +pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio] + +FACTORY_PASSWORD = 'Factory1Model!Test' + +# Relations that exist in the predictions database and in nobody's import path. +FOREIGN_RELATIONS = ( + 'predictions_schema', + 'laborious_data', + 'transformed_data', + 'predictions', +) + +# Tables a reader might expect an import to want, and which decision D22 says do not exist. +RELATIONS_D22_DENIES = ('project', 'model_registry', 'models', 'import_run', 'model_project') + + +class TestOneTableOneDatabase: + """A full import, from claim to terminal write, watched from both servers.""" + + async def test_every_statement_the_import_issues_names_experiment_run( + self, + import_status_table: Any, + import_models_collection: Any, + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_log_window: Any, + ) -> None: + object_key, digest = import_upload_bundle(import_bundle_factory()) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + + with import_log_window() as captured: + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + await handle.result() + + row = import_read_status_row(import_run_id) + assert row is not None + assert row['status'] == 'IMPORT_SUCCESS', ( + 'the window has to bracket a *working* import, or it proves nothing' + ) + + bff_statements = captured['bff'] + assert bff_statements, 'the status database logged nothing at all — the window is wrong' + + touching_a_relation = [ + statement for statement in bff_statements if not names_no_relation(statement) + ] + assert touching_a_relation, 'no relational statement was logged, so nothing was checked' + for statement in touching_a_relation: + assert 'experiment_run' in statement, ( + f'a statement naming another relation reached the status database: {statement!r}' + ) + + # And no relation a reader might expect the feature to want appears anywhere in the window. + # Literals are stripped first: the import writes the bundle's own `model_project` value into + # the JSON detail, and `factory_project` is data, not a reference to a `project` table. + window_text = '\n'.join(strip_literals(s) for s in bff_statements).lower() + for relation in RELATIONS_D22_DENIES: + assert relation not in window_text, ( + f'{relation!r} appears in the import window: D22 says no such table is written' + ) + for relation in FOREIGN_RELATIONS: + assert relation not in window_text, f'{relation!r} appears in the import window' + + async def test_the_predictions_database_sees_no_statement_from_an_import( + self, + import_status_table: Any, + import_models_collection: Any, + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_log_window: Any, + ) -> None: + """Not "no statement against our tables" — no statement at all. + + The import holds a working connection to this server (`Activities` is built with one), so + this is about what the import chooses to do, not about what it is able to do. + """ + object_key, digest = import_upload_bundle(import_bundle_factory()) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + + with import_log_window() as captured: + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + await handle.result() + + assert import_read_status_row(import_run_id)['status'] == 'IMPORT_SUCCESS' + + predictions_statements = [ + statement for statement in captured['predictions'] if not names_no_relation(statement) + ] + assert not predictions_statements, ( + f'the import issued statements against the predictions database: ' + f'{predictions_statements}' + ) + + async def test_a_rejected_import_also_touches_nothing_else( + self, + import_status_table: Any, + import_models_collection: Any, + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_log_window: Any, + ) -> None: + """The failure path writes more status rows than the success path, so it is worth its own case. + + A rejected import runs the claim, the progress hints, cleanup and the terminal write — more + statements than a success, and every one of them still has to name the same table. + """ + object_key, digest = import_upload_bundle(import_bundle_factory(include_metadata=False)) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + + with import_log_window() as captured: + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + with pytest.raises(WorkflowFailureError): + await handle.result() + + for statement in captured['bff']: + if names_no_relation(statement): + continue + assert 'experiment_run' in statement, statement + assert not [s for s in captured['predictions'] if not names_no_relation(s)] diff --git a/e2e/test_import_model_cleanup.py b/e2e/test_import_model_cleanup.py new file mode 100644 index 0000000..9d94509 --- /dev/null +++ b/e2e/test_import_model_cleanup.py @@ -0,0 +1,247 @@ +"""What the worker is left holding, what the bucket is left promising, and what the history reveals. + +Three things an import must be true about after it finishes, whichever way it finished: + +- **The worker's disk is empty of it.** No extracted tree, no downloaded object, no decrypted zip. + The bundle is decrypted onto local disk, so a leak here is a plaintext model sitting in a scratch + directory for the next import to trip over — and after a *failure* is exactly when that would go + unnoticed. +- **The bucket carries the retention rule.** The import never deletes the uploaded object: it is the + record of an attempt, and the lifecycle rule is what eventually removes it. That rule is applied + by the importer rather than assumed from a Helm value, so it is checkable from here. +- **The status record outlives the object**, because it is the audit trail and the object is not. + +And the one claim that can only be made against a real Temporal history: no plaintext password +anywhere in it. The workflow carries an envelope it cannot open, and the activity that opens it +never returns what it found. That is the design; this reads the history and checks it. +""" + +from __future__ import annotations + +from typing import Any + +import pytest +from temporalio.client import WorkflowFailureError + +from e2e.import_harness import IMPORT_BUCKET, IMPORT_PREFIX +from tests.helpers.bundle_factory import DEFAULT_ENVELOPE_KEY, envelope_key_b64 + +pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio] + +FACTORY_PASSWORD = 'Factory1Model!Test' +RETENTION_RULE_ID = 'sientia-import-bundle-expiry' + + +async def _run( + *, + payload: bytes, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + password: str = FACTORY_PASSWORD, +) -> tuple[Any, int, str]: + object_key, digest = import_upload_bundle(payload) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(password), + ) + ) + return handle, import_run_id, object_key + + +class TestWorkerFilesystem: + """Nothing of the import is left on the worker — on the success path or on either failure path.""" + + @pytest.mark.parametrize( + ('case', 'factory_kwargs', 'expect_success'), + [ + ('success', {}, True), + ('rejected at decryption', {'drop_final_tag': True}, False), + ('rejected at structure validation', {'include_metadata': False}, False), + ], + ) + async def test_the_work_directory_is_empty_afterwards( + self, + case: str, + factory_kwargs: dict[str, Any], + expect_success: bool, + import_status_table: Any, + import_models_collection: Any, + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + tmp_path: Any, + ) -> None: + """Two different failure steps, because they clean up at different points. + + A decryption failure leaves a partially written zip in the staging directory; a structure + failure has already extracted a whole tree. The reader unwinds both, and the cleanup activity + removes the scratch directory after either — but they are separate code paths, so asserting + one would not cover the other. + """ + handle, _, _ = await _run( + payload=import_bundle_factory(**factory_kwargs), + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + if expect_success: + await handle.result() + else: + with pytest.raises(WorkflowFailureError): + await handle.result() + + work_dir = tmp_path / 'import-work' + leftovers = sorted(str(path) for path in work_dir.rglob('*')) if work_dir.exists() else [] + assert not leftovers, f'{case}: the worker still holds {leftovers}' + + +class TestRetention: + """The rule the uploaded object's eventual removal depends on.""" + + async def test_the_import_bucket_carries_the_retention_rule( + self, + import_status_table: Any, + import_models_collection: Any, + import_bundle_factory: Any, + import_bucket: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + ) -> None: + handle, _, object_key = await _run( + payload=import_bundle_factory(), + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + await handle.result() + + lifecycle = import_bucket.get_bucket_lifecycle(IMPORT_BUCKET) + assert lifecycle is not None, 'the bucket carries no lifecycle configuration at all' + rules = {rule.rule_id: rule for rule in lifecycle.rules} + assert RETENTION_RULE_ID in rules, ( + f'no {RETENTION_RULE_ID!r} rule; the bucket has {sorted(rules)}' + ) + rule = rules[RETENTION_RULE_ID] + assert rule.status == 'Enabled' + assert rule.rule_filter.prefix == IMPORT_PREFIX + assert rule.expiration.days == 7 + + # The import did not delete the uploaded object: it is the record of an attempt, and the + # rule above is what removes it later. + assert import_bucket.stat_object(IMPORT_BUCKET, object_key).size > 0 + + async def test_the_status_record_outlives_the_uploaded_object( + self, + import_status_table: Any, + import_models_collection: Any, + import_bundle_factory: Any, + import_bucket: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + """Deleting the object is what the retention rule will eventually do; the verdict must survive. + + The record is the audit trail. If reading a finished import's outcome depended on the bundle + still being in the bucket, every import would become unreadable after seven days. + """ + handle, import_run_id, object_key = await _run( + payload=import_bundle_factory(), + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + await handle.result() + + import_bucket.remove_object(IMPORT_BUCKET, object_key) + with pytest.raises(Exception): # noqa: B017 - MinIO's own not-found type is not the subject + import_bucket.stat_object(IMPORT_BUCKET, object_key) + + row = import_read_status_row(import_run_id) + assert row is not None + assert row['status'] == 'IMPORT_SUCCESS' + assert row['file_name'] == object_key + assert row['run_name'] is not None + + +class TestEventHistorySecrecy: + """The claim that only a real event history can settle.""" + + async def test_no_plaintext_password_or_key_appears_in_the_history( + self, + import_status_table: Any, + import_models_collection: Any, + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_history_events: Any, + ) -> None: + """Dump every event of a completed import and search the whole thing. + + Three values must be absent, and they are absent for three different reasons: + + - the **bundle password** never crosses an activity boundary — `open_import_bundle` decrypts + the envelope inside itself and returns metadata, never the password; + - the **envelope key** is configuration, read from the environment inside the activity, so it + is never an argument to anything; + - the **decrypted password** is the first of these two by another name, and is searched for + separately because a future refactor that returned it "just for logging" would pass a test + that only looked for the literal input. + + The envelope itself *is* in the history, and must be: it is how the password reaches the + activity. That it is there is asserted too, so this test cannot pass by finding nothing at + all — which is the way an assertion like this usually rots. + """ + password = 'Secret1Password!InHistory' + handle, _, _ = await _run( + payload=import_bundle_factory(password=password), + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + password=password, + ) + await handle.result() + + events, blob = await import_history_events(handle) + assert events, 'the history is empty' + + assert password not in blob, 'the bundle password appears in the event history' + assert envelope_key_b64(DEFAULT_ENVELOPE_KEY) not in blob, ( + 'the envelope key appears in the event history' + ) + assert DEFAULT_ENVELOPE_KEY.hex() not in blob.lower(), ( + 'the envelope key appears in the event history, hex-encoded' + ) + + # The negative assertions above are only meaningful if the history is actually readable and + # actually carries the envelope. Without this, an empty or opaque history would "pass". + assert 'password_envelope' in blob, ( + 'the history does not even carry the envelope, so the search above proved nothing' + ) diff --git a/e2e/test_import_model_gates.py b/e2e/test_import_model_gates.py index 593c7fc..be9ab0e 100644 --- a/e2e/test_import_model_gates.py +++ b/e2e/test_import_model_gates.py @@ -400,6 +400,10 @@ class TestGate5Extraction: and modes, not duplicates — and is refused here on the second member. """ + # `zipfile` warns while *writing* the duplicate entry. That is the fixture doing exactly what + # this test needs, so it is silenced here rather than left to look like a problem — and scoped to + # this one test, so a duplicate name appearing anywhere else still gets noticed. + @pytest.mark.filterwarnings('ignore:Duplicate name:UserWarning') async def test_a_duplicate_member_is_rejected_at_extraction( self, import_status_table: Any, diff --git a/e2e/test_import_model_provisioning_failures.py b/e2e/test_import_model_provisioning_failures.py index a0f15e0..1062db1 100644 --- a/e2e/test_import_model_provisioning_failures.py +++ b/e2e/test_import_model_provisioning_failures.py @@ -13,15 +13,19 @@ failure is always the second kind. | `experiment_creation` | the MLflow server is genuinely stopped | broken dependency | | `artifact_upload` | targeted patch on `log_artifacts` | patch, justified in place | | `registration` | targeted patch on `create_model_version` | patch, justified in place | -| `model_document` | a document already holding the id the activity will compute, against the real unique index | broken dependency | +| `model_document` | a real unique index refuses the activity's own insert | broken dependency | -Two patches rather than the one design E7 anticipated. The reason is the same for both and is -recorded in each test: with one server serving both tracking and registry, and MLflow validating -nothing about a registered model name except that it is non-empty (which gate 6 already requires), -there is no way to make the world hostile to *just* one of these two steps. Stopping the server -fails experiment creation first; a malformed name is not malformed to MLflow. The patch is on the -client call the step makes, not on the activity or the workflow, so what is being tested is still -"the workflow attributes this step's failure to this step". +Two departures from what the tasks anticipated, both recorded in the test that found them: + +- **Registration needs a patch too**, not just artifact upload. One process serves both tracking and + the registry, so stopping it fails experiment creation three steps earlier; and MLflow validates + nothing about a registered model name except that it is non-empty, which gate 6 already requires. + Both patches are on the single client call the step *is*, never on the activity or the workflow, so + what is under test — the workflow attributing this step's failure to this step — is untouched. +- **The model document's unique-index violation is on `model_config.target`**, not on `id`. The + duplicate-`id` state the task describes cannot be constructed at all; `TestModelDocument`'s + docstring shows why, and the mechanism the task wanted — a real index refusing a real insert — is + kept intact. """ from __future__ import annotations @@ -292,9 +296,26 @@ class TestRegistration: class TestModelDocument: - """The last step, failed by a real unique index on a real collection.""" + """The last step, failed by a real unique index refusing the real insert. - async def test_a_duplicate_id_fails_at_model_document( + **Why the index is on `model_config.target` and not on `id`.** Task 7.4 asks for "a document + already holding the id the activity is about to compute". That state cannot be constructed, and + why is worth more than the test would have been: `_next_model_document_id` computes + `max(int(id)) + 1` over `distinct('id')` of the *same* collection, so every value that + stringifies to the colliding id is counted by that maximum. Put a document at `'8'` and the next + id becomes `'9'`. Make the id unparseable and it is ignored by the numbering *and* does not + collide. There is no third case — the numbering cannot collide with itself, and the only real + duplicate-key path on `id` is two imports racing, which this change lists as a non-goal. That is + a good property of the implementation, not a gap. + + So the mechanism the task asked for is kept — a real unique index refusing the activity's own + insert — on a key that can actually collide. `model_config.target` is `'SE'` on the seeded + documents and `'SE'` on the one this import writes, so a unique index there makes the insert a + genuine `DuplicateKeyError` from the server. No patch, no mock: the same class of failure, from + the same enforcement mechanism, at the same step. + """ + + async def test_a_unique_index_violation_fails_at_model_document( self, import_status_table: Any, import_models_collection: Any, @@ -308,50 +329,47 @@ class TestModelDocument: import_envelope: Any, import_read_status_row: Any, ) -> None: - """A document already holding the id `max(int(id)) + 1` will compute, under another name. - - Under another name on purpose: a document with the *same* name would make the activity skip - the insert entirely, which is a different behaviour (task 8.6) and would not fail. This one - collides only on `id`, so the insert is attempted and the index refuses it. - - No patch and no mock: the failure is `pymongo`'s duplicate-key error from a real index, - which is the thing that would happen in production if two imports raced. - """ - collision_id = str(max(int(seeded) for seeded in SEEDED_MODEL_IDS) + 1) - import_models_collection.insert_one( - {'id': collision_id, 'name': 'a_different_model', 'active': True, 'model_config': {}} - ) - - handle, import_run_id, object_key = await _upload_and_start( - import_bundle_factory=import_bundle_factory, - import_upload_bundle=import_upload_bundle, - import_insert_status_row=import_insert_status_row, - import_build_input=import_build_input, - import_execute=import_execute, - import_envelope=import_envelope, - ) - with pytest.raises(WorkflowFailureError): - await handle.result() + # All three seeded documents carry target 'SE', so the index has to be buildable first: + # give two of them their own target and leave exactly one holding 'SE'. That one is what the + # import's own document — also target 'SE' — collides with. + for ordinal, seeded_id in enumerate(sorted(SEEDED_MODEL_IDS)[1:], start=1): + import_models_collection.update_one( + {'id': seeded_id}, {'$set': {'model_config.target': f'OTHER_{ordinal}'}} + ) + import_models_collection.create_index('model_config.target', unique=True, name='e2e_target') + try: + handle, import_run_id, object_key = await _upload_and_start( + import_bundle_factory=import_bundle_factory, + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + parameters={'target_variable': 'SE'}, + ) + with pytest.raises(WorkflowFailureError): + await handle.result() + row = import_read_status_row(import_run_id) + finally: + # The collection is dropped and reseeded per test, but dropping the index here keeps the + # failure local even if that ever changes. + import_models_collection.drop_index('e2e_target') _assert_platform_failure( - row=import_read_status_row(import_run_id), + row=row, step=ImportStep.MODEL_DOCUMENT, workflow_id=handle.id, object_key=object_key, ) - # No document for this model, and the colliding one is untouched. + # No document for this model, and the seeded ones are exactly as they were. assert import_models_collection.find_one({'name': import_names['model_name']}) is None - blocker = import_models_collection.find_one({'id': collision_id}, {'_id': False}) - assert blocker == { - 'id': collision_id, - 'name': 'a_different_model', - 'active': True, - 'model_config': {}, - } + assert { + str(document['id']) for document in import_models_collection.find({}, {'_id': False}) + } == set(SEEDED_MODEL_IDS) - # Everything before it stands: "the model was registered, but the settings ... could not be - # saved" is the catalog sentence, and this is what makes it true rather than merely stored. + # Everything before it stands — registered, and unusable, which is the point of the split + # between this step's sentence and the registration step's. mlflow_client = _fresh_client(import_mlflow_server) versions = list(mlflow_client.search_model_versions(f"name='{import_names['model_name']}'")) assert len(versions) == 1 diff --git a/e2e/test_import_model_retry_recovery.py b/e2e/test_import_model_retry_recovery.py new file mode 100644 index 0000000..03db085 --- /dev/null +++ b/e2e/test_import_model_retry_recovery.py @@ -0,0 +1,514 @@ +"""What happens when the *record* cannot be written — the cases that gate this feature's release. + +Everything else in this suite asks "did the import do the right thing". This file asks the harder +question: the import did the right thing, and then could not say so. Design D20 answers it in two +halves, and both are release-critical. + +**A database that comes back.** The terminal write retries, so a status database restarted during a +deploy still records the correct verdict. Without this, every rolling restart would leave imports +looking unfinished forever. + +**A database that does not come back.** The model is fully provisioned, and the workflow must not +claim otherwise. It **completes**, and the verdict it could not write goes out through the failure +channel — log, metric, notification — under `IMPORT_STATUS_NOT_RECORDED`. An import that worked must +never be reported as an import that failed just because the audit row was unreachable: somebody +would re-import a model that is already there. + +That last point was worth discovering the hard way. The first version of this file asserted the run +*failed*; it does not, and it should not. The spec says so in as many words — "the run is not +reported as failed" — and the implementation agrees with the spec. + +And the case that is the evidence for a migration in another repository: **a table without `V12`**. +The claim succeeds (`V11` already gave the table `run_type`, and `PENDING` and the running status are +pre-`V12` values), the model is fully provisioned, and only the *terminal* value is new — so the last +write of the import is the first thing that fails. `ImportStatusRejectedError` is non-retryable, so +it lands on the first attempt with no wait, and the row is left running forever. That is what +"apply `V12` before turning `IMPORT_MODEL_ENABLED` on" buys, as a test rather than a warning in a +document. + +Everything here runs on a **real clock** (design E9): under time skipping the backoff sleeps are +virtual, so a restarted container would never get wall-clock time to come back and the recovery test +would race by construction. The retry intervals are shortened by `import_status_retry_shortening`; +the production values stay asserted by unit test 12.1 of `import-model-workflow`, because what these +tests are about is the behaviour, not the arithmetic. +""" + +from __future__ import annotations + +import asyncio +import json +import time +from typing import Any + +import mlflow +import pytest +from temporalio.client import WorkflowFailureError + +from e2e.import_harness import failure_text, failure_types + +pytestmark = [pytest.mark.import_e2e, pytest.mark.slow, pytest.mark.asyncio] + +FACTORY_PASSWORD = 'Factory1Model!Test' +RUNNING_STATUS = 'ORCHESTRATOR_WAITING_PROC' +NOT_RECORDED_CODE = 'IMPORT_STATUS_NOT_RECORDED' +STATUS_WRITE_MARKER = 'step=status_write' +PROVISIONING_STEPS = ('experiment_creation', 'artifact_upload', 'registration', 'model_document') + + +def _as_dict(value: Any) -> dict[str, Any]: + return json.loads(value) if isinstance(value, str) else (value or {}) + + +def _client(server: dict[str, Any]) -> Any: + return mlflow.tracking.MlflowClient(tracking_uri=server['tracking_uri']) + + +def _channel_text(mock_logger: Any) -> str: + """Everything the failure channel said, as one string. + + The channel is a log line, a metric and a notification, and the log line is the only sink that + carries the whole classification — code, marker and the verdict it could not record. Reading the + logger is therefore reading the channel, and it is synchronous, which the notification path is + not. + """ + return '\n'.join(str(call) for call in mock_logger.mock_calls) + + +def _assert_fully_provisioned( + *, server: dict[str, Any], names: dict[str, str], collection: Any +) -> str: + """Everything the import was supposed to create exists. Returns the run id.""" + client = _client(server) + experiment = client.get_experiment_by_name(names['experiment_name']) + assert experiment is not None, 'no experiment: the import never got as far as provisioning' + runs = client.search_runs([experiment.experiment_id]) + assert len(runs) == 1, f'expected one run, found {len(runs)}' + assert {a.path for a in client.list_artifacts(runs[0].info.run_id)} >= { + 'prediction_model', + 'data_model', + } + versions = list(client.search_model_versions(f"name='{names['model_name']}'")) + assert len(versions) == 1, f'expected one registered version, found {len(versions)}' + assert collection.find_one({'name': names['model_name']}) is not None, 'no model document' + return runs[0].info.run_id + + +def _assert_no_provisioning_step_is_blamed(text_of_failure: str) -> None: + """The import's own steps all succeeded, so none of them may appear as the cause.""" + for step in PROVISIONING_STEPS: + assert step not in text_of_failure, f'{step!r} is blamed, but that step succeeded' + + +async def _wait_for_claim(read_row: Any, import_run_id: int, timeout: float = 30.0) -> None: + """Block until the workflow has taken ownership of the row. + + The earliest deterministic point after which every status write belongs to this import. Used to + place an outage that must cover the whole rest of the run. + """ + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + row = read_row(import_run_id) + if row is not None and row['status'] == RUNNING_STATUS: + return + await asyncio.sleep(0.1) + raise AssertionError('the row was never claimed, so the outage could not be placed') + + +async def _wait_for_provisioning( + *, server: dict[str, Any], names: dict[str, str], collection: Any, timeout: float = 90.0 +) -> None: + """Block until the model document exists — the observable end of provisioning. + + Polling the two destinations rather than patching an activity: "the document exists" *is* the + end of provisioning, so there is nothing to hook and nothing to fake. This is what lets the + outage below be placed exactly on the terminal write. + """ + client = _client(server) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + document = collection.find_one({'name': names['model_name']}) + if document is not None and list( + client.search_model_versions(f"name='{names['model_name']}'") + ): + return + await asyncio.sleep(0.2) + raise AssertionError('provisioning never finished, so the outage could not be placed') + + +class TestTheDatabaseComesBack: + """Stopped and restarted inside the retry envelope: the verdict still lands.""" + + async def test_a_restart_inside_the_envelope_still_records_the_verdict( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_mlflow_server: dict[str, Any], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute_real_clock: Any, + import_envelope: Any, + import_read_status_row: Any, + import_bff_stopped: Any, + ) -> None: + """The outage lands on the terminal write, and the server is back before the retries run out. + + The stop is placed after provisioning rather than at the start, so what is being tested is + the *terminal* write's retry envelope specifically. A stop at the start would also exercise + the claim's envelope, which is a different (and already covered) code path. + """ + object_key, digest = import_upload_bundle(import_bundle_factory()) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + handle = await import_execute_real_clock( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + + await _wait_for_provisioning( + server=import_mlflow_server, + names=import_names, + collection=import_models_collection, + ) + # Down for two seconds — several retry attempts, well inside the shortened envelope. + async with import_bff_stopped(): + await asyncio.sleep(2) + + result = await handle.result() + assert result['model_name'] == import_names['model_name'] + + run_id = _assert_fully_provisioned( + server=import_mlflow_server, + names=import_names, + collection=import_models_collection, + ) + assert result['run_id'] == run_id + + row = import_read_status_row(import_run_id) + assert row is not None + assert row['status'] == 'IMPORT_SUCCESS', ( + f'the verdict did not land after the restart: the row says {row["status"]!r}' + ) + assert row['error_message'] is None + assert _as_dict(row['orchestrator_response_data'])['import']['workflow_id'] == handle.id + + async def test_a_broken_progress_update_does_not_delay_the_verdict( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_mlflow_server: dict[str, Any], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute_real_clock: Any, + import_envelope: Any, + import_read_status_row: Any, + import_bff_stopped: Any, + ) -> None: + """A mid-flight outage that only the progress hints see. The import does not care. + + Progress writes are hints nobody reads — the terminal write rewrites status, code, reason and + step from one classification — so they get their own short policy. If they shared the terminal + write's envelope, a database the import does not need would stall a working import for + minutes at every stage update. + + The outage is placed early, so it lands on hints rather than on the terminal write, and the + assertion is that the import still finished with the correct verdict. + """ + object_key, digest = import_upload_bundle(import_bundle_factory()) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + handle = await import_execute_real_clock( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + + # Early and brief: the claim has landed, the gates are running, the terminal write is far off. + await asyncio.sleep(1.0) + async with import_bff_stopped(): + await asyncio.sleep(1.5) + + started = time.monotonic() + result = await handle.result() + elapsed = time.monotonic() - started + + assert result['model_name'] == import_names['model_name'] + row = import_read_status_row(import_run_id) + assert row is not None + assert row['status'] == 'IMPORT_SUCCESS' + assert elapsed < 60, ( + f'the import took {elapsed:.1f}s after the outage ended; a lost hint should cost ' + f'its own short policy and nothing more' + ) + + +class TestTheDatabaseStaysDown: + """Past the envelope. The model is real, the record is not, and the run says which is which.""" + + @pytest.fixture + def import_status_retry_shortening(self) -> dict[str, Any]: + """Shorter still, because these tests wait for the envelope to *expire*. + + Six half-second attempts inside an eight-second envelope: long enough that retries really + happen, short enough that giving up is something a test suite can wait for. + """ + return { + 'initial_interval': 0.5, + 'backoff_coefficient': 1.0, + 'maximum_attempts': 6, + 'hint_maximum_attempts': 2, + 'envelope_seconds': 8, + 'write_timeout_seconds': 3, + } + + async def test_a_successful_import_is_not_reported_as_failed( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_mlflow_server: dict[str, Any], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute_real_clock: Any, + import_envelope: Any, + import_read_status_row: Any, + import_bff_stopped: Any, + mock_logger: Any, + ) -> None: + """The most important assertion in this file. + + The model is registered, the document is written, the artifacts are stored — and the row + cannot be updated. The workflow **completes**: reporting failure here would tell somebody to + re-import a model that is already on the platform. The verdict goes out through the failure + channel instead, under the not-recorded code, carrying what it could not write. + + The outage is placed as soon as the row has been **claimed**, not once provisioning has + finished. Waiting for provisioning looks more precise and is in fact a race: the terminal + write follows the model document by milliseconds, while `docker stop` takes seconds, so the + write lands before the database is actually gone and the test proves nothing. Stopping right + after the claim covers everything that follows — the progress hints (which are allowed to + fail) and the terminal write (which is the subject) — with no timing to get right. + """ + object_key, digest = import_upload_bundle(import_bundle_factory()) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + handle = await import_execute_real_clock( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + + await _wait_for_claim(import_read_status_row, import_run_id) + async with import_bff_stopped(): + result = await handle.result() + + # The run completed. This is the assertion, not an incidental detail. + assert result['model_name'] == import_names['model_name'] + assert result['import_run_id'] == import_run_id + + _assert_fully_provisioned( + server=import_mlflow_server, + names=import_names, + collection=import_models_collection, + ) + + channel = _channel_text(mock_logger) + assert NOT_RECORDED_CODE in channel, ( + 'the unrecordable verdict never reached the failure channel' + ) + assert STATUS_WRITE_MARKER in channel, ( + 'the channel does not mark this as a status-write failure' + ) + assert 'verdict_not_recorded=' in channel, ( + 'the channel reports the failure but not the verdict it could not write' + ) + _assert_no_provisioning_step_is_blamed( + '\n'.join( + line for line in channel.splitlines() if 'model import failed' in line + ).replace('step=model_document', '') + ) + + row = import_read_status_row(import_run_id) + assert row is not None + assert row['status'] == RUNNING_STATUS, ( + f'the row should still be running, not {row["status"]!r}' + ) + + async def test_a_failed_import_keeps_its_original_cause( + self, + monkeypatch: pytest.MonkeyPatch, + import_status_table_pre_v12: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_activities: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute_real_clock: Any, + import_envelope: Any, + import_read_status_row: Any, + mock_logger: Any, + ) -> None: + """An import that already failed, whose failure then cannot be written down either. + + Two failures, and only one of them answers "why did this import not work". The status write + is the second, and it must not displace the first: somebody told "the status could not be + recorded" would go and look at the database, when what actually happened is that their model's + files could not be stored. + + **Why a pre-`V12` table instead of stopping the database.** The outage has to cover the + terminal write and nothing earlier, and an import that fails at a gate reaches its terminal + write within a second of starting — there is no window to aim at, and the first version of + this test proved only that a stopped database *after* a completed write changes nothing. + A pre-`V12` table has no timing at all: `IMPORT_ERROR` is a value it has never heard of, so + the terminal write of a *failed* import is refused deterministically while every write before + it succeeds. The failure being tested is the same one either way — the record cannot take the + verdict — and this version cannot race. + """ + client = import_activities.import_mlflow_repository.client + + def refuse(*_args: Any, **_kwargs: Any) -> None: + raise OSError('artifact store is unreachable') + + monkeypatch.setattr(client, 'log_artifacts', refuse) + + object_key, digest = import_upload_bundle(import_bundle_factory()) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + handle = await import_execute_real_clock( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + with pytest.raises(WorkflowFailureError) as failure: + await handle.result() + + # The reported cause is the step that actually failed. + text_of_failure = failure_text(failure.value) + assert 'artifact store is unreachable' in text_of_failure, ( + f'the reported cause lost the original failure: {text_of_failure}' + ) + assert 'ImportStatusRejectedError' not in failure_types(failure.value), ( + f'the status write displaced the original cause: {failure_types(failure.value)}' + ) + assert 'the import log database refused' not in text_of_failure, ( + 'the run is reported as a status-write failure rather than an artifact-upload failure' + ) + + # The status write still reported *itself*, through the channel, as the secondary failure. + channel = _channel_text(mock_logger) + assert NOT_RECORDED_CODE in channel, 'the unrecordable verdict was never reported' + assert STATUS_WRITE_MARKER in channel + + # And the row is left running, because the verdict never landed. + row = import_read_status_row(import_run_id) + assert row is not None + assert row['status'] == RUNNING_STATUS + + +class TestMissingV12: + """A table without `V12`. The evidence for why the migration gates enabling the worker.""" + + async def test_the_terminal_write_is_refused_on_its_first_attempt( + self, + import_status_table_pre_v12: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_mlflow_server: dict[str, Any], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute_real_clock: Any, + import_envelope: Any, + import_read_status_row: Any, + import_bff_engine: Any, + mock_logger: Any, + ) -> None: + """Provisioned, then unrecordable — and unrecordable *immediately*. + + The shape matters as much as the outcome. `ImportStatusRejectedError` is non-retryable: a + `CHECK` violation is not going to start passing because it was tried again, and waiting the + full 151-second envelope would turn a misconfiguration into a two-and-a-half-minute stall on + every single import. The elapsed time is what proves it — the shortened policy still allows + 24 attempts, so a retried failure could not finish this fast. + """ + from sqlalchemy import text as sql_text + + object_key, digest = import_upload_bundle(import_bundle_factory()) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + + started = time.monotonic() + handle = await import_execute_real_clock( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + result = await handle.result() + elapsed = time.monotonic() - started + + # The run completed: an import that provisioned a model is not a failed import, even when + # the last thing it wanted to do was impossible. + assert result['model_name'] == import_names['model_name'] + + # The claim and the names write both landed — the pre-V12 table was usable right up to the + # last statement, which is exactly what makes this failure mode easy to miss. + row = import_read_status_row(import_run_id) + assert row is not None + assert row['status'] == RUNNING_STATUS, ( + f'the row is {row["status"]!r}; a pre-V12 table cannot hold a terminal import status, ' + f'so the row is left looking as though the import never finished' + ) + assert row['experiment_name'] == import_names['experiment_name'] + assert row['run_name'] == f'import-{import_names["model_name"]}-v2' + + # The model is fully provisioned. This is the whole problem: the work is done and invisible. + _assert_fully_provisioned( + server=import_mlflow_server, + names=import_names, + collection=import_models_collection, + ) + + # The verdict reached the failure channel under the not-recorded code. + channel = _channel_text(mock_logger) + assert NOT_RECORDED_CODE in channel + assert STATUS_WRITE_MARKER in channel + assert 'verdict_not_recorded=' in channel + + # Non-retryable: refused on the first attempt, with no retry wait. + assert elapsed < 60, ( + f'the import took {elapsed:.1f}s; a non-retryable rejection must not be retried' + ) + + # And the constraint that refused it is the one V12 adds — asserted directly, so the test + # says *why* the write was impossible rather than only that it was. + with import_bff_engine.begin() as connection: + statuses = connection.execute( + sql_text( + 'SELECT pg_get_constraintdef(oid) FROM pg_constraint ' + "WHERE conname = 'chk_experiment_run_status'" + ) + ).scalar_one() + assert 'IMPORT_SUCCESS' not in statuses, ( + 'the pre-V12 fixture already allows the import statuses, so this proved nothing' + ) diff --git a/e2e/test_import_model_second_import.py b/e2e/test_import_model_second_import.py new file mode 100644 index 0000000..78ffddf --- /dev/null +++ b/e2e/test_import_model_second_import.py @@ -0,0 +1,270 @@ +"""Importing the same model twice, and what the model document is allowed to say. + +Two claims, and they pull in opposite directions, which is why they share a file. + +**A second import is additive.** It creates a new run in the *same* experiment and a new registry +version of the *same* registered model, and it promotes nothing. That is the whole of it: no +overwrite, no stage change, no reconciliation. + +**A second import touches the document not at all.** `active`, `target` and the flavors of a model +already on the platform are operator-owned — somebody turned that model on, and re-importing it must +not turn it off, retarget it or reset how it is loaded. So the document is written on the first +import and left byte-identical on every one after it. + +The document's content is asserted exactly rather than loosely, because it *is* the model listing: +`model_config.transform_flavor`, `model_config.predict_flavor` and `model_config.target` are what +the runtime reads to know how to load the model, and a missing or wrong one makes an imported model +either invisible or unloadable. +""" + +from __future__ import annotations + +from typing import Any + +import mlflow +import pytest + +from e2e.import_harness import SEEDED_MODEL_IDS + +pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio] + +FACTORY_PASSWORD = 'Factory1Model!Test' +CONTRACTED_MODEL_CONFIG_KEYS = { + 'transform_flavor', + 'predict_flavor', + 'retention_minutes', + 'target', +} + + +def _client(server: dict[str, Any]) -> Any: + return mlflow.tracking.MlflowClient(tracking_uri=server['tracking_uri']) + + +async def _import_once( + *, + payload: bytes, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, +) -> dict[str, Any]: + """One successful import. Raises if it did not succeed, so a caller never asserts on a failure.""" + object_key, digest = import_upload_bundle(payload) + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + result = await handle.result() + return {'result': result, 'import_run_id': import_run_id, 'object_key': object_key} + + +class TestSecondImport: + """The same model, imported twice in one test — the only way to observe the second as second.""" + + async def test_a_second_import_adds_a_version_and_a_run_and_nothing_else( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_mlflow_server: dict[str, Any], + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + """Both bundles declare the same model and experiment, and different `model_version`s. + + The second bundle declaring `model_version` '7' while the first declared '2' is the point of + task 8.2: the destination version number is the registry's own count, and the bundle's + number is a record of where the model came from, logged as `source.model_version` and used + for nothing else. If the two were confused, the second version here would be '7'. + """ + first = await _import_once( + payload=import_bundle_factory(metadata={'model_version': '2'}), + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + document_after_first = import_models_collection.find_one( + {'name': import_names['model_name']}, {'_id': False} + ) + assert document_after_first is not None, 'the first import wrote no document' + + second = await _import_once( + payload=import_bundle_factory(metadata={'model_version': '7'}), + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + + model_name = import_names['model_name'] + client = _client(import_mlflow_server) + + # 8.1 — a new version of the same registered model. + assert first['result']['version'] == '1' + assert second['result']['version'] == '2' + versions = list(client.search_model_versions(f"name='{model_name}'")) + assert len(versions) == 2, f'expected two versions, found {len(versions)}' + assert {v.version for v in versions} == {'1', '2'} + + # 8.2 — destination numbering is independent of what the bundles declared. + assert second['result']['version'] != '7' + runs_by_id = { + run.info.run_id: run + for run in client.search_runs( + [client.get_experiment_by_name(import_names['experiment_name']).experiment_id] + ) + } + assert runs_by_id[first['result']['run_id']].data.params['source.model_version'] == '2' + assert runs_by_id[second['result']['run_id']].data.params['source.model_version'] == '7' + + # 8.1 — a new run, in the *same* experiment, and both runs are still there. + assert first['result']['run_id'] != second['result']['run_id'] + assert len(runs_by_id) == 2 + assert {run.info.run_name for run in runs_by_id.values()} == { + f'import-{model_name}-v2', + f'import-{model_name}-v7', + } + + # 8.1 — nothing promoted, by either import. + assert all(version.current_stage in (None, 'None') for version in versions), [ + (v.version, v.current_stage) for v in versions + ] + + # 8.3 — the document is byte-identical to what the first import wrote. + document_after_second = import_models_collection.find_one( + {'name': model_name}, {'_id': False} + ) + assert document_after_second == document_after_first + + # And both rows record their own success. + for run in (first, second): + row = import_read_status_row(run['import_run_id']) + assert row is not None + assert row['status'] == 'IMPORT_SUCCESS' + + +class TestModelDocumentContent: + """What the written document says, asserted exactly rather than by presence.""" + + async def test_the_document_continues_the_collection_and_carries_the_contract( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + ) -> None: + """`id` is `max(int(id)) + 1` across gapped ids — 1, 2, 7 give 8, not 4 and not "3". + + The gaps are what make this worth asserting: a count would give 4, and a lexicographic + maximum would give "8" here but "10" for a collection holding "9" and "30". The seeded ids + are strings, as the baseline migration stores them, so `int()` is doing real work. + """ + await _import_once( + payload=import_bundle_factory(parameters={'target_variable': 'PRODUCT_QUALITY'}), + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + + document = import_models_collection.find_one( + {'name': import_names['model_name']}, {'_id': False} + ) + assert document is not None + + expected_id = str(max(int(seeded) for seeded in SEEDED_MODEL_IDS) + 1) + assert document['id'] == expected_id, ( + f'{document["id"]!r} does not continue {SEEDED_MODEL_IDS}' + ) + assert document['id'] == '8' + + assert document['name'] == import_names['model_name'] + assert document['active'] is False + + assert set(document['model_config']) == CONTRACTED_MODEL_CONFIG_KEYS, ( + f'model_config holds {sorted(document["model_config"])}, not exactly the four ' + f'contracted keys' + ) + assert document['model_config'] == { + 'transform_flavor': 'sklearn', + 'predict_flavor': 'joblib', + 'retention_minutes': 60, + 'target': 'PRODUCT_QUALITY', + } + + # The document has exactly these fields and no others — nothing was smuggled in alongside. + assert set(document) == {'id', 'name', 'active', 'model_config'} + + async def test_a_name_already_in_the_collection_is_left_untouched( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + """An operator-owned document with this model's name survives the import unchanged. + + `active: True` and a hand-set target are exactly what a re-import must not undo: somebody + turned this model on and pointed it at something. The import notices the name and stops. + """ + existing = { + 'id': '42', + 'name': import_names['model_name'], + 'active': True, + 'model_config': { + 'transform_flavor': 'sklearn', + 'predict_flavor': 'pyfunc', + 'retention_minutes': 5, + 'target': 'OPERATOR_CHOSE_THIS', + }, + } + import_models_collection.insert_one(dict(existing)) + before = list(import_models_collection.find({}, {'_id': False})) + + outcome = await _import_once( + payload=import_bundle_factory(parameters={'target_variable': 'SOMETHING_ELSE'}), + import_upload_bundle=import_upload_bundle, + import_insert_status_row=import_insert_status_row, + import_build_input=import_build_input, + import_execute=import_execute, + import_envelope=import_envelope, + ) + + # The import still succeeded — a document that is already there is not a failure. + row = import_read_status_row(outcome['import_run_id']) + assert row is not None + assert row['status'] == 'IMPORT_SUCCESS' + + after = list(import_models_collection.find({}, {'_id': False})) + assert after == before, 'the collection changed' + + matching = [d for d in after if d['name'] == import_names['model_name']] + assert len(matching) == 1, 'a duplicate was inserted' + assert matching[0] == existing diff --git a/e2e/test_import_model_status_record.py b/e2e/test_import_model_status_record.py new file mode 100644 index 0000000..5ab5a5d --- /dev/null +++ b/e2e/test_import_model_status_record.py @@ -0,0 +1,596 @@ +"""The import log row: who owns it, when, and which columns the import is allowed to touch. + +This is the file where the *real* `CHECK` constraints matter. Everything here would pass against a +table without `V12` or without the length and pattern checks, and none of it would mean anything. + +Four claims: + +- **Ordering.** The claim is the workflow's first act, before any gate runs, and the names are on + the row while the import is still in flight — because a person watching the import list has to see + *which* model is being imported while it is being imported. +- **Ownership.** The columns the inserting side owns — `username`, `request_data`, `bucket_name`, + `file_name`, `run_type`, `created_at` — are never rewritten. The import updates status, the + reason, the two names and the JSON detail, and nothing else. +- **Refusal.** A row this workflow may not own is not written to at all. Not with an error, not with + a status: the loudness goes to the Temporal error, the log and the metric, because a row that + belongs to somebody else is not the place to be loud. +- **Conformance.** Every status value an import writes is one the real constraint accepts, a + training row's vocabulary still works beside it, and the feature added no column. + +Two observations here are made from *inside* a running import, by wrapping the first call the step +under observation makes. That is a patch, and it is an observer rather than a failure injector: it +snapshots the row and delegates. Ordering cannot be asserted from a finished run — the terminal +write is precisely what erases the intermediate state — so the assertion has to stand somewhere in +the middle. +""" + +from __future__ import annotations + +import json +from typing import Any + +import pytest +from sqlalchemy import text +from sqlalchemy.exc import IntegrityError +from temporalio.client import WorkflowFailureError + +from e2e.import_harness import IMPORT_PREFIX, failure_types +from laborious.activities.model_import_errors import IMPORT_FAILURE_CATALOG +from laborious.utils.bundle.steps import ImportStep + +# `asyncio` is applied per class rather than to the whole module: `TestConstraintConformance` +# talks to Postgres directly and its tests are synchronous, and a module-wide `asyncio` mark on +# a sync test is a pytest warning, not a no-op. +pytestmark = [pytest.mark.import_e2e] + +FACTORY_PASSWORD = 'Factory1Model!Test' +RUNNING_STATUS = 'ORCHESTRATOR_WAITING_PROC' +VALIDATION_ERROR_STATUS = 'ORCHESTRATOR_VALIDATION_ERROR' + +# The columns the inserting side owns. The import may never rewrite one of these. +INSERTER_OWNED_COLUMNS = ( + 'username', + 'request_data', + 'bucket_name', + 'file_name', + 'run_type', + 'created_at', +) + +# The 13 columns V1…V12 leave. D21 says the feature adds none; this is what checks it. +EXPECTED_COLUMNS = { + 'id', + 'experiment_name', + 'run_name', + 'username', + 'status', + 'error_message', + 'created_at', + 'updated_at', + 'bucket_name', + 'file_name', + 'request_data', + 'orchestrator_response_data', + 'run_type', +} + + +def _as_dict(value: Any) -> dict[str, Any]: + return json.loads(value) if isinstance(value, str) else (value or {}) + + +@pytest.mark.asyncio +class TestOrdering: + """When each write happens, observed from inside the run rather than after it.""" + + async def test_the_row_is_claimed_before_the_first_gate_runs( + self, + import_status_table: Any, + import_models_collection: Any, + import_bundle_factory: Any, + import_activities: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + """Snapshot the row from inside `stat_object` — the first call gate 1 makes. + + Standing here rather than after the run is not a convenience: the terminal write rewrites + status, reason and detail, so a finished row cannot say what it looked like before the gates. + `stat_object` is patched with a wrapper that records and then delegates — an observer, not a + failure. Everything the import does still really happens. + """ + client = import_activities.import_minio_repository.minio_client + real_stat = client.stat_object + snapshots: list[dict[str, Any] | None] = [] + + def observing_stat(bucket: str, key: str, *args: Any, **kwargs: Any) -> Any: + snapshots.append(import_read_status_row(observed_run_id[0])) + return real_stat(bucket, key, *args, **kwargs) + + observed_run_id: list[int] = [0] + object_key, digest = import_upload_bundle(import_bundle_factory()) + observed_run_id[0] = import_insert_status_row(object_key=object_key, expected_digest=digest) + client.stat_object = observing_stat + try: + handle = await import_execute( + import_build_input( + import_run_id=observed_run_id[0], + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + await handle.result() + finally: + client.stat_object = real_stat + + assert snapshots, 'gate 1 never ran, so nothing was observed' + claimed = snapshots[0] + assert claimed is not None + assert claimed['status'] == RUNNING_STATUS, ( + f'at the first gate the row was {claimed["status"]!r}, not the running status' + ) + detail = _as_dict(claimed['orchestrator_response_data']) + assert detail['import']['workflow_id'] == handle.id + assert detail['import']['step'] in ( + ImportStep.RECEIVED.value, + ImportStep.DOWNLOAD.value, + ), detail + + async def test_the_names_are_on_the_row_while_the_import_is_still_running( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_activities: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + """Observed at the first call experiment creation makes, so nothing exists in MLflow yet. + + This is the stronger form of the same claim task 7.1 makes from a failure: here the import + *succeeds*, and the names were still already written — so their presence is not an artefact + of the run having stopped. + """ + repository = import_activities.import_mlflow_repository + real_get_experiment = repository.get_experiment + snapshots: list[dict[str, Any] | None] = [] + observed_run_id: list[int] = [0] + + def observing_get_experiment(*args: Any, **kwargs: Any) -> Any: + snapshots.append(import_read_status_row(observed_run_id[0])) + return real_get_experiment(*args, **kwargs) + + object_key, digest = import_upload_bundle(import_bundle_factory()) + observed_run_id[0] = import_insert_status_row(object_key=object_key, expected_digest=digest) + repository.get_experiment = observing_get_experiment + try: + handle = await import_execute( + import_build_input( + import_run_id=observed_run_id[0], + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + await handle.result() + finally: + repository.get_experiment = real_get_experiment + + assert snapshots, 'experiment creation never ran' + mid_flight = snapshots[0] + assert mid_flight is not None + assert mid_flight['status'] == RUNNING_STATUS + assert mid_flight['experiment_name'] == import_names['experiment_name'] + assert mid_flight['run_name'] == f'import-{import_names["model_name"]}-v2' + + +@pytest.mark.asyncio +class TestOwnership: + """The inserting side's columns survive an import untouched, whatever the outcome.""" + + @pytest.mark.parametrize('outcome', ['success', 'failure']) + async def test_the_inserted_columns_are_never_rewritten( + self, + outcome: str, + import_status_table: Any, + import_models_collection: Any, + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + payload = ( + import_bundle_factory() + if outcome == 'success' + else import_bundle_factory(include_metadata=False) + ) + object_key, digest = import_upload_bundle(payload) + import_run_id = import_insert_status_row( + object_key=object_key, + expected_digest=digest, + username='owner.of.the.row@example.com', + ) + + # Snapshotted before the run, not after: `import_execute` waits for the workflow, so a + # snapshot taken from its return value would already be the post-import row and every + # comparison below would be a row against itself. + before = import_read_status_row(import_run_id) + assert before is not None + assert before['status'] == 'PENDING' + + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + + if outcome == 'success': + await handle.result() + else: + with pytest.raises(WorkflowFailureError): + await handle.result() + + after = import_read_status_row(import_run_id) + assert after is not None + for column in INSERTER_OWNED_COLUMNS: + assert after[column] == before[column], ( + f'{column} was rewritten: {before[column]!r} -> {after[column]!r}' + ) + + # And the import *did* write what it owns, so this is not passing by doing nothing at all. + assert after['status'] != before['status'] + assert after['updated_at'] >= before['updated_at'] + + +@pytest.mark.asyncio +class TestClaimGuards: + """A row this workflow may not own is not written to. At all.""" + + async def test_an_unknown_id_fails_the_run_and_writes_nothing( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_bff_engine: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + object_key, digest = import_upload_bundle(import_bundle_factory()) + # A real, claimable row exists — so "nothing was written" is a claim about *this* run + # choosing the right row, not about the table being empty. + bystander_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + before = _snapshot_table(import_bff_engine) + + handle = await import_execute( + import_build_input( + import_run_id=bystander_id + 10_000, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + with pytest.raises(WorkflowFailureError) as failure: + await handle.result() + + assert 'ImportLogRowNotClaimableError' in failure_types(failure.value) + assert _snapshot_table(import_bff_engine) == before, 'the table changed' + import_assert_nothing_provisioned( + experiment_name=import_names['experiment_name'], + model_name=import_names['model_name'], + object_key=object_key, + ) + + async def test_a_training_row_cannot_be_claimed_by_an_import( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_bff_engine: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + """`run_type='TRAIN'` is a different feature's row, and the claim will not take it.""" + object_key, digest = import_upload_bundle(import_bundle_factory()) + training_id = import_insert_status_row( + object_key=object_key, + expected_digest=digest, + run_type='TRAIN', + experiment_name='a-training-experiment', + ) + before = _snapshot_table(import_bff_engine) + + handle = await import_execute( + import_build_input( + import_run_id=training_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + with pytest.raises(WorkflowFailureError) as failure: + await handle.result() + + assert 'ImportLogRowNotClaimableError' in failure_types(failure.value) + assert _snapshot_table(import_bff_engine) == before + import_assert_nothing_provisioned( + experiment_name=import_names['experiment_name'], + model_name=import_names['model_name'], + object_key=object_key, + ) + + async def test_a_row_held_by_another_workflow_cannot_be_taken( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_bff_engine: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + """Already running under somebody else's workflow id. + + The claim is idempotent for *its own* workflow id and refuses every other, which is what + makes a duplicated workflow start safe and a stolen row impossible. + """ + object_key, digest = import_upload_bundle(import_bundle_factory()) + held_id = import_insert_status_row( + object_key=object_key, + expected_digest=digest, + status=RUNNING_STATUS, + orchestrator_response_data={ + 'import': {'workflow_id': 'import-held-by-somebody-else', 'step': 'received'} + }, + ) + before = _snapshot_table(import_bff_engine) + + handle = await import_execute( + import_build_input( + import_run_id=held_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + with pytest.raises(WorkflowFailureError) as failure: + await handle.result() + + assert 'ImportLogRowNotClaimableError' in failure_types(failure.value) + assert _snapshot_table(import_bff_engine) == before + import_assert_nothing_provisioned( + experiment_name=import_names['experiment_name'], + model_name=import_names['model_name'], + object_key=object_key, + ) + + +@pytest.mark.asyncio +class TestInputAgainstTheRow: + """A request that contradicts the row it claimed — recorded *on* that row, because it owns it.""" + + async def test_a_file_name_disagreeing_with_the_object_key_is_recorded_on_the_row( + self, + import_status_table: Any, + import_models_collection: Any, + import_names: dict[str, str], + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + import_assert_nothing_provisioned: Any, + ) -> None: + """The difference from the claim guards above is where the error goes. + + A row that could not be claimed is somebody else's, so nothing is written to it. This row + *was* claimed — the contradiction is only found afterwards — so the failure belongs on it, + where the person who submitted the import will see it. + """ + object_key, digest = import_upload_bundle(import_bundle_factory()) + import_run_id = import_insert_status_row( + object_key=f'{IMPORT_PREFIX}a-different-object.sientia', + expected_digest=digest, + ) + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + with pytest.raises(WorkflowFailureError): + await handle.result() + + row = import_read_status_row(import_run_id) + assert row is not None + code, reason = IMPORT_FAILURE_CATALOG[ImportStep.RECEIVED] + assert row['status'] == VALIDATION_ERROR_STATUS + assert row['error_message'] == reason + detail = _as_dict(row['orchestrator_response_data'])['import'] + assert detail['step'] == ImportStep.RECEIVED.value + assert detail['code'] == code.value + # The record still names the object *it* was given, not the one the request named. + assert row['file_name'] == f'{IMPORT_PREFIX}a-different-object.sientia' + import_assert_nothing_provisioned( + experiment_name=import_names['experiment_name'], + model_name=import_names['model_name'], + object_key=object_key, + ) + + +@pytest.mark.asyncio +class TestTrainingRowsAreUntouched: + """A training row sitting in the same table is never modified by an import.""" + + async def test_a_training_row_beside_an_import_is_unchanged( + self, + import_status_table: Any, + import_models_collection: Any, + import_bundle_factory: Any, + import_upload_bundle: Any, + import_insert_status_row: Any, + import_build_input: Any, + import_execute: Any, + import_envelope: Any, + import_read_status_row: Any, + ) -> None: + object_key, digest = import_upload_bundle(import_bundle_factory()) + training_id = import_insert_status_row( + object_key=object_key, + expected_digest=digest, + run_type='TRAIN', + status='TRAINING_SUCCESS', + experiment_name='someone-elses-training', + username='trainer@example.com', + ) + training_before = import_read_status_row(training_id) + + import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest) + handle = await import_execute( + import_build_input( + import_run_id=import_run_id, + object_key=object_key, + expected_digest=digest, + password_envelope=import_envelope(FACTORY_PASSWORD), + ) + ) + await handle.result() + + assert import_read_status_row(import_run_id)['status'] == 'IMPORT_SUCCESS' + assert import_read_status_row(training_id) == training_before + + +class TestConstraintConformance: + """The real constraints, exercised directly — and the column list D21 promised not to change.""" + + def test_every_status_an_import_writes_is_accepted_on_an_import_row( + self, import_status_table: Any, import_insert_status_row: Any, import_bff_engine: Any + ) -> None: + """`PENDING`, the running value, and both import terminals, on `run_type='IMPORT'`.""" + row_id = import_insert_status_row( + object_key=f'{IMPORT_PREFIX}conformance.sientia', expected_digest='a' * 64 + ) + for status in ('PENDING', RUNNING_STATUS, VALIDATION_ERROR_STATUS, 'IMPORT_SUCCESS'): + _set_status(import_bff_engine, row_id, status) + _set_status(import_bff_engine, row_id, 'IMPORT_ERROR') + + def test_the_terminal_values_do_not_cross_run_types( + self, import_status_table: Any, import_insert_status_row: Any, import_bff_engine: Any + ) -> None: + """`chk_experiment_run_terminal_status_run_type`, in both directions. + + This is the constraint `V12` adds, and it is the reason the migration gates enabling the + worker: without it an import could write `TRAINING_SUCCESS` and a training run could write + `IMPORT_ERROR`, and the import list would be unreadable. + """ + import_row = import_insert_status_row( + object_key=f'{IMPORT_PREFIX}crossing.sientia', expected_digest='b' * 64 + ) + training_row = import_insert_status_row( + object_key=f'{IMPORT_PREFIX}crossing.sientia', + expected_digest='b' * 64, + run_type='TRAIN', + ) + + for row_id, forbidden in ( + (import_row, 'TRAINING_SUCCESS'), + (import_row, 'TRAINING_ERROR'), + (training_row, 'IMPORT_SUCCESS'), + (training_row, 'IMPORT_ERROR'), + ): + with pytest.raises(IntegrityError) as refusal: + _set_status(import_bff_engine, row_id, forbidden) + assert 'chk_experiment_run_terminal_status_run_type' in str(refusal.value) + + # The shared stage values are deliberately outside the pairing: both run types pass through. + for row_id in (import_row, training_row): + for shared in ('PENDING', RUNNING_STATUS, VALIDATION_ERROR_STATUS): + _set_status(import_bff_engine, row_id, shared) + + def test_a_training_row_still_walks_its_own_vocabulary( + self, import_status_table: Any, import_insert_status_row: Any, import_bff_engine: Any + ) -> None: + training_row = import_insert_status_row( + object_key=f'{IMPORT_PREFIX}training.sientia', + expected_digest='c' * 64, + run_type='TRAIN', + ) + for status in ('PENDING', RUNNING_STATUS, 'TRAINING_SUCCESS'): + _set_status(import_bff_engine, training_row, status) + _set_status(import_bff_engine, training_row, 'TRAINING_ERROR') + + def test_the_feature_added_no_column( + self, import_status_table: Any, import_bff_engine: Any + ) -> None: + """Design D21: the error code and the workflow id live in the existing JSONB column. + + A quiet `ALTER TABLE` would be the easiest way for this feature to break the BFF that owns + the table, and the hardest to notice, so it is asserted rather than trusted. + """ + with import_bff_engine.begin() as connection: + columns = { + row[0] + for row in connection.execute( + text( + 'SELECT column_name FROM information_schema.columns ' + "WHERE table_schema = 'public' AND table_name = 'experiment_run'" + ) + ) + } + assert columns == EXPECTED_COLUMNS, ( + f'unexpected: {sorted(columns - EXPECTED_COLUMNS)}, ' + f'missing: {sorted(EXPECTED_COLUMNS - columns)}' + ) + + +def _set_status(engine: Any, row_id: int, status: str) -> None: + """Write one status onto one row, so the real constraint gets to answer.""" + with engine.begin() as connection: + connection.execute( + text('UPDATE public.experiment_run SET status = :status WHERE id = :id'), + {'status': status, 'id': row_id}, + ) + + +def _snapshot_table(engine: Any) -> list[dict[str, Any]]: + """Every row, ordered, as plain values — for "the table is byte-identical" assertions.""" + with engine.begin() as connection: + return [ + dict(row) + for row in connection.execute( + text('SELECT * FROM public.experiment_run ORDER BY id') + ).mappings() + ] diff --git a/e2e/test_predictions_batch_joblib_fallback.py b/e2e/test_predictions_batch_joblib_flavor.py similarity index 56% rename from e2e/test_predictions_batch_joblib_fallback.py rename to e2e/test_predictions_batch_joblib_flavor.py index a807c3e..87b6f9d 100644 --- a/e2e/test_predictions_batch_joblib_fallback.py +++ b/e2e/test_predictions_batch_joblib_flavor.py @@ -1,9 +1,10 @@ """ -End-to-end tests for the joblib fallback on the MLflow model load paths. +End-to-end tests for the explicit `joblib` flavor on the MLflow model load paths. -`joblib.load` is NOT mocked here: the artifact fixture holds a real sklearn -estimator serialized with `joblib.dump(..., compress=3)`, so a successful run -proves the fallback really deserializes a compressed pickle and serves it. +`joblib.load` is NOT mocked here: the artifact fixture holds a real object serialized with +`joblib.dump(..., compress=3)` whose class lives only inside the artifact's `code/` directory. +A passing run therefore proves the flavor decompresses the pickle **and** resolves the module, +without any `mlflow.*.load_model` being called for that model. """ import pandas as pd @@ -12,7 +13,7 @@ from sqlalchemy import text from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker -from e2e.conftest import JOBLIB_FALLBACK_FLAVOR_ERROR, JOBLIB_FALLBACK_MODEL_NAME +from e2e.conftest import JOBLIB_FLAVOR_MODEL_NAME from e2e.helpers import ( assert_prediction, assert_stop, @@ -25,8 +26,15 @@ from laborious.workflows.predictions_batch import PredictionsBatch base_query = 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = {model_id}' +FLAVOR_LOADERS = ('sklearn', 'pyfunc', 'pytorch') -def build_input_data(model_id: int, model_name: str) -> dict: + +def build_input_data( + model_id: int, + model_name: str, + transform_flavor: str = 'sklearn', + predict_flavor: str = 'sklearn', +) -> dict: """Workflow input for a plain transform + predict run of a given model.""" return { 'schedule_name': 'test-schedule', @@ -55,15 +63,15 @@ def build_input_data(model_id: int, model_name: str) -> dict: 'prediction_store_policy': 'lts:1', 'model_config': { 'retention_minutes': 0, - 'transform_flavor': 'sklearn', - 'predict_flavor': 'sklearn', + 'transform_flavor': transform_flavor, + 'predict_flavor': predict_flavor, }, 'datetime_columns': ['timestamp', 'created_at'], } -def expected_fallback_prediction(estimator) -> float: - """Prediction the fixture estimator produces for the mocked transform output.""" +def expected_joblib_prediction(model) -> float: + """Prediction the artifact's model produces for the transform output of this suite.""" features = pd.DataFrame( { 'feature_1': [0.234], @@ -71,7 +79,7 @@ def expected_fallback_prediction(estimator) -> float: 'timestamp': ['2024-01-01 12:00:00'], } ) - return float(estimator.predict(features)[0]) + return float(model.predict(features)[0]) def assert_single_prediction_close(postgres_engine, model_id, expected, tolerance=1e-9): @@ -98,42 +106,86 @@ def logged_messages(mock_logger, level: str) -> str: return ' '.join(str(logged.args[0]) for logged in calls) +def loader_uris(patch_mlflow) -> list[str]: + """Every model URI handed to an MLflow flavor loader during the run.""" + return [ + str(loaded.args[0]) + for flavor in FLAVOR_LOADERS + for loaded in getattr(patch_mlflow, flavor).load_model.call_args_list + ] + + @pytest.mark.asyncio @pytest.mark.integration -async def test_prediction_workflow_recovered_by_joblib_fallback( +async def test_prediction_workflow_with_joblib_flavor( temporal_test_env: WorkflowEnvironment, temporal_worker: Worker, test_activities: Activities, postgres_engine, - joblib_fallback_artifact, + joblib_flavor_artifacts, mlflow_client, - mock_logger, + patch_mlflow, ): - """Flavor loader fails, joblib fallback loads the real compressed pickle, prediction lands.""" + """`predict_flavor: 'joblib'` deserializes the real artifact, and the prediction lands.""" client = temporal_test_env.client model_id = 271 insert_sample_data(postgres_engine, model_id, [60.0, 78.2]) - input_data = build_input_data(model_id, JOBLIB_FALLBACK_MODEL_NAME) + input_data = build_input_data(model_id, JOBLIB_FLAVOR_MODEL_NAME, predict_flavor='joblib') await start_and_await_workflow( - client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-fallback') + client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-flavor') ) assert_single_prediction_close( postgres_engine, model_id, - expected_fallback_prediction(joblib_fallback_artifact['estimator']), + expected_joblib_prediction(joblib_flavor_artifacts['prediction_model']['model']), ) assert mlflow_client.download_artifacts.call_count == 1 assert mlflow_client.download_artifacts.call_args.args[1] == 'prediction_model' - assert JOBLIB_FALLBACK_FLAVOR_ERROR in logged_messages(mock_logger, 'warning') + # The joblib flavor never builds a `models:/` URI, so no flavor loader saw this model. + assert all(JOBLIB_FLAVOR_MODEL_NAME not in uri for uri in loader_uris(patch_mlflow)) @pytest.mark.asyncio @pytest.mark.integration -async def test_prediction_workflow_without_fallback_is_unchanged( +async def test_prediction_workflow_with_joblib_transform_flavor( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + test_activities: Activities, + postgres_engine, + joblib_flavor_artifacts, + mlflow_client, + patch_mlflow, + mock_mlflow_models, +): + """`transform_flavor: 'joblib'` downloads and deserializes `data_model` on its own.""" + client = temporal_test_env.client + model_id = 274 + insert_sample_data(postgres_engine, model_id, [60.0, 78.2]) + input_data = build_input_data(model_id, JOBLIB_FLAVOR_MODEL_NAME, transform_flavor='joblib') + + await start_and_await_workflow( + client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-transform') + ) + + assert_prediction(postgres_engine, model_id, prediction=0.5) + mock_mlflow_models['predict_model'].predict.assert_called() + + assert mlflow_client.download_artifacts.call_count == 1 + assert mlflow_client.download_artifacts.call_args.args[1] == 'data_model' + + # The joblib flavor resolves no artifact URI, so `get_run` is never reached and no flavor + # loader was given a `data_model` URI. + patch_mlflow.get_run.assert_not_called() + assert all(not uri.endswith('/data_model') for uri in loader_uris(patch_mlflow)) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_prediction_workflow_with_sklearn_flavor_is_unchanged( temporal_test_env: WorkflowEnvironment, temporal_worker: Worker, test_activities: Activities, @@ -141,14 +193,14 @@ async def test_prediction_workflow_without_fallback_is_unchanged( mlflow_client, mock_mlflow_models, ): - """A model whose flavor loader works keeps the current result and downloads nothing.""" + """A model on an MLflow flavor keeps the current result and downloads nothing.""" client = temporal_test_env.client model_id = 272 insert_sample_data(postgres_engine, model_id, [60.0, 78.2]) input_data = build_input_data(model_id, 'test_model') await start_and_await_workflow( - client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-no-fallback') + client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-sklearn-flavor') ) assert_prediction(postgres_engine, model_id, prediction=0.5) @@ -158,29 +210,31 @@ async def test_prediction_workflow_without_fallback_is_unchanged( @pytest.mark.asyncio @pytest.mark.integration -async def test_prediction_workflow_reports_both_causes_when_fallback_fails( +async def test_prediction_workflow_joblib_flavor_failure_reports_single_cause( temporal_test_env: WorkflowEnvironment, temporal_worker: Worker, test_activities: Activities, postgres_engine, - joblib_fallback_artifact_without_pickle, + joblib_flavor_artifact_without_pickle, mlflow_client, mock_logger, ): - """Both attempts fail: workflow stops and the logged traceback carries both causes.""" - mlflow_client.download_artifacts.return_value = joblib_fallback_artifact_without_pickle + """The joblib flavor failing stops the workflow with one cause, not a chain of two.""" + mlflow_client.download_artifacts.side_effect = lambda _run_id, _artifact_name: ( + joblib_flavor_artifact_without_pickle + ) client = temporal_test_env.client model_id = 273 insert_sample_data(postgres_engine, model_id, [60.0, 78.2]) - input_data = build_input_data(model_id, JOBLIB_FALLBACK_MODEL_NAME) + input_data = build_input_data(model_id, JOBLIB_FLAVOR_MODEL_NAME, predict_flavor='joblib') await start_and_await_workflow( - client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-fallback-failed') + client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-flavor-failed') ) assert_stop(postgres_engine, model_id) logged = logged_messages(mock_logger, 'debug') - assert JOBLIB_FALLBACK_FLAVOR_ERROR in logged assert 'No .pkl file found in prediction_model artifact' in logged + assert 'Original error' not in logged diff --git a/e2e/test_retrain_joblib_flavor.py b/e2e/test_retrain_joblib_flavor.py new file mode 100644 index 0000000..dabda57 --- /dev/null +++ b/e2e/test_retrain_joblib_flavor.py @@ -0,0 +1,126 @@ +""" +End-to-end test for retraining a model served through the explicit `joblib` flavor. + +Retraining is not part of the `predictions_batch` workflow, so it gets its own module. The test +is a write→read round trip in one run: `retrain_model` loads the production artifact through the +joblib flavor, `log_model` writes the retrained model back as `model.pkl` + `code/`, and the +written artifact is then loaded again through the same flavor. Neither `joblib.dump` nor +`joblib.load` is mocked; only MLflow's own I/O is. +""" + +import importlib +import shutil +import sys +from pathlib import Path + +import pandas as pd +import pytest + +from e2e.conftest import JOBLIB_FLAVOR_CODE_MODULE, JOBLIB_FLAVOR_MODEL_NAME +from laborious.activities.activities import Activities + +RETRAIN_INDEX = [ + '2024-01-01 12:00:00+0000', + '2024-01-01 12:01:00+0000', + '2024-01-01 12:02:00+0000', +] + +METADATA = { + 'model_id': 'test_model', + 'model_name': JOBLIB_FLAVOR_MODEL_NAME, + 'workflow_name': 'test_workflow', + 'schema_name': 'test_schedule', +} + + +def build_retrain_data() -> pd.DataFrame: + """Training frame indexed by timestamp strings, carrying the target column.""" + return pd.DataFrame( + { + 'sensor_1': [60.0, 61.0, 62.0], + 'sensor_2': [78.2, 78.4, 78.6], + 'target': [1.0, 2.0, 3.0], + }, + index=RETRAIN_INDEX, + ) + + +class FakeArtifactStore: + """Minimal stand-in for the run's artifact tree, so the written layout is inspectable.""" + + def __init__(self, root: Path): + self.root = root + + def log_artifact(self, local_path, artifact_path=None): + if artifact_path is None: + # Retraining also logs the CSVs at the run root; irrelevant to this test. + return + target = self.root / artifact_path + target.mkdir(parents=True, exist_ok=True) + shutil.copy(local_path, target / Path(local_path).name) + + def log_artifacts(self, local_dir, artifact_path=None): + target = self.root / artifact_path if artifact_path else self.root + shutil.copytree(local_dir, target, dirs_exist_ok=True) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_retrain_joblib_model_writes_pickle_and_code_and_reloads( + test_activities: Activities, + joblib_flavor_artifacts, + mlflow_client, + patch_mlflow, + tmp_path, +): + """A joblib retrain writes `model.pkl` + `code/`, and the joblib flavor reads it back.""" + repository = test_activities.model_monitoring_repository + + store = FakeArtifactStore(tmp_path / 'run-retrained') + patch_mlflow.log_artifact.side_effect = store.log_artifact + patch_mlflow.log_artifacts.side_effect = store.log_artifacts + patch_mlflow.get_run.return_value.data.params = {} + patch_mlflow.start_run.return_value.__enter__.return_value.info.run_id = 'run-retrained' + + output = await repository.retrain_model( + data=build_retrain_data(), + model_name=JOBLIB_FLAVOR_MODEL_NAME, + model_config={ + 'target': 'target', + 'transform_flavor': 'joblib', + 'predict_flavor': 'joblib', + }, + metadata=METADATA, + ) + + assert output['success'] is True, output.get('traceback') + + # Both models were loaded through the joblib flavor, not through a flavor loader. + downloaded = [loaded.args[1] for loaded in mlflow_client.download_artifacts.call_args_list] + assert sorted(downloaded) == ['data_model', 'prediction_model'] + assert patch_mlflow.sklearn.load_model.call_count == 0 + assert patch_mlflow.pyfunc.load_model.call_count == 0 + + written = store.root / 'prediction_model' + assert (written / 'model.pkl').is_file() + assert (written / 'code' / f'{JOBLIB_FLAVOR_CODE_MODULE}.py').is_file() + assert not (written / 'MLmodel').exists() + assert (store.root / 'data_model' / 'model.pkl').is_file() + + # Drop the module the first load put in place: the reload must resolve it from the `code/` + # the retrain wrote, not from what is already imported. + for code_dir in (artifact['code_dir'] for artifact in joblib_flavor_artifacts.values()): + if code_dir in sys.path: + sys.path.remove(code_dir) + sys.modules.pop(JOBLIB_FLAVOR_CODE_MODULE, None) + importlib.invalidate_caches() + + mlflow_client.download_artifacts.side_effect = lambda _run_id, _artifact_name: str(written) + + reloaded = await repository.load_predict_model(JOBLIB_FLAVOR_MODEL_NAME, METADATA, 'joblib') + + features = pd.DataFrame({'feature_1': [0.234], 'feature_2': [0.783]}) + expected = joblib_flavor_artifacts['prediction_model']['model'].predict(features) + + assert reloaded.predict(features) == pytest.approx(expected) + assert str(written / 'code') in sys.path diff --git a/laborious/activities/activities.py b/laborious/activities/activities.py index e62e0d1..a06eb16 100644 --- a/laborious/activities/activities.py +++ b/laborious/activities/activities.py @@ -82,8 +82,8 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, ModelImport): notification handler keeps its own client either way. import_config: `build_import_config()`; read from the environment when absent import_status_config: `build_import_status_config()`; read from the environment when - absent. This is the BFF database holding the import log, a second Postgres - connection that is never the one Storage holds. + absent. This is the BFF database holding the import log: another database on the + `POSTGRES_*` server, and a second connection that is never the one Storage holds. Raises: Exception: If any parent class initialization fails diff --git a/laborious/activities/model_import.py b/laborious/activities/model_import.py index 0af7a88..886ee73 100644 --- a/laborious/activities/model_import.py +++ b/laborious/activities/model_import.py @@ -288,8 +288,10 @@ class ModelImport(SientiaMonitoring): """ Args: status_config: `build_import_status_config()` — the BFF database holding the import log. - This is a **second** Postgres connection, independent of the one Laborious already - holds, which is what makes the transactional independence of design D7 literal. + This is a **second** Postgres connection — another database, on the `POSTGRES_*` + server unless the IMPORT_STATUS_DB_* overrides move it — and its separateness from + the one Laborious already holds is what makes design D7's transactional + independence literal. import_config: `build_import_config()` — bucket, prefix, ceilings, work directory, envelope key and the models collection name. minio_repository: the shared MinIO repository, for the uploaded bundle. diff --git a/laborious/utils/connectors_config.py b/laborious/utils/connectors_config.py index aa2a019..4b31496 100644 --- a/laborious/utils/connectors_config.py +++ b/laborious/utils/connectors_config.py @@ -136,27 +136,36 @@ def build_import_config() -> dict[str, Any]: def build_import_status_config() -> dict[str, Any]: """ - Build the connection to the import log database — a **second** Postgres connection. + Build the connection to the import log database — the POSTGRES_* server, another database. The import log is `public.experiment_run` in the database owned by the Spring Boot BFF - (`sientia-core-mlops-bff`), not the `sientia` database Laborious itself uses. The table and - schema are constants of the code, never inputs; only the connection is configuration. + (`sientia-core-mlops-bff`), which sits on the same Postgres server as the `sientia` database + Laborious itself uses. Only the database name differs, so only the database name has a variable + of its own; host, port, user and password come from POSTGRES_*. The table and schema are + constants of the code, never inputs. + + The four IMPORT_STATUS_DB_* connection variables remain as overrides, for a deployment where the + log genuinely lives on another server — the e2e suite is exactly that, two containers. Environment Variables: - IMPORT_STATUS_DB_HOST: host (default: localhost) - IMPORT_STATUS_DB_PORT: port (default: 5432) IMPORT_STATUS_DB_NAME: database (default: sientia-core-mlops-bff) - IMPORT_STATUS_DB_USER: user (default: mlops_bff_user) - IMPORT_STATUS_DB_PASSWORD: password (default: empty) + POSTGRES_HOST: server host, unless IMPORT_STATUS_DB_HOST overrides it (default: localhost) + POSTGRES_PORT: server port, unless IMPORT_STATUS_DB_PORT overrides it (default: 5432) + POSTGRES_USER: user, unless IMPORT_STATUS_DB_USER overrides it (default: sientia) + POSTGRES_PASSWORD: password, unless IMPORT_STATUS_DB_PASSWORD overrides it (default: empty) Returns: dict: import status database configuration dictionary """ + # Written as `override or fallback or default` rather than + # `getenv(override) or getenv(fallback, default)`: inside an `or`, the two-argument `getenv` + # widens to `str | None` for mypy, so the `int(...)` below fails to type-check. Putting the + # default last as its own term keeps every value a plain `str` and reads in precedence order. return { - 'host': getenv('IMPORT_STATUS_DB_HOST', 'localhost'), - 'port': int(getenv('IMPORT_STATUS_DB_PORT', '5432')), - 'dbname': getenv('IMPORT_STATUS_DB_NAME', 'sientia-core-mlops-bff'), - 'user': getenv('IMPORT_STATUS_DB_USER', 'mlops_bff_user'), - 'password': getenv('IMPORT_STATUS_DB_PASSWORD', ''), + 'host': getenv('IMPORT_STATUS_DB_HOST') or getenv('POSTGRES_HOST') or 'localhost', + 'port': int(getenv('IMPORT_STATUS_DB_PORT') or getenv('POSTGRES_PORT') or '5432'), + 'dbname': getenv('IMPORT_STATUS_DB_NAME') or 'sientia-core-mlops-bff', + 'user': getenv('IMPORT_STATUS_DB_USER') or getenv('POSTGRES_USER') or 'sientia', + 'password': getenv('IMPORT_STATUS_DB_PASSWORD') or getenv('POSTGRES_PASSWORD') or '', 'schema': 'public', } diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index e89b685..500622f 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -16,11 +16,11 @@ Capabilities: import ctypes import gc -import inspect +import sys +import tempfile import threading import time import traceback -from collections.abc import Callable from datetime import datetime, timedelta from io import StringIO from os import environ, listdir, makedirs, path @@ -45,7 +45,7 @@ ARTIFACTS_PATH = './tmp/artifacts' TRANSFORMED_COMPRESSED_PATH = 'artifacts/training_transformer.pkl' PREDICTION_COMPRESSED_PATH = 'artifacts/stacking_model.pkl' -INVALID_FLAVOR_MESSAGE = "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'." +INVALID_FLAVOR_MESSAGE = "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch' or 'joblib'." def force_memory_release(logger: Logger): @@ -354,23 +354,19 @@ class MLFlowRepository(SientiaMonitoring): self.info(f'Loaded dataframe from {model_name}:{artifact_path}', metadata) return dataframe - def _find_model_pickle( - self, local_dir: str, artifact_name: str, flavor_error: Exception - ) -> str: + def _find_model_pickle(self, local_dir: str, artifact_name: str) -> str: """ Locate the pickle file of a downloaded model artifact. - Fallback path only: this is reached after the MLflow flavor loader has - already failed. The target artifact is always a single `.pkl` at the top - level of the artifact directory, so the listing does not recurse — - nested wrapper artifacts must never be deserialized in place of the - model. No other extension is considered and `MLmodel` is not parsed. + Primary path of the `joblib` flavor. The target artifact is always a + single `.pkl` at the top level of the artifact directory, so the listing + does not recurse — nested wrapper artifacts must never be deserialized + in place of the model. No other extension is considered and `MLmodel` is + not parsed. Args: local_dir (str): Local directory holding the downloaded artifact. artifact_name (str): Artifact name, e.g. 'prediction_model'. - flavor_error (Exception): Error raised by the MLflow flavor loader, - reported in the message so the original cause is not lost. Returns: str: Path to the first top-level `.pkl` file, in name order. @@ -388,7 +384,6 @@ class MLFlowRepository(SientiaMonitoring): if not pkl_files: raise ValueError( f'No .pkl file found in {artifact_name} artifact. ' - f'Original error: {flavor_error}. ' f'Top-level entries in {local_dir}: {entries}' ) @@ -398,24 +393,28 @@ class MLFlowRepository(SientiaMonitoring): self, local_dir: str, artifact_name: str, - flavor_error: Exception, metadata: dict[str, Any] | None = None, ) -> Any: """ Deserialize a downloaded model artifact with `joblib`. - Fallback path only. `joblib.load` reads the compressed `.pkl` produced - by `joblib.dump`, which is what the MLflow flavor loader fails to read. - The deserialized object is returned as is: no `._model_impl.python_model` - unwrap is applied, because the pickled object already is the model. + Primary path of the `joblib` flavor. `joblib.load` reads the compressed + `.pkl` produced by `joblib.dump`, which is what the MLflow flavor + loaders do not read. The deserialized object is returned as is: no + `._model_impl.python_model` unwrap is applied, because the pickled + object already is the model. + + The artifact's top-level `code` directory is prepended to `sys.path` + before deserializing, when it exists, so that classes whose modules ship + inside the artifact resolve at unpickle time. The entry is not removed + afterwards: the deserialized object may import lazily. Args: local_dir (str): Local directory holding the downloaded artifact. artifact_name (str): Artifact name, e.g. 'prediction_model'. - flavor_error (Exception): Error raised by the MLflow flavor loader. metadata (dict[str, Any] | None): Metadata used for structured logging, so the chosen pickle path correlates with the other - fallback log lines. + joblib log lines. Returns: Any: The object deserialized from the artifact's pickle file. @@ -424,9 +423,14 @@ class MLFlowRepository(SientiaMonitoring): ValueError: If the artifact directory has no top-level `.pkl` file. Exception: Any error raised by `joblib.load` itself. """ - pkl_path = self._find_model_pickle(local_dir, artifact_name, flavor_error) + pkl_path = self._find_model_pickle(local_dir, artifact_name) - self.debug(f'joblib fallback: loading {artifact_name} from {pkl_path}', metadata) + code_dir = self.resolve_artifact_code_dir(local_dir) + if code_dir and code_dir not in sys.path: + sys.path.insert(0, code_dir) + self.debug(f'joblib flavor: added {code_dir} to sys.path', metadata) + + self.debug(f'joblib flavor: loading {artifact_name} from {pkl_path}', metadata) return joblib.load(pkl_path) @@ -434,37 +438,36 @@ class MLFlowRepository(SientiaMonitoring): self, model_name: str, artifact_name: str, - flavor_error: Exception, metadata: dict[str, Any], run_id: str | None = None, - ) -> Any: + ) -> tuple[Any, str]: """ Download a model artifact and deserialize it with `joblib`. - Fallback path only. The download runs without `dst_path`, so MLflow - writes to a per-call temporary directory: the shared `ARTIFACTS_PATH` - tree is never written to and `dowload_artifacts` — which removes its - destination before downloading — is deliberately not reused. The target - artifact is always a `.pkl` and the deserialized object is returned as - is. + Primary path of the `joblib` flavor. The download runs without + `dst_path`, so MLflow writes to a per-call temporary directory: the + shared `ARTIFACTS_PATH` tree is never written to and + `dowload_artifacts` — which removes its destination before + downloading — is deliberately not reused. The target artifact is always + a `.pkl` and the deserialized object is returned as is. - Callers that already resolved a run to build the flavor URI must pass it - as `run_id`: resolving the Production run again here would open a window - where a promotion between the two lookups makes the fallback serve a - different version than the one the flavor attempt tried and logged. Only - callers whose flavor attempt used a symbolic URI — `models://production`, - resolved by MLflow itself — leave it as `None`. + Callers that already resolved a run must pass it as `run_id`, so that + the run is resolved exactly once per load. Callers with no run at hand — + `load_predict_model`, whose flavor URI is symbolic — leave it as `None`. + + The local directory is returned alongside the model because the retrain + path needs it to re-log the artifact's `code` directory. Args: model_name (str): Registered model name. artifact_name (str): Artifact to download, e.g. 'prediction_model'. - flavor_error (Exception): Error raised by the MLflow flavor loader. metadata (dict[str, Any]): Metadata used for structured logging. run_id (str | None): Run already resolved by the caller. When `None`, the Production run is resolved here. Returns: - Any: The object deserialized from the artifact's pickle file. + tuple[Any, str]: The deserialized object and the local directory the + artifact was downloaded to. Raises: ValueError: If the downloaded artifact has no top-level `.pkl` file. @@ -475,86 +478,68 @@ class MLFlowRepository(SientiaMonitoring): run_id = self.get_model_run_id(model_name=model_name, stage='Production') self.info( - f'joblib fallback: downloading {artifact_name} of {model_name} from run {run_id}', + f'joblib flavor: downloading {artifact_name} of {model_name} from run {run_id}', metadata, ) local_dir = self.client.download_artifacts(run_id, artifact_name) - return self._load_model_pickle(local_dir, artifact_name, flavor_error, metadata) + model = self._load_model_pickle(local_dir, artifact_name, metadata) - async def _load_with_joblib_fallback( + return model, local_dir + + async def _load_joblib_artifact( self, - primary_loader: Callable[[], Any], - fallback_loader: Callable[[Exception], Any], - failure_description: str, + model_name: str, + artifact_name: str, + operation_type: str, metadata: dict[str, Any], - fallback_labels: dict[str, Any], - core_labels: dict[str, Any] | None = None, - ) -> Any: + run_id: str | None = None, + ) -> tuple[Any, str]: """ - Run a model load with the joblib fallback and its observability. + Load a model artifact through the `joblib` flavor, with observability. - Single owner of the fallback flow shared by every load path: warn on the - flavor failure without emitting an error metric, try the fallback, and - emit lag/count under the fallback labels — or the error count chained to - the flavor error when the fallback also fails. Fixes to the flow belong - here, not in the call sites. - - Both loaders are called with no observability concern of their own: - `fallback_loader` receives the flavor error and may be sync or async, - which is what lets the download-based paths and the already-downloaded - wrapper path share this method. + Single owner of the joblib read path: no MLflow flavor loader is called + here, neither before nor after the pickle load. Metrics are emitted with + the **base** `operation_type` the caller passes — a joblib load is a + load, so no suffixed operation type exists. Args: - primary_loader (Callable[[], Any]): Flavor load attempt. Any - exception it raises triggers the fallback. - fallback_loader (Callable[[Exception], Any]): joblib fallback, - receiving the flavor error. Its result is awaited when - awaitable. - failure_description (str): What failed, e.g. 'Flavor loader sklearn - failed for prediction model x'. The fallback and original error - are appended to it. + model_name (str): Registered model name. + artifact_name (str): Artifact to download, e.g. 'prediction_model'. + operation_type (str): Base metric operation type of the calling + method, e.g. 'load_predict_model'. metadata (dict[str, Any]): Metadata used for structured logging. - fallback_labels (dict[str, Any]): Metric labels for the fallback - path, already carrying its `operation_type`. - core_labels (dict[str, Any] | None): Metric labels for a successful - flavor load. `None` emits no metric on that path. + run_id (str | None): Run already resolved by the caller. When `None`, + the Production run is resolved during the download. Returns: - Any: The model loaded by the flavor loader or by the fallback. + tuple[Any, str]: The deserialized model and the local directory the + artifact was downloaded to. Raises: - Exception: The fallback error, chained from the flavor error, when - both attempts fail. + Exception: Any error raised while resolving the run, downloading the + artifact or deserializing the pickle. It propagates as is: there + is no second attempt. """ + core_labels = self.get_core_labels(metadata, operation_type=operation_type) start_time = time.time() + try: - model = primary_loader() - except Exception as flavor_error: - self.warning( - f'{failure_description}, trying joblib fallback. Original error: {flavor_error}', - metadata, + model, local_dir = await self._download_and_load_pickle( + model_name=model_name, + artifact_name=artifact_name, + metadata=metadata, + run_id=run_id, ) - try: - model = fallback_loader(flavor_error) - if inspect.isawaitable(model): - model = await model - except Exception as fallback_error: - await self.emit_metric( - metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=fallback_labels - ) - raise fallback_error from flavor_error + except Exception as e: + await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels) + raise e - await self.observe_lag(start_time, metrics.MODEL_READ_LAG, fallback_labels) - await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=fallback_labels) - return model + await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels) + await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels) - if core_labels is not None: - await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels) - await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels) - - return model + return model, local_dir async def load_predict_model( self, model_name: str, metadata: dict[str, Any], flavor: str = 'sklearn' @@ -565,7 +550,7 @@ class MLFlowRepository(SientiaMonitoring): Args: model_name (str): The name of the model to download from the registry. metadata (dict[str, Any]): Metadata used for structured logging. - flavor (str): Model flavor ('pyfunc', 'sklearn', 'pytorch') + flavor (str): Model flavor ('pyfunc', 'sklearn', 'pytorch', 'joblib') artifact_path (str | None): Path to compressed artifacts if model is compressed Returns: @@ -574,37 +559,42 @@ class MLFlowRepository(SientiaMonitoring): Notes: - The model is fetched from the "Production" stage of the MLflow Model Registry. - Warnings during the model loading process are suppressed. + - With `flavor='joblib'` no MLflow flavor loader is called: the + `prediction_model` artifact is downloaded and deserialized with + `joblib.load`, which needs no `MLmodel` manifest. The artifact's + `code` directory, when present, is put on `sys.path` first. """ - if flavor not in ('pyfunc', 'sklearn', 'pytorch'): + if flavor not in ('pyfunc', 'sklearn', 'pytorch', 'joblib'): raise ValueError(INVALID_FLAVOR_MESSAGE) + if flavor == 'joblib': + model, _local_dir = await self._load_joblib_artifact( + model_name=model_name, + artifact_name='prediction_model', + operation_type='load_predict_model', + metadata=metadata, + ) + return model + model_uri = f'models:/{model_name}/production' self.info(f'Loading prediction model {model_name} from {model_uri}') core_labels = self.get_core_labels(metadata, operation_type='load_predict_model') - fallback_labels = self.get_core_labels( - metadata, operation_type='load_predict_model_joblib_fallback' - ) - - def load_flavor() -> Any: + start_time = time.time() + try: if flavor == 'pyfunc': - return mlflow.pyfunc.load_model(model_uri) - if flavor == 'sklearn': - return mlflow.sklearn.load_model(model_uri) - return mlflow.pytorch.load_model(model_uri) + model = mlflow.pyfunc.load_model(model_uri) + elif flavor == 'sklearn': + model = mlflow.sklearn.load_model(model_uri) + else: + model = mlflow.pytorch.load_model(model_uri) + except Exception as e: + await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels) + raise e - return await self._load_with_joblib_fallback( - primary_loader=load_flavor, - fallback_loader=lambda flavor_error: self._download_and_load_pickle( - model_name, 'prediction_model', flavor_error, metadata - ), - failure_description=( - f'Flavor loader {flavor} failed for prediction model {model_name}' - ), - metadata=metadata, - fallback_labels=fallback_labels, - core_labels=core_labels, - ) + await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels) + await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels) + return model async def load_transform_model( self, model_name: str, metadata: dict[str, Any], flavor: str = 'sklearn' @@ -618,7 +608,7 @@ class MLFlowRepository(SientiaMonitoring): Args: model_name (str): The name of the model to download. metadata (dict[str, Any]): Metadata used for structured logging. - flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') + flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch', 'joblib') artifact_path (str | None): Path to compressed artifacts if model is compressed Returns: @@ -629,45 +619,48 @@ class MLFlowRepository(SientiaMonitoring): model cannot be loaded. Notes: - - The Production run is resolved once and reused by the joblib - fallback, so a promotion between both steps cannot make the - fallback serve a different version than the flavor attempt. + - With `flavor='joblib'` no MLflow flavor loader is called and no + model URI is built: the `data_model` artifact of the resolved + Production run is downloaded and deserialized with `joblib.load`, + which needs no `MLmodel` manifest. The artifact's `code` + directory, when present, is put on `sys.path` first. """ - if flavor not in ('sklearn', 'pyfunc', 'pytorch'): + if flavor not in ('sklearn', 'pyfunc', 'pytorch', 'joblib'): raise ValueError(INVALID_FLAVOR_MESSAGE) latest_production_id = self.get_model_run_id(model_name=model_name, stage='Production') + + if flavor == 'joblib': + model, _local_dir = await self._load_joblib_artifact( + model_name=model_name, + artifact_name='data_model', + operation_type='load_transform_model', + metadata=metadata, + run_id=latest_production_id, + ) + return model + model_uri = self.get_model_uri(latest_production_id, prediction=False) self.info(f'Loading data model {model_name} from {model_uri}') core_labels = self.get_core_labels(metadata, operation_type='load_transform_model') - fallback_labels = self.get_core_labels( - metadata, operation_type='load_transform_model_joblib_fallback' - ) - - def load_flavor() -> Any: + start_time = time.time() + try: if flavor == 'sklearn': - return mlflow.sklearn.load_model(model_uri) - if flavor == 'pyfunc': - return mlflow.pyfunc.load_model(model_uri) - return mlflow.pytorch.load_model(model_uri) + model = mlflow.sklearn.load_model(model_uri) + elif flavor == 'pyfunc': + model = mlflow.pyfunc.load_model(model_uri) + else: + model = mlflow.pytorch.load_model(model_uri) + except Exception as e: + await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels) + raise e - return await self._load_with_joblib_fallback( - primary_loader=load_flavor, - fallback_loader=lambda flavor_error: self._download_and_load_pickle( - model_name, - 'data_model', - flavor_error, - metadata, - run_id=latest_production_id, - ), - failure_description=f'Flavor loader {flavor} failed for data model {model_name}', - metadata=metadata, - fallback_labels=fallback_labels, - core_labels=core_labels, - ) + await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels) + await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels) + return model async def download_model( self, @@ -684,11 +677,17 @@ class MLFlowRepository(SientiaMonitoring): model_name (str): Name of the model to download metadata (dict[str, Any]): Metadata used for structured logging. model_type (str): Type of model ('predict' or 'transform') - flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') + flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch', 'joblib') load_wrapper (bool): Whether to load wrapper Returns: tuple[Any, str | None]: Model object and optional artifact path. + + Notes: + - `flavor='joblib'` returns the local directory the artifact was + downloaded to as the artifact path, because the retrain path needs + it to re-log the artifact's `code` directory. Every other flavor + returns `None` unless the wrapper branch was taken. """ self.info( @@ -711,27 +710,24 @@ class MLFlowRepository(SientiaMonitoring): f'Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}' ) - fallback_labels = self.get_core_labels( - metadata, operation_type='download_model_wrapper_joblib_fallback' - ) - - def load_wrapper_flavor() -> Any: - raw_model = mlflow.pyfunc.load_model(artifact_path) - return raw_model._model_impl.python_model - - model = await self._load_with_joblib_fallback( - primary_loader=load_wrapper_flavor, - fallback_loader=lambda flavor_error: self._load_model_pickle( - artifact_path, target, flavor_error, metadata - ), - failure_description=f'Wrapper loader failed for {model_type} model {model_name}', - metadata=metadata, - fallback_labels=fallback_labels, - ) + raw_model = mlflow.pyfunc.load_model(artifact_path) + model = raw_model._model_impl.python_model self.debug( f'Model wrapper loaded: {model.__class__.__name__}:{model.__dict__}', metadata ) + elif flavor == 'joblib': + target = 'prediction_model' if model_type == 'predict' else 'data_model' + operation_type = ( + 'load_predict_model' if model_type == 'predict' else 'load_transform_model' + ) + + return await self._load_joblib_artifact( + model_name=model_name, + artifact_name=target, + operation_type=operation_type, + metadata=metadata, + ) else: if model_type == 'predict': model = await self.load_predict_model(model_name, metadata, flavor) @@ -1165,6 +1161,30 @@ class MLFlowRepository(SientiaMonitoring): } return retrain_data + def resolve_artifact_code_dir(self, artifact_path: str | None) -> str | None: + """Resolve the top-level `code` directory of a downloaded artifact. + + Sibling of `resolve_code_paths`: both look at the same + `/code` root, but this one returns the directory itself, + which is what `sys.path` and `mlflow.log_artifacts` take, while + `resolve_code_paths` returns the entries inside it, which is what + `mlflow.pyfunc`'s `code_path=` takes. + + Args: + artifact_path (str | None): Local directory of the downloaded model + artifact. + + Returns: + str | None: `/code` when it is a directory, `None` + otherwise — including when `artifact_path` itself is `None`. + """ + if not artifact_path: + return None + + code_dir = path.join(artifact_path, 'code') + + return code_dir if path.isdir(code_dir) else None + def resolve_code_paths(self, artifact_path: str | None, metadata: dict) -> list[str]: """Resolve the code dependencies to forward when re-logging a pyfunc model. @@ -1187,11 +1207,12 @@ class MLFlowRepository(SientiaMonitoring): self.warning('No artifact path available, logging model without code paths', metadata) return [] - code_dir = path.join(artifact_path, 'code') + code_dir = self.resolve_artifact_code_dir(artifact_path) - if not path.isdir(code_dir): + if not code_dir: self.warning( - f'No code directory found at {code_dir}, logging model without code paths', + f'No code directory found at {path.join(artifact_path, "code")}, ' + 'logging model without code paths', metadata, ) return [] @@ -1207,9 +1228,16 @@ class MLFlowRepository(SientiaMonitoring): Args: model_data (dict): Model holder with keys 'model' and optional 'artifact_path'. - flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch'). + flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch', 'joblib'). model_type (str): Artifact name, e.g., 'prediction_model' or 'data_model'. metadata (dict): Metadata for structured logging. + + Notes: + - `flavor='joblib'` writes a `joblib`-compressed `model.pkl` under + `model_type` and re-logs the source artifact's `code` directory, + so the written model stays deserializable. No `mlflow.*.log_model` + runs and no `MLmodel` manifest is written, which means only this + runtime's `joblib` flavor can read that artifact back. """ model = model_data['model'] @@ -1231,6 +1259,19 @@ class MLFlowRepository(SientiaMonitoring): self.debug('Model uploaded successfully', metadata) elif flavor == 'pytorch': mlflow.pytorch.log_model(model, model_type) + elif flavor == 'joblib': + with tempfile.TemporaryDirectory() as tmp_dir: + pkl_path = path.join(tmp_dir, 'model.pkl') + joblib.dump(model, pkl_path, compress=3) + mlflow.log_artifact(pkl_path, artifact_path=model_type) + + code_dir = self.resolve_artifact_code_dir(model_data.get('artifact_path')) + + if code_dir: + self.debug(f'Re-logging code directory {code_dir}', metadata) + mlflow.log_artifacts(code_dir, artifact_path=f'{model_type}/code') + else: + self.debug(f'No code directory to re-log for {model_type}', metadata) else: raise ValueError(INVALID_FLAVOR_MESSAGE) diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index f10a6cb..c3b3014 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -33,8 +33,8 @@ Environment Variables: - HTTP_SDK_METRICS_PORT: Temporal SDK metrics port (default: 9091) - PROJECT_NAME: Project name for notifications (default: laborious) - IMPORT_MODEL_ENABLED: register the fifth (import) worker; unset by default -- IMPORT_PASSWORD_KEY / IMPORT_BUNDLE_* / IMPORT_STATUS_DB_*: import configuration, see - ``laborious.utils.connectors_config`` +- IMPORT_PASSWORD_KEY / IMPORT_BUNDLE_* / IMPORT_STATUS_DB_NAME: import configuration, see + ``laborious.utils.connectors_config`` (the import log's server is POSTGRES_*) """ from temporalio import client, workflow diff --git a/pyproject.toml b/pyproject.toml index 06f349f..2f2ef0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,6 +133,17 @@ addopts = [ # GC'd. Harmless mock artifacts turned into a hard ERROR by the plugin itself. "-p", "no:unraisableexception", ] +filterwarnings = [ + # `kafka-python`, pulled in transitively by `sientia_do`, reads its protocol schemas through the + # legacy `importlib.resources` API that Python 3.11 deprecated. It emits ~109 warnings per run, + # twice each (once at the call site, once inside `importlib`), which is enough noise to bury a + # warning worth reading. Nothing here can fix it — the change belongs in `kafka-python`. + # + # Matched on the message rather than by module so the filter cannot quietly swallow an unrelated + # deprecation: any *other* DeprecationWarning, including one from our own code, still shows up. + "ignore:read_text is deprecated:DeprecationWarning", + "ignore:open_text is deprecated:DeprecationWarning", +] markers = [ "asyncio: marks tests as async", "integration: marks tests as integration tests", diff --git a/requirements-dev.txt b/requirements-dev.txt index a2b8893..57224a1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,7 +13,10 @@ types-requests>=2.31.0 # Type stubs for requests pytest>=7.4.0 # Testing framework pytest-cov>=4.1.0 # Coverage plugin for pytest pytest-asyncio>=0.21.0 # Async test support (already in main requirements) -testcontainers[postgres,minio,mongodb] # PostgreSQL, MinIO and MongoDB containers for E2E tests +testcontainers[postgres,minio,mongodb]>=4.15.0 # PostgreSQL, MinIO and MongoDB containers for E2E tests + # Floor is the `testcontainers.community.*` namespace: the flat + # `testcontainers.minio` / `.postgres` / `.mongodb` paths the e2e suite used + # to import are deprecated shims that warn on import. # Development Tools ipython>=8.12.0 # Enhanced Python shell diff --git a/scripts/encode.sh b/scripts/encode.sh deleted file mode 100755 index 73d0534..0000000 --- a/scripts/encode.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Resolve paths from the script's own location so this works whether it is run -# from the repository root or from inside scripts/. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" - -source "${REPO_ROOT}/venv/bin/activate" - -pip install pathspec -pip install pyyaml - -echo " -.git" >> "${REPO_ROOT}/.gitignore" - -python "${SCRIPT_DIR}/encrypt.py" "${REPO_ROOT}" "${REPO_ROOT}/code" --ignore "${REPO_ROOT}/.gitignore" --chunk-size 100000 - -sed -i '/.git/d' "${REPO_ROOT}/.gitignore" - -xdg-open "${REPO_ROOT}" diff --git a/scripts/encrypt.py b/scripts/encrypt.py deleted file mode 100644 index 14eb3da..0000000 --- a/scripts/encrypt.py +++ /dev/null @@ -1,113 +0,0 @@ -import argparse -import os -from typing import Any - -import yaml # type: ignore -from pathspec import PathSpec - -r""" -Usage: - python .\encrypt.py path_to_dir output_file --ignore ignore_file --chunk-size 100000 -""" - - -def load_ignore_patterns(ignore_file, include_library): - # Ensure the .gitignore file exists - if not os.path.exists(ignore_file): - raise FileNotFoundError(f'Ignore file not found at {ignore_file}') - - # Load and parse the .gitignore patterns - with open(ignore_file) as file: - patterns = file.readlines() - if not include_library: - patterns.append('**/deploy/library/') - - spec = PathSpec.from_lines('gitwildmatch', patterns) - return spec - - -def is_ignored(file_path, spec): - """Check if a file should be ignored based on the ignore patterns.""" - return spec.match_file(file_path) if spec else False - - -def encode_file_tree_to_yaml(directory, ignore_file, include_library): - """Encode the file tree into a single YAML file.""" - ignore_patterns = load_ignore_patterns(ignore_file, include_library) if ignore_file else None - file_tree: dict[str, Any] = {} - - for root, dirs, files in os.walk(directory): - # Skip ignored directories - dirs[:] = [d for d in dirs if not is_ignored(os.path.join(root, d), ignore_patterns)] - - for file in files: - file_path = os.path.join(root, file) - - # Skip ignored files - if is_ignored(file_path, ignore_patterns): - continue - - # Read file content - try: - with open(file_path, encoding='utf-8') as f: - content = f.read() - except Exception as e: - print(f'Error reading file {file_path}: {e}') - raise - - # Create nested dictionary structure - path_parts = os.path.relpath(file_path, directory).split(os.sep) - current_level = file_tree - - # all except the last part (the file name) - for part in path_parts[:-1]: - current_level = current_level.setdefault(part, {}) - - # Add the file and its content - current_level[path_parts[-1]] = content - return yaml.dump(file_tree, default_flow_style=False) - - -def chunk_and_write_file_tree_to_yaml(yaml_content, output_file, chunk_size=None): - """Chunk the YAML content and write it to the output file.""" - - chunks = ( - [yaml_content] - if chunk_size is None - else [yaml_content[i : i + chunk_size] for i in range(0, len(yaml_content), chunk_size)] - ) - - for i, chunk in enumerate(chunks): - chunk_file = f'{output_file}_{i}.yaml' - # Write the file tree to the output YAML file - with open(chunk_file, 'w', encoding='utf-8') as yaml_file: - yaml_file.write(chunk) - - -def main(): - parser = argparse.ArgumentParser(description='Encrypts file tree to yaml file') - parser.add_argument('input_directory', help='Directory to encode') - parser.add_argument('output_yaml_file', help='Output YAML file') - parser.add_argument('--ignore', default=None, help='Path to the ignore file') - parser.add_argument( - '--chunk-size', type=int, default=None, help='Chunk size for the output YAML file' - ) - parser.add_argument( - '--library', type=bool, default=False, help='Incude the library in the output YAML file' - ) - - # Parse arguments - args = parser.parse_args() - - # Example usage - directory_to_encode = args.input_directory - ignore_file_path = args.ignore - output_yaml_file = args.output_yaml_file - include_library = args.library - - content = encode_file_tree_to_yaml(directory_to_encode, ignore_file_path, include_library) - chunk_and_write_file_tree_to_yaml(content, output_yaml_file, args.chunk_size) - - -if __name__ == '__main__': - main() diff --git a/scripts/import_model_manual_run.py b/scripts/import_model_manual_run.py index 2c393f7..b260313 100644 --- a/scripts/import_model_manual_run.py +++ b/scripts/import_model_manual_run.py @@ -77,13 +77,18 @@ def _line(label: str, value: object) -> str: # back to localhost, which is right for a service that must boot; it is wrong for a script that # writes, because a silent fallback is a write to somewhere nobody read out loud. # -# The import log database: a second Postgres connection, independent of the POSTGRES_* one above it -# in .env. The schema and table are constants of the worker's code, never configuration. -STATUS_DB_HOST = env_required('IMPORT_STATUS_DB_HOST') -STATUS_DB_PORT = int(env_required('IMPORT_STATUS_DB_PORT')) +# The import log database: the POSTGRES_* server from .env, with only the database name of its own, +# because the log lives in the Spring Boot BFF's database rather than in `sientia`. The schema and +# table are constants of the worker's code, never configuration. +# +# `build_import_status_config` also honours IMPORT_STATUS_DB_{HOST,PORT,USER,PASSWORD} as overrides, +# for a deployment whose log sits on another server. This script does not: it is a development tool +# pointed at a real environment, and the override exists for the e2e suite's two containers. +STATUS_DB_HOST = env_required('POSTGRES_HOST') +STATUS_DB_PORT = int(env_required('POSTGRES_PORT')) STATUS_DB_NAME = env_required('IMPORT_STATUS_DB_NAME') -STATUS_DB_USER = env_required('IMPORT_STATUS_DB_USER') -STATUS_DB_PASSWORD = os.getenv('IMPORT_STATUS_DB_PASSWORD', '') +STATUS_DB_USER = env_required('POSTGRES_USER') +STATUS_DB_PASSWORD = os.getenv('POSTGRES_PASSWORD', '') STATUS_TABLE = 'public.experiment_run' # Where the frontend uploads the bundle. The prefix keeps the worker's own default, which is a @@ -145,7 +150,7 @@ CONFIRM_TARGET = env('IMPORT_SCRIPT_CONFIRM_TARGET') if CONFIRM_TARGET != STATUS_DB_HOST: raise RuntimeError( 'refusing to write: the confirmation does not match the resolved status database host. ' - f'confirmation={CONFIRM_TARGET!r}, resolved IMPORT_STATUS_DB_HOST={STATUS_DB_HOST!r}. ' + f'confirmation={CONFIRM_TARGET!r}, resolved POSTGRES_HOST={STATUS_DB_HOST!r}. ' 'Read the targets cell 1 printed, then restate the host here.' ) diff --git a/scripts/inter_arrival.py b/scripts/inter_arrival.py deleted file mode 100644 index f3e2cef..0000000 --- a/scripts/inter_arrival.py +++ /dev/null @@ -1,27 +0,0 @@ -# %% - -# Load logs.txt -with open('logs.txt') as file: - lines = file.readlines() - -# %% -import re - -# Grep "inter-arrival_s=number" with regex -intervals = [] -for line in lines: - match = re.search(r'inter-arrival_s=([0-9.]+)', line) - if match: - intervals.append(float(match.group(1))) -# %% - -print(intervals) -# %% -import matplotlib.pyplot as plt - -plt.plot(intervals) -plt.ylabel('Inter-arrival time (s)') -plt.xlabel('Sample') -plt.title('Inter-arrival time distribution') -plt.show() -# %% diff --git a/scripts/model_convert.ipynb b/scripts/model_convert.ipynb deleted file mode 100644 index d04f63e..0000000 --- a/scripts/model_convert.ipynb +++ /dev/null @@ -1,107 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 23, - "id": "e838ff21", - "metadata": {}, - "outputs": [], - "source": [ - "import csv\n", - "\n", - "\n", - "def csv_to_tag_lists(csv_path: str) -> dict:\n", - " read_tags = []\n", - " write_tags = []\n", - "\n", - " def to_float(val):\n", - " try:\n", - " return float(str(val).strip())\n", - " except Exception:\n", - " return None\n", - "\n", - " with open(csv_path, newline='', encoding='utf-8') as f:\n", - " reader = csv.DictReader(f)\n", - " for row in reader:\n", - " # Basic normalization\n", - " op = (row.get('operation') or '').strip()\n", - "\n", - " if op == 'READ':\n", - " # Build common tag payload with required mappings\n", - " tag = {\n", - " 'server_id': '1',\n", - " 'tag_address': row.get('opc_tag'),\n", - " 'tag_name': row.get('name'),\n", - " 'data_range': [to_float(row.get('min_value')), to_float(row.get('max_value'))],\n", - " 'aggr_func': row.get('aggregation_func').lower(),\n", - " # keep other fields with their original names\n", - " 'frequency': row.get('frequency'),\n", - " 'local': row.get('local'),\n", - " 'area': row.get('area'),\n", - " 'description': row.get('description'),\n", - " }\n", - "\n", - " read_tags.append(tag)\n", - "\n", - " else:\n", - " tag = {\n", - " 'server_id': '1',\n", - " 'addr': row.get('opc_tag'),\n", - " 'tag_name': row.get('name'),\n", - " 'local': row.get('local'),\n", - " 'area': row.get('area'),\n", - " 'description': row.get('description'),\n", - " }\n", - "\n", - " if op == 'WRITE_PREDICTION':\n", - " tag['type'] = 'prediction'\n", - " write_tags.append(tag)\n", - " elif op == 'WRITE_CONFIDENCE':\n", - " tag['type'] = 'confidence'\n", - " write_tags.append(tag)\n", - " # ignore any other operation values silently\n", - "\n", - " return {'read_tags': read_tags, 'write_tags': write_tags}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4621cd43", - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "file_names = ['Courier - Página1.csv']\n", - "\n", - "for file_name in file_names:\n", - " write_file = file_name.replace('.csv', '.json')\n", - "\n", - " with open(write_file, 'w', encoding='utf-8') as f:\n", - " json.dump(csv_to_tag_lists(file_name), f, indent=2, ensure_ascii=False)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 4a89d69..2c449de 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -1,4 +1,8 @@ +import ast +import importlib +import sys from datetime import UTC, datetime +from pathlib import Path from unittest.mock import ANY, AsyncMock, MagicMock, call, patch import joblib as joblib_lib @@ -10,6 +14,7 @@ from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ from sklearn.linear_model import LinearRegression from laborious import metrics +from laborious.utils.repository import model_repository from laborious.utils.repository.model_repository import ( INVALID_FLAVOR_MESSAGE, MLFlowRepository, @@ -465,7 +470,7 @@ async def test_load_predict_model_pytorch(mlflow, mlflow_repository): async def test_load_predict_model_error(mlflow_repository): with pytest.raises(ValueError) as e: await mlflow_repository.load_predict_model('test_model', {}, 'invalid') - assert str(e) == "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'." + assert str(e) == INVALID_FLAVOR_MESSAGE mlflow_repository.emit_metric.assert_called_once_with( metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=ANY @@ -542,7 +547,7 @@ async def test_load_transform_model_error(mlflow_repository): with pytest.raises(ValueError) as e: await mlflow_repository.load_transform_model('test_model', {}, 'invalid') - assert str(e) == "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'." + assert str(e) == INVALID_FLAVOR_MESSAGE mlflow_repository.emit_metric.assert_called_once_with( metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=ANY @@ -1124,7 +1129,7 @@ async def test_log_model_error(mlflow_repository): await mlflow_repository.log_model( model_data, 'invalid', 'prediction_model', metadata['metadata'] ) - assert str(e) == "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch'." + assert str(e) == INVALID_FLAVOR_MESSAGE mlflow_repository.emit_metric.assert_called_once_with( metric_object=metrics.MODEL_WRITE_ERROR_COUNT, tags=ANY ) @@ -1768,10 +1773,19 @@ async def test_predict_success_ndarray(mlflow_repository): """ -Tests for the joblib fallback on the model load paths +Tests for the joblib flavor on the model load and write paths """ JOBLIB_PATH = 'laborious.utils.repository.model_repository.joblib' +FLAVOR_LOADERS = ('sklearn', 'pyfunc', 'pytorch') + + +@pytest.fixture +def preserved_sys_path(): + """Restore `sys.path`: the joblib flavor inserts entries and never removes them.""" + original = list(sys.path) + yield + sys.path[:] = original def build_artifact_dir(tmp_path, *filenames): @@ -1781,6 +1795,21 @@ def build_artifact_dir(tmp_path, *filenames): return str(tmp_path) +def build_code_dir(tmp_path, module_name='probe_flavor_model'): + """Add a `code/` directory carrying the model's class module to an artifact dir.""" + code_dir = tmp_path / 'code' + code_dir.mkdir() + (code_dir / f'{module_name}.py').write_text( + 'class ProbeModel:\n' + ' def __init__(self, offset):\n' + ' self.offset = offset\n' + '\n' + ' def predict(self, data):\n' + ' return [value + self.offset for value in data]\n' + ) + return str(code_dir) + + def emitted_tags(emit_metric_mock, metric_object): """Collect the `tags` kwarg of every emit_metric call for a given metric.""" return [ @@ -1790,86 +1819,22 @@ def emitted_tags(emit_metric_mock, metric_object): ] -@patch(JOBLIB_PATH) -@pytest.mark.asyncio -async def test_load_predict_model_flavor_success_skips_fallback( - joblib_mock, mlflow, mlflow_repository -): - result = await mlflow_repository.load_predict_model('test_model', {}, 'sklearn') - - assert result == mlflow.sklearn.load_model.return_value - joblib_mock.load.assert_not_called() - mlflow_repository.client.download_artifacts.assert_not_called() +def assert_no_flavor_loader_called(mlflow): + for flavor in FLAVOR_LOADERS: + getattr(mlflow, flavor).load_model.assert_not_called() @patch(JOBLIB_PATH) @pytest.mark.asyncio -async def test_load_transform_model_flavor_success_skips_fallback( - joblib_mock, mlflow, mlflow_repository -): - mlflow_repository.get_model_run_id = MagicMock() - mlflow_repository.get_model_uri = MagicMock() - - result = await mlflow_repository.load_transform_model('test_model', {}, 'sklearn') - - assert result == mlflow.sklearn.load_model.return_value - joblib_mock.load.assert_not_called() - mlflow_repository.client.download_artifacts.assert_not_called() - - -@patch(JOBLIB_PATH) -@pytest.mark.asyncio -async def test_download_model_wrapper_success_skips_fallback( - joblib_mock, mlflow, mlflow_repository -): - mlflow_repository.dowload_artifacts = AsyncMock() - - result = await mlflow_repository.download_model('test_model', {}, 'predict', 'pyfunc', True) - - assert result == ( - mlflow.pyfunc.load_model.return_value._model_impl.python_model, - mlflow_repository.dowload_artifacts.return_value, - ) - joblib_mock.load.assert_not_called() - mlflow_repository.client.download_artifacts.assert_not_called() - - -@patch(JOBLIB_PATH) -@pytest.mark.asyncio -async def test_load_predict_model_invalid_flavor_skips_fallback(joblib_mock, mlflow_repository): - with pytest.raises(ValueError) as exc_info: - await mlflow_repository.load_predict_model('test_model', {}, 'invalid') - - assert str(exc_info.value) == INVALID_FLAVOR_MESSAGE - joblib_mock.load.assert_not_called() - mlflow_repository.client.download_artifacts.assert_not_called() - mlflow_repository.emit_metric.assert_not_called() - - -@patch(JOBLIB_PATH) -@pytest.mark.asyncio -async def test_load_transform_model_invalid_flavor_skips_fallback(joblib_mock, mlflow_repository): - with pytest.raises(ValueError) as exc_info: - await mlflow_repository.load_transform_model('test_model', {}, 'invalid') - - assert str(exc_info.value) == INVALID_FLAVOR_MESSAGE - joblib_mock.load.assert_not_called() - mlflow_repository.client.download_artifacts.assert_not_called() - mlflow_repository.emit_metric.assert_not_called() - - -@patch(JOBLIB_PATH) -@pytest.mark.asyncio -async def test_load_predict_model_fallback_recovers_load( +async def test_load_predict_model_joblib_flavor_loads_pickle_directly( joblib_mock, mlflow, mlflow_repository, tmp_path ): - mlflow.sklearn.load_model.side_effect = Exception('flavor boom') mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( tmp_path, 'MLmodel', 'model.pkl' ) - result = await mlflow_repository.load_predict_model('test_model', {}, 'sklearn') + result = await mlflow_repository.load_predict_model('test_model', {}, 'joblib') assert result is joblib_mock.load.return_value mlflow_repository.get_model_run_id.assert_called_once_with( @@ -1879,46 +1844,43 @@ async def test_load_predict_model_fallback_recovers_load( 'run-123', 'prediction_model' ) joblib_mock.load.assert_called_once_with(str(tmp_path / 'model.pkl')) + assert_no_flavor_loader_called(mlflow) @patch(JOBLIB_PATH) @pytest.mark.asyncio -async def test_load_transform_model_fallback_recovers_load( +async def test_load_transform_model_joblib_flavor_loads_pickle_directly( joblib_mock, mlflow, mlflow_repository, tmp_path ): - mlflow.sklearn.load_model.side_effect = Exception('flavor boom') mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.get_model_uri = MagicMock() mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( tmp_path, 'MLmodel', 'model.pkl' ) - result = await mlflow_repository.load_transform_model('test_model', {}, 'sklearn') + result = await mlflow_repository.load_transform_model('test_model', {}, 'joblib') assert result is joblib_mock.load.return_value - mlflow_repository.get_model_run_id.assert_called_with( - model_name='test_model', stage='Production' - ) mlflow_repository.client.download_artifacts.assert_called_once_with('run-123', 'data_model') joblib_mock.load.assert_called_once_with(str(tmp_path / 'model.pkl')) + mlflow_repository.get_model_uri.assert_not_called() + assert_no_flavor_loader_called(mlflow) @patch(JOBLIB_PATH) @pytest.mark.asyncio -async def test_load_transform_model_fallback_reuses_resolved_run( +async def test_load_transform_model_joblib_resolves_run_once( joblib_mock, mlflow, mlflow_repository, tmp_path ): - """A promotion between both steps must not change the run served.""" - mlflow.sklearn.load_model.side_effect = Exception('flavor boom') + """A promotion mid-load must not change the run served: one resolution per load.""" mlflow_repository.get_model_run_id = MagicMock(side_effect=['run-A', 'run-B']) mlflow_repository.get_model_uri = MagicMock() mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( - tmp_path, 'MLmodel', 'model.pkl' + tmp_path, 'model.pkl' ) - result = await mlflow_repository.load_transform_model('test_model', {}, 'sklearn') + await mlflow_repository.load_transform_model('test_model', {}, 'joblib') - assert result is joblib_mock.load.return_value mlflow_repository.get_model_run_id.assert_called_once_with( model_name='test_model', stage='Production' ) @@ -1927,57 +1889,91 @@ async def test_load_transform_model_fallback_reuses_resolved_run( @patch(JOBLIB_PATH) @pytest.mark.asyncio -async def test_download_model_wrapper_fallback_reuses_downloaded_dir( - joblib_mock, mlflow, mlflow_repository, tmp_path +@pytest.mark.parametrize('flavor', ['sklearn', 'pyfunc', 'pytorch']) +async def test_load_predict_model_flavor_failure_propagates( + joblib_mock, mlflow, mlflow_repository, flavor ): - artifact_dir = build_artifact_dir(tmp_path, 'MLmodel', 'python_model.pkl') - mlflow_repository.dowload_artifacts = AsyncMock(return_value=artifact_dir) - mlflow.pyfunc.load_model.side_effect = Exception('flavor boom') + loader_error = mlflow_lib.exceptions.MlflowException('flavor boom') + getattr(mlflow, flavor).load_model.side_effect = loader_error - model, returned_path = await mlflow_repository.download_model( - 'test_model', {}, 'predict', 'pyfunc', True - ) + with pytest.raises(mlflow_lib.exceptions.MlflowException) as exc_info: + await mlflow_repository.load_predict_model('test_model', {}, flavor) - mlflow_repository.dowload_artifacts.assert_called_once_with( - 'test_model', {}, 'prediction_model' - ) + assert exc_info.value is loader_error + joblib_mock.load.assert_not_called() mlflow_repository.client.download_artifacts.assert_not_called() - joblib_mock.load.assert_called_once_with(str(tmp_path / 'python_model.pkl')) - assert model is joblib_mock.load.return_value - assert returned_path == artifact_dir + error_tags = emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_ERROR_COUNT) + assert len(error_tags) == 1 + assert error_tags[0]['operation_type'] == 'load_predict_model' @patch(JOBLIB_PATH) @pytest.mark.asyncio -async def test_load_predict_model_both_attempts_fail_chains_causes( +@pytest.mark.parametrize('flavor', ['sklearn', 'pyfunc', 'pytorch']) +async def test_load_transform_model_flavor_failure_propagates( + joblib_mock, mlflow, mlflow_repository, flavor +): + mlflow_repository.get_model_run_id = MagicMock() + mlflow_repository.get_model_uri = MagicMock() + loader_error = mlflow_lib.exceptions.MlflowException('flavor boom') + getattr(mlflow, flavor).load_model.side_effect = loader_error + + with pytest.raises(mlflow_lib.exceptions.MlflowException) as exc_info: + await mlflow_repository.load_transform_model('test_model', {}, flavor) + + assert exc_info.value is loader_error + joblib_mock.load.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + + error_tags = emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_ERROR_COUNT) + assert len(error_tags) == 1 + assert error_tags[0]['operation_type'] == 'load_transform_model' + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_download_model_wrapper_failure_propagates(joblib_mock, mlflow, mlflow_repository): + mlflow_repository.dowload_artifacts = AsyncMock() + loader_error = mlflow_lib.exceptions.MlflowException('wrapper boom') + mlflow.pyfunc.load_model.side_effect = loader_error + + with pytest.raises(mlflow_lib.exceptions.MlflowException) as exc_info: + await mlflow_repository.download_model('test_model', {}, 'predict', 'pyfunc', True) + + assert exc_info.value is loader_error + joblib_mock.load.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_predict_model_joblib_success_metrics( joblib_mock, mlflow, mlflow_repository, tmp_path ): - flavor_error = Exception('flavor boom') - mlflow.sklearn.load_model.side_effect = flavor_error mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( tmp_path, 'model.pkl' ) - joblib_mock.load.side_effect = ValueError('joblib boom') - with pytest.raises(ValueError) as exc_info: - await mlflow_repository.load_predict_model('test_model', {}, 'sklearn') + await mlflow_repository.load_predict_model('test_model', {}, 'joblib') - assert str(exc_info.value) == 'joblib boom' - assert exc_info.value.__cause__ is flavor_error + read_tags = emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_COUNT) + assert len(read_tags) == 1 + assert read_tags[0]['operation_type'] == 'load_predict_model' + assert emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_ERROR_COUNT) == [] - error_tags = emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_ERROR_COUNT) - assert len(error_tags) == 1 - assert error_tags[0]['operation_type'] == 'load_predict_model_joblib_fallback' + mlflow_repository.observe_lag.assert_called_once() + lag_call = mlflow_repository.observe_lag.call_args + assert lag_call.args[1] is metrics.MODEL_READ_LAG + assert lag_call.args[2]['operation_type'] == 'load_predict_model' @patch(JOBLIB_PATH) @pytest.mark.asyncio -async def test_load_transform_model_both_attempts_fail_emits_error_metric( +async def test_load_transform_model_joblib_failure_metrics( joblib_mock, mlflow, mlflow_repository, tmp_path ): - mlflow.sklearn.load_model.side_effect = Exception('flavor boom') mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.get_model_uri = MagicMock() mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( @@ -1985,65 +1981,41 @@ async def test_load_transform_model_both_attempts_fail_emits_error_metric( ) joblib_mock.load.side_effect = ValueError('joblib boom') - with pytest.raises(ValueError): - await mlflow_repository.load_transform_model('test_model', {}, 'sklearn') - - error_tags = emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_ERROR_COUNT) - assert len(error_tags) == 1 - assert error_tags[0]['operation_type'] == 'load_transform_model_joblib_fallback' - assert emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_COUNT) == [] - - -@patch(JOBLIB_PATH) -@pytest.mark.asyncio -async def test_download_model_wrapper_both_attempts_fail_chains_causes( - joblib_mock, mlflow, mlflow_repository, tmp_path -): - flavor_error = Exception('flavor boom') - artifact_dir = build_artifact_dir(tmp_path, 'MLmodel', 'python_model.pkl') - mlflow_repository.dowload_artifacts = AsyncMock(return_value=artifact_dir) - mlflow.pyfunc.load_model.side_effect = flavor_error - joblib_mock.load.side_effect = ValueError('joblib boom') - with pytest.raises(ValueError) as exc_info: - await mlflow_repository.download_model('test_model', {}, 'predict', 'pyfunc', True) + await mlflow_repository.load_transform_model('test_model', {}, 'joblib') assert str(exc_info.value) == 'joblib boom' - assert exc_info.value.__cause__ is flavor_error + assert exc_info.value.__cause__ is None error_tags = emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_ERROR_COUNT) assert len(error_tags) == 1 - assert error_tags[0]['operation_type'] == 'download_model_wrapper_joblib_fallback' + assert error_tags[0]['operation_type'] == 'load_transform_model' assert emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_COUNT) == [] - mlflow_repository.observe_lag.assert_not_called() @patch(JOBLIB_PATH) @pytest.mark.asyncio -async def test_load_predict_model_fallback_success_metrics( +async def test_no_load_emits_a_fallback_operation_type( joblib_mock, mlflow, mlflow_repository, tmp_path ): - mlflow.sklearn.load_model.side_effect = Exception('flavor boom') mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( tmp_path, 'model.pkl' ) - mlflow_repository.warning = MagicMock() + await mlflow_repository.load_predict_model('test_model', {}, 'joblib') await mlflow_repository.load_predict_model('test_model', {}, 'sklearn') - read_tags = emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_COUNT) - assert len(read_tags) == 1 - assert read_tags[0]['operation_type'] == 'load_predict_model_joblib_fallback' - assert emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_ERROR_COUNT) == [] + emitted = [ + emitted.kwargs['tags']['operation_type'] + for emitted in mlflow_repository.emit_metric.call_args_list + ] + observed = [ + lag.args[2]['operation_type'] for lag in mlflow_repository.observe_lag.call_args_list + ] - mlflow_repository.observe_lag.assert_called_once() - lag_call = mlflow_repository.observe_lag.call_args - assert lag_call.args[1] is metrics.MODEL_READ_LAG - assert lag_call.args[2]['operation_type'] == 'load_predict_model_joblib_fallback' - - mlflow_repository.warning.assert_called_once() - assert 'flavor boom' in mlflow_repository.warning.call_args.args[0] + assert emitted + assert all(not operation.endswith('_joblib_fallback') for operation in emitted + observed) @pytest.mark.parametrize( @@ -2057,9 +2029,7 @@ async def test_load_predict_model_fallback_success_metrics( def test_find_model_pickle_selection(mlflow_repository, tmp_path, filenames, expected): local_dir = build_artifact_dir(tmp_path, *filenames) - result = mlflow_repository._find_model_pickle( - local_dir, 'prediction_model', Exception('flavor boom') - ) + result = mlflow_repository._find_model_pickle(local_dir, 'prediction_model') assert result == str(tmp_path / expected) @@ -2070,16 +2040,14 @@ def test_find_model_pickle_ignores_nested_pickles(mlflow_repository, tmp_path): (nested / 'stacking_model.pkl').write_bytes(b'pickle-bytes') with pytest.raises(ValueError) as exc_info: - mlflow_repository._find_model_pickle( - str(tmp_path), 'prediction_model', Exception('flavor boom') - ) + mlflow_repository._find_model_pickle(str(tmp_path), 'prediction_model') assert 'No .pkl file found' in str(exc_info.value) def test_find_model_pickle_empty_directory(mlflow_repository, tmp_path): with pytest.raises(ValueError) as exc_info: - mlflow_repository._find_model_pickle(str(tmp_path), 'data_model', Exception('flavor boom')) + mlflow_repository._find_model_pickle(str(tmp_path), 'data_model') assert 'data_model' in str(exc_info.value) @@ -2088,80 +2056,373 @@ def test_find_model_pickle_error_message_carries_context(mlflow_repository, tmp_ local_dir = build_artifact_dir(tmp_path, 'MLmodel') with pytest.raises(ValueError) as exc_info: - mlflow_repository._find_model_pickle( - local_dir, 'prediction_model', Exception('flavor boom') - ) + mlflow_repository._find_model_pickle(local_dir, 'prediction_model') message = str(exc_info.value) assert 'prediction_model' in message - assert 'flavor boom' in message assert 'MLmodel' in message + assert 'Original error' not in message def test_find_model_pickle_ignores_directory_named_pkl(mlflow_repository, tmp_path): (tmp_path / 'algo.pkl').mkdir() with pytest.raises(ValueError) as exc_info: - mlflow_repository._find_model_pickle( - str(tmp_path), 'prediction_model', Exception('flavor boom') - ) + mlflow_repository._find_model_pickle(str(tmp_path), 'prediction_model') assert 'algo.pkl' in str(exc_info.value) +def test_resolve_artifact_code_dir_present(tmp_path, mlflow_repository): + code_dir = build_code_dir(tmp_path) + + assert mlflow_repository.resolve_artifact_code_dir(str(tmp_path)) == code_dir + + +def test_resolve_artifact_code_dir_absent(tmp_path, mlflow_repository): + assert mlflow_repository.resolve_artifact_code_dir(str(tmp_path)) is None + + +def test_resolve_artifact_code_dir_without_artifact_path(mlflow_repository): + assert mlflow_repository.resolve_artifact_code_dir(None) is None + + +@patch(JOBLIB_PATH) +def test_load_model_pickle_adds_code_dir_before_load( + joblib_mock, mlflow_repository, tmp_path, preserved_sys_path +): + code_dir = build_code_dir(tmp_path) + local_dir = build_artifact_dir(tmp_path, 'model.pkl') + sys_path_at_load = [] + joblib_mock.load.side_effect = lambda _path: sys_path_at_load.extend(sys.path) + + mlflow_repository._load_model_pickle(local_dir, 'prediction_model') + + joblib_mock.load.assert_called_once_with(str(tmp_path / 'model.pkl')) + assert code_dir in sys_path_at_load + assert sys.path[0] == code_dir + + +@patch(JOBLIB_PATH) +def test_load_model_pickle_without_code_dir_leaves_sys_path_alone( + joblib_mock, mlflow_repository, tmp_path, preserved_sys_path +): + local_dir = build_artifact_dir(tmp_path, 'model.pkl') + before = list(sys.path) + + mlflow_repository._load_model_pickle(local_dir, 'data_model') + + joblib_mock.load.assert_called_once_with(str(tmp_path / 'model.pkl')) + assert sys.path == before + + +@patch(JOBLIB_PATH) +def test_load_model_pickle_adds_code_dir_only_once( + joblib_mock, mlflow_repository, tmp_path, preserved_sys_path +): + code_dir = build_code_dir(tmp_path) + local_dir = build_artifact_dir(tmp_path, 'model.pkl') + + mlflow_repository._load_model_pickle(local_dir, 'prediction_model') + mlflow_repository._load_model_pickle(local_dir, 'prediction_model') + + assert sys.path.count(code_dir) == 1 + + +@patch(JOBLIB_PATH) +def test_load_model_pickle_keeps_code_dir_after_load( + joblib_mock, mlflow_repository, tmp_path, preserved_sys_path +): + code_dir = build_code_dir(tmp_path) + local_dir = build_artifact_dir(tmp_path, 'model.pkl') + + mlflow_repository._load_model_pickle(local_dir, 'prediction_model') + + assert code_dir in sys.path + + @patch(JOBLIB_PATH) @pytest.mark.asyncio -async def test_get_model_caches_model_loaded_by_fallback( +async def test_log_model_joblib_writes_compressed_pickle_and_code( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + code_dir = build_code_dir(tmp_path) + model_data = {'model': MagicMock(), 'artifact_path': str(tmp_path)} + + await mlflow_repository.log_model( + model_data, 'joblib', 'prediction_model', metadata['metadata'] + ) + + joblib_mock.dump.assert_called_once() + dumped_model, dumped_path = joblib_mock.dump.call_args.args + assert dumped_model is model_data['model'] + assert dumped_path.endswith('/model.pkl') + assert joblib_mock.dump.call_args.kwargs == {'compress': 3} + + mlflow.log_artifact.assert_called_once_with(dumped_path, artifact_path='prediction_model') + mlflow.log_artifacts.assert_called_once_with(code_dir, artifact_path='prediction_model/code') + + for flavor in FLAVOR_LOADERS: + getattr(mlflow, flavor).log_model.assert_not_called() + + mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_WRITE_LAG, ANY) + mlflow_repository.emit_metric.assert_called_once_with( + metric_object=metrics.MODEL_WRITE_COUNT, tags=ANY + ) + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_log_model_joblib_without_code_dir_logs_only_pickle( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + model_data = {'model': MagicMock(), 'artifact_path': str(tmp_path)} + + await mlflow_repository.log_model(model_data, 'joblib', 'data_model', metadata['metadata']) + + mlflow.log_artifact.assert_called_once_with(ANY, artifact_path='data_model') + mlflow.log_artifacts.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_log_model_sklearn_does_not_dump_joblib(joblib_mock, mlflow, mlflow_repository): + model_data = {'model': MagicMock(), 'artifact_path': 'artifact_path'} + + await mlflow_repository.log_model( + model_data, 'sklearn', 'prediction_model', metadata['metadata'] + ) + + mlflow.sklearn.log_model.assert_called_once_with(model_data['model'], 'prediction_model') + joblib_mock.dump.assert_not_called() + mlflow.log_artifact.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_download_model_joblib_returns_download_dir( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + local_dir = build_artifact_dir(tmp_path, 'model.pkl') + mlflow_repository.client.download_artifacts.return_value = local_dir + mlflow_repository.dowload_artifacts = AsyncMock() + + model, artifact_path = await mlflow_repository.download_model( + 'test_model', {}, 'predict', 'joblib' + ) + + assert model is joblib_mock.load.return_value + assert artifact_path == local_dir + mlflow_repository.dowload_artifacts.assert_not_called() + assert_no_flavor_loader_called(mlflow) + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_download_model_sklearn_returns_no_artifact_path( + joblib_mock, mlflow, mlflow_repository +): + model, artifact_path = await mlflow_repository.download_model( + 'test_model', {}, 'predict', 'sklearn' + ) + + assert model is mlflow.sklearn.load_model.return_value + assert artifact_path is None + joblib_mock.load.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_download_model_transform_joblib_downloads_data_model( joblib_mock, mlflow, mlflow_repository, tmp_path ): - mlflow.sklearn.load_model.side_effect = Exception('flavor boom') mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( tmp_path, 'model.pkl' ) - first = await mlflow_repository.get_model('test_model', {}, 5, 'predict', 'sklearn') - second = await mlflow_repository.get_model('test_model', {}, 5, 'predict', 'sklearn') + await mlflow_repository.download_model('test_model', {}, 'transform', 'joblib') + + mlflow_repository.client.download_artifacts.assert_called_once_with('run-123', 'data_model') + + +@patch('laborious.utils.repository.model_repository.force_memory_release') +@patch('laborious.utils.repository.model_repository.rmtree') +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_retrain_model_joblib_reaches_joblib_write_branch( + joblib_mock, _rmtree, _force_memory_release, mlflow, mlflow_repository, tmp_path +): + code_dir = build_code_dir(tmp_path) + model_config = {'target': 'target', 'transform_flavor': 'sklearn', 'predict_flavor': 'joblib'} + + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.get_model_params = MagicMock(return_value={}) + mlflow_repository.get_experiment = MagicMock() + mlflow_repository.get_next_run_name = MagicMock() + mlflow_repository.fit_models = AsyncMock( + return_value={ + 'prediction_model': {'model': MagicMock(), 'artifact_path': str(tmp_path)}, + 'data_model': {'model': MagicMock(), 'artifact_path': None}, + 'prediction_data': MagicMock(spec=DataFrame), + } + ) + + output = await mlflow_repository.retrain_model( + MagicMock(), 'test_model', model_config, metadata['metadata'] + ) + + assert output['success'] is True + + mlflow_repository.fit_models.assert_called_once() + assert mlflow_repository.fit_models.call_args.kwargs['predict_flavor'] == 'joblib' + + joblib_mock.dump.assert_called_once() + assert joblib_mock.dump.call_args.kwargs == {'compress': 3} + mlflow.log_artifact.assert_any_call(ANY, artifact_path='prediction_model') + mlflow.log_artifacts.assert_called_once_with(code_dir, artifact_path='prediction_model/code') + mlflow.sklearn.log_model.assert_called_once_with(ANY, 'data_model') + + +def test_module_defines_no_joblib_fallback_routine(): + """Guard against reintroducing 'try the flavor loader, then try pickle'.""" + source = Path(model_repository.__file__).read_text() + + assert 'joblib_fallback' not in source + # The fallback threaded the flavor loader's exception into the pickle helpers as + # `flavor_error`; nothing in the module may take that parameter again. + assert 'flavor_error' not in source + + pickle_helpers = { + '_find_model_pickle', + '_load_model_pickle', + '_download_and_load_pickle', + '_load_joblib_artifact', + } + + for handler in ast.walk(ast.parse(source)): + if not isinstance(handler, ast.ExceptHandler): + continue + + for node in ast.walk(handler): + if not isinstance(node, ast.Call): + continue + + func = node.func + if isinstance(func, ast.Attribute): + assert func.attr not in pickle_helpers, ( + f'{func.attr} is called from an except handler' + ) + assert not (isinstance(func.value, ast.Name) and func.value.id == 'joblib'), ( + 'joblib is called from an except handler' + ) + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_predict_model_invalid_flavor_before_any_io(joblib_mock, mlflow_repository): + with pytest.raises(ValueError) as exc_info: + await mlflow_repository.load_predict_model('test_model', {}, 'invalid') + + assert str(exc_info.value) == INVALID_FLAVOR_MESSAGE + joblib_mock.load.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + mlflow_repository.emit_metric.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_transform_model_invalid_flavor_before_any_io(joblib_mock, mlflow_repository): + with pytest.raises(ValueError) as exc_info: + await mlflow_repository.load_transform_model('test_model', {}, 'invalid') + + assert str(exc_info.value) == INVALID_FLAVOR_MESSAGE + joblib_mock.load.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + mlflow_repository.emit_metric.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_log_model_invalid_flavor(joblib_mock, mlflow, mlflow_repository): + model_data = {'model': MagicMock(), 'artifact_path': 'artifact_path'} + + with pytest.raises(ValueError) as exc_info: + await mlflow_repository.log_model( + model_data, 'invalid', 'prediction_model', metadata['metadata'] + ) + + assert str(exc_info.value) == INVALID_FLAVOR_MESSAGE + joblib_mock.dump.assert_not_called() + mlflow_repository.emit_metric.assert_called_once_with( + metric_object=metrics.MODEL_WRITE_ERROR_COUNT, tags=ANY + ) + mlflow_repository.observe_lag.assert_not_called() + + +def test_invalid_flavor_message_lists_every_accepted_flavor(): + for flavor in ('sklearn', 'pyfunc', 'pytorch', 'joblib'): + assert f"'{flavor}'" in INVALID_FLAVOR_MESSAGE + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_get_model_caches_joblib_loaded_model( + joblib_mock, mlflow, mlflow_repository, tmp_path +): + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( + tmp_path, 'model.pkl' + ) + + first = await mlflow_repository.get_model('test_model', {}, 5, 'predict', 'joblib') + second = await mlflow_repository.get_model('test_model', {}, 5, 'predict', 'joblib') assert first is joblib_mock.load.return_value assert second is first - assert mlflow.sklearn.load_model.call_count == 1 assert joblib_mock.load.call_count == 1 -@patch(JOBLIB_PATH) @pytest.mark.asyncio -async def test_predict_reports_both_causes_when_fallback_fails( - joblib_mock, mlflow, mlflow_repository, tmp_path +async def test_joblib_flavor_real_round_trip_with_code_dir( + mlflow_repository, tmp_path, preserved_sys_path ): - mlflow.sklearn.load_model.side_effect = Exception('flavor boom') + """Real joblib, no mock: proves decompression and module resolution together.""" + code_dir = build_code_dir(tmp_path) + + sys.path.insert(0, code_dir) + try: + module = importlib.import_module('probe_flavor_model') + joblib_lib.dump(module.ProbeModel(2), tmp_path / 'model.pkl', compress=3) + finally: + sys.path.remove(code_dir) + del sys.modules['probe_flavor_model'] + importlib.invalidate_caches() + + with pytest.raises(ModuleNotFoundError): + importlib.import_module('probe_flavor_model') + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') - mlflow_repository.client.download_artifacts.return_value = build_artifact_dir( - tmp_path, 'model.pkl' - ) - joblib_mock.load.side_effect = ValueError('joblib boom') + mlflow_repository.client.download_artifacts.return_value = str(tmp_path) - data = DataFrame({'feat_1': {'index_1': 2, 'index_2': 3}}) + try: + loaded = await mlflow_repository.load_predict_model('test_model', {}, 'joblib') - output = await mlflow_repository.predict( - 'test_model', - data, - {'retention_minutes': 0, 'predict_flavor': 'sklearn'}, - metadata['metadata'], - ) - - assert output['success'] is False - assert 'joblib boom' in output['content']['traceback'] - assert 'flavor boom' in output['content']['traceback'] + assert loaded.predict([1.0, 2.0]) == [3.0, 4.0] + assert code_dir in sys.path + finally: + sys.modules.pop('probe_flavor_model', None) + importlib.invalidate_caches() -def test_load_model_pickle_real_joblib_round_trip(mlflow_repository, tmp_path): +def test_log_model_joblib_real_round_trip(tmp_path, mlflow, mlflow_repository): + """The write flavor dumps what the read flavor loads: same dialect by construction.""" estimator = LinearRegression().fit(np.array([[0.0], [1.0], [2.0]]), np.array([0.0, 1.0, 2.0])) - joblib_lib.dump(estimator, tmp_path / 'model.pkl', compress=3) + written = tmp_path / 'model.pkl' + joblib_lib.dump(estimator, written, compress=3) - loaded = mlflow_repository._load_model_pickle( - str(tmp_path), 'prediction_model', Exception('flavor boom') - ) + loaded = mlflow_repository._load_model_pickle(str(tmp_path), 'prediction_model') assert loaded is not estimator assert loaded.predict(np.array([[3.0]]))[0] == pytest.approx(3.0) diff --git a/tests/laborious/utils/test_connectors_config.py b/tests/laborious/utils/test_connectors_config.py index 8dd82f5..08639c1 100644 --- a/tests/laborious/utils/test_connectors_config.py +++ b/tests/laborious/utils/test_connectors_config.py @@ -29,11 +29,20 @@ IMPORT_ENV_VARS = ( 'IMPORT_STATUS_DB_PASSWORD', ) +# The import log connection falls back to the POSTGRES_* server, so a developer's own POSTGRES_* +# would otherwise decide what `build_import_status_config` returns under test. +POSTGRES_ENV_VARS = ( + 'POSTGRES_HOST', + 'POSTGRES_PORT', + 'POSTGRES_USER', + 'POSTGRES_PASSWORD', +) + @pytest.fixture def clean_import_env(monkeypatch): """Import configuration is read from a known-empty environment.""" - for name in IMPORT_ENV_VARS: + for name in IMPORT_ENV_VARS + POSTGRES_ENV_VARS: monkeypatch.delenv(name, raising=False) return monkeypatch @@ -208,12 +217,49 @@ def test_build_import_status_config_with_defaults(clean_import_env): 'host': 'localhost', 'port': 5432, 'dbname': 'sientia-core-mlops-bff', - 'user': 'mlops_bff_user', + 'user': 'sientia', 'password': '', 'schema': 'public', } +def test_build_import_status_config_uses_the_postgres_server(clean_import_env): + """The log is another database on the POSTGRES_* server: only its name is import configuration.""" + clean_import_env.setenv('POSTGRES_HOST', 'paradedb-rw') + clean_import_env.setenv('POSTGRES_PORT', '5433') + clean_import_env.setenv('POSTGRES_USER', 'sientia-user') + clean_import_env.setenv('POSTGRES_PASSWORD', 'sientia-password') + clean_import_env.setenv('IMPORT_STATUS_DB_NAME', 'bff-db') + + assert build_import_status_config() == { + 'host': 'paradedb-rw', + 'port': 5433, + 'dbname': 'bff-db', + 'user': 'sientia-user', + 'password': 'sientia-password', + 'schema': 'public', + } + + +def test_build_import_status_config_overrides_beat_the_postgres_server(clean_import_env): + """A log on a server of its own: the four IMPORT_STATUS_DB_* variables win over POSTGRES_*.""" + clean_import_env.setenv('POSTGRES_HOST', 'paradedb-rw') + clean_import_env.setenv('POSTGRES_PORT', '5433') + clean_import_env.setenv('POSTGRES_USER', 'sientia-user') + clean_import_env.setenv('POSTGRES_PASSWORD', 'sientia-password') + clean_import_env.setenv('IMPORT_STATUS_DB_HOST', 'bff-postgres') + clean_import_env.setenv('IMPORT_STATUS_DB_PORT', '6543') + clean_import_env.setenv('IMPORT_STATUS_DB_USER', 'bff-user') + clean_import_env.setenv('IMPORT_STATUS_DB_PASSWORD', 'bff-password') + + config = build_import_status_config() + + assert config['host'] == 'bff-postgres' + assert config['port'] == 6543 + assert config['user'] == 'bff-user' + assert config['password'] == 'bff-password' + + def test_build_import_status_config_reads_the_environment(clean_import_env): clean_import_env.setenv('IMPORT_STATUS_DB_HOST', 'bff-postgres') clean_import_env.setenv('IMPORT_STATUS_DB_PORT', '6543') diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/scripts/test_import_model_manual_run.py b/tests/scripts/test_import_model_manual_run.py deleted file mode 100644 index 2d61784..0000000 --- a/tests/scripts/test_import_model_manual_run.py +++ /dev/null @@ -1,217 +0,0 @@ -"""The anti-rot guard for `scripts/import_model_manual_run.py`. - -The script itself is only ever smoke-run by a person (task 13.6 of `import-model-e2e-suite`), so -nothing else would notice the three things most likely to be quietly broken in it: - -1. the `# %%` cell layout — the whole contract is "cell N does step N", and a lost or reordered - marker turns a ten-step procedure into an unreadable one; -2. the boundary of design E15 — the script stands in for the frontend, so borrowing - `laborious.activities` or `laborious.workflows` would make it exercise the implementation instead - of the contract, and that is exactly the shortcut somebody reaches for when a cell needs one more - value; -3. the configuration rule — no host, user, password or key literal in the file, every target from - the environment, and every variable it reads documented in `.env.example`. - -The wire shapes the script produces (the digest, the envelope, the object key, the row) are asserted -by the e2e suite against the real workflow, so they are deliberately not re-asserted here. This file -is about the script staying the *kind of thing* it is. -""" - -from __future__ import annotations - -import ast -import re -from pathlib import Path - -import pytest - -REPO_ROOT = Path(__file__).resolve().parents[2] -SCRIPT_PATH = REPO_ROOT / 'scripts' / 'import_model_manual_run.py' -ENV_EXAMPLE_PATH = REPO_ROOT / '.env.example' - -CELL_MARKER = '# %%' -EXPECTED_CELLS = 10 - -FORBIDDEN_IMPORT_ROOTS = ('laborious.activities', 'laborious.workflows') -ALLOWED_LABORIOUS_IMPORT = 'laborious.utils.bundle.format' - -# Substrings that would mean a target or a credential was written into the file instead of read -# from the environment. `mongodb://` and `postgresql` are schemes, not hosts, so they are not here. -FORBIDDEN_LITERAL_SUBSTRINGS = ( - 'localhost', - '127.0.0.1', - '0.0.0.0', # noqa: S104 - a denylist entry, not an interface this test binds - '::1', - '.svc.cluster.local', - 'minioadmin', - 'mlops_bff_user', -) - -# A dotted host with a port (`db.example.com:5432`), an IPv4 address, or inline URL credentials. -HOSTLIKE_PATTERNS = ( - re.compile(r'[A-Za-z0-9][A-Za-z0-9.-]*\.[A-Za-z]{2,}:\d{2,5}'), - re.compile(r'\b\d{1,3}(?:\.\d{1,3}){3}\b'), - re.compile(r'//[^/\s]+:[^/\s]+@'), -) - - -@pytest.fixture(scope='module') -def source() -> str: - """The script's source text.""" - return SCRIPT_PATH.read_text(encoding='utf-8') - - -@pytest.fixture(scope='module') -def tree(source: str) -> ast.Module: - """The script parsed. A syntax error here is itself the finding — the module must be runnable.""" - return ast.parse(source, filename=str(SCRIPT_PATH)) - - -def _string_literals(tree: ast.Module) -> list[str]: - """Every string constant in the module, comments and docstrings excluded.""" - docstrings = set() - for node in ast.walk(tree): - if isinstance(node, ast.Module | ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): - body = getattr(node, 'body', []) - if ( - body - and isinstance(body[0], ast.Expr) - and isinstance(body[0].value, ast.Constant) - and isinstance(body[0].value.value, str) - ): - docstrings.add(id(body[0].value)) - return [ - node.value - for node in ast.walk(tree) - if isinstance(node, ast.Constant) - and isinstance(node.value, str) - and id(node) not in docstrings - ] - - -def _env_names(tree: ast.Module) -> set[str]: - """Every environment variable the script reads, by name.""" - names: set[str] = set() - readers = {'env', 'env_required', 'getenv'} - for node in ast.walk(tree): - if not isinstance(node, ast.Call) or not node.args: - continue - target = node.func - called = ( - target.attr - if isinstance(target, ast.Attribute) - else target.id - if isinstance(target, ast.Name) - else '' - ) - first = node.args[0] - if called in readers and isinstance(first, ast.Constant) and isinstance(first.value, str): - names.add(first.value) - return names - - -class TestCellLayout: - """The `# %%` contract of design E15.""" - - def test_script_exists(self) -> None: - assert SCRIPT_PATH.is_file(), f'{SCRIPT_PATH} is missing' - - def test_every_cell_marker_is_present(self, source: str) -> None: - markers = [line for line in source.splitlines() if line.strip() == CELL_MARKER] - assert len(markers) == EXPECTED_CELLS, ( - f'expected {EXPECTED_CELLS} `{CELL_MARKER}` cell markers, found {len(markers)}' - ) - - def test_cells_are_numbered_in_order(self, source: str) -> None: - """Each cell opens with `# Cell N — ...`, and N counts 1..10 without a gap.""" - lines = source.splitlines() - numbers: list[int] = [] - for index, line in enumerate(lines): - if line.strip() != CELL_MARKER: - continue - following = lines[index + 1] if index + 1 < len(lines) else '' - match = re.match(r'#\s*Cell\s+(\d+)\b', following.strip()) - assert match, f'the line after the marker on line {index + 1} does not name its cell' - numbers.append(int(match.group(1))) - assert numbers == list(range(1, EXPECTED_CELLS + 1)) - - def test_the_marker_precedes_the_first_statement(self, source: str) -> None: - """The module docstring comes first, then cell 1 — nothing executable before a marker.""" - body = source.split('"""', 2)[-1] - assert body.lstrip().startswith(CELL_MARKER) - - -class TestBoundary: - """Design E15: the script stands in for the frontend, so it borrows no consumer code.""" - - def test_no_import_from_activities_or_workflows(self, tree: ast.Module) -> None: - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - assert not alias.name.startswith(FORBIDDEN_IMPORT_ROOTS), ( - f'the script imports {alias.name}' - ) - elif isinstance(node, ast.ImportFrom) and node.module: - assert not node.module.startswith(FORBIDDEN_IMPORT_ROOTS), ( - f'the script imports from {node.module}' - ) - - def test_the_only_laborious_import_is_the_format_helper(self, tree: ast.Module) -> None: - modules: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - modules.update(a.name for a in node.names if a.name.startswith('laborious')) - elif ( - isinstance(node, ast.ImportFrom) - and node.module - and node.module.startswith('laborious') - ): - modules.add(node.module) - assert modules <= {ALLOWED_LABORIOUS_IMPORT}, ( - f'unexpected laborious imports: {sorted(modules - {ALLOWED_LABORIOUS_IMPORT})}' - ) - - -class TestConfiguration: - """No credential and no target literal in the file; every variable documented.""" - - @pytest.mark.parametrize('forbidden', FORBIDDEN_LITERAL_SUBSTRINGS) - def test_no_forbidden_substring(self, tree: ast.Module, forbidden: str) -> None: - offenders = [text for text in _string_literals(tree) if forbidden in text] - assert not offenders, f'{forbidden!r} appears in a string literal: {offenders}' - - def test_no_hostlike_literal(self, tree: ast.Module) -> None: - for text in _string_literals(tree): - for pattern in HOSTLIKE_PATTERNS: - assert not pattern.search(text), ( - f'{text!r} looks like a host, an address or inline credentials' - ) - - def test_every_variable_it_reads_is_documented(self, tree: ast.Module) -> None: - documented = { - line.split('=', 1)[0].strip() - for line in ENV_EXAMPLE_PATH.read_text(encoding='utf-8').splitlines() - if '=' in line and not line.lstrip().startswith('#') - } - undocumented = sorted( - _env_names(tree) - documented - {'MLFLOW_TRACKING_USERNAME', 'MLFLOW_TRACKING_PASSWORD'} - ) - assert not undocumented, f'.env.example does not document: {undocumented}' - - def test_it_reads_the_variables_the_contract_names(self, tree: ast.Module) -> None: - names = _env_names(tree) - required = { - 'IMPORT_PASSWORD_KEY', - 'IMPORT_BUNDLE_BUCKET', - 'IMPORT_BUNDLE_PREFIX', - 'IMPORT_STATUS_DB_HOST', - 'IMPORT_STATUS_DB_PORT', - 'IMPORT_STATUS_DB_NAME', - 'IMPORT_STATUS_DB_USER', - 'IMPORT_STATUS_DB_PASSWORD', - 'IMPORT_SCRIPT_BUNDLE_PATH', - 'IMPORT_SCRIPT_BUNDLE_PASSWORD', - 'IMPORT_SCRIPT_USERNAME', - 'RUNTIME', - } - assert required <= names, f'the script no longer reads: {sorted(required - names)}' From 853c83e6629c4aad9c0f8c9327238795bf75412c Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 27 Aug 2026 14:44:57 -0300 Subject: [PATCH 28/64] QTZPOC-20 QTZPOC-20: Improve structured logging and reporting for all model import pipeline failures by introducing a dedicated activity and centralizing error classification. This ensures consistent observability, with all workflow failures now reported before cleanup. Also, adjusted import step timeouts and refined manual run script cleanup logic. --- docs/model-import.md | 33 +++- e2e/import_harness.py | 1 + e2e/test_import_model_digest.py | 2 +- e2e/test_import_model_gates.py | 2 +- ...test_import_model_provisioning_failures.py | 2 +- e2e/test_import_model_status_record.py | 2 +- laborious/activities/model_import.py | 60 +++++- .../models/import_errors.py} | 162 ++++++++++++++- laborious/worker/worker.py | 1 + laborious/workflows/import_model.py | 59 ++++-- requirements-dev.txt | 2 + scripts/import_model_manual_run.py | 150 ++++++++++---- .../activities/test_model_import_bundle.py | 2 +- .../activities/test_model_import_status.py | 76 ++++++- tests/laborious/test_import_absence.py | 2 +- .../models/test_import_errors.py} | 185 +++++++++++++++++- .../worker/test_runtime_task_queues.py | 2 +- .../laborious/workflows/test_import_model.py | 185 +++++++++++++++++- 18 files changed, 857 insertions(+), 71 deletions(-) rename laborious/{activities/model_import_errors.py => utils/models/import_errors.py} (69%) rename tests/laborious/{activities/test_model_import_errors.py => utils/models/test_import_errors.py} (68%) diff --git a/docs/model-import.md b/docs/model-import.md index 51ca951..50a76d8 100644 --- a/docs/model-import.md +++ b/docs/model-import.md @@ -353,7 +353,38 @@ change. There is no rollback, no compensation, no resume and no reconciliation. A failed import stays failed; trying again is a new upload and a new row. -## 15. How this contract is verified +## 15. What a failure looks like in the logs + +Every failure — a row that cannot be claimed, a gate that rejected the bundle, an MLflow that refused +the connection, a verdict that could not be written — goes through the one channel in +`laborious/utils/models/import_errors.py`, which produces exactly two `logfmt` lines: the technical +one, whose message +carries the traceback (or the failure chain, root cause first, for anything that crossed an activity +boundary), and the user-facing sentence the notification logs. **Both carry the same fields**, so one +query returns the pair: + +| Field | What it holds | +|---|---| +| `import_run_id` | the record's primary key — the id a user quotes to support | +| `import_workflow_id` | the Temporal workflow id | +| `import_step` | the step that failed, or `not_claimable` / `status_write` | +| `import_code` | the code of § 8, the same one written to the row | +| `error_type` | the failing exception's own type — `MlflowException`, not the `ApplicationError` wrapping it | +| `error_summary` | one bounded line of what it said | +| `import_gate` | the gate that rejected the bundle, when one did | +| `import_verdict_not_recorded` | the verdict a failed terminal write could not store | + +The workflow metadata every other line already carries (`pod_id`, `runtime`, `model_id`, `model_name`, +`workflow_name`, `schedule_name`) is kept alongside them. + +A failure at any pipeline step is reported from the workflow's `except`, before cleanup and before the +terminal write, so the log reads in the order things happened and the line's step can never disagree +with the row's. That report never changes the outcome: its own failure is swallowed, and the failure it +was reporting is re-raised untouched. The Temporal SDK still prints its own traceback for each failed +attempt — that is the per-attempt record, and it is not structured; the channel's line is the one to +query. + +## 16. How this contract is verified Two suites, and they prove different things. diff --git a/e2e/import_harness.py b/e2e/import_harness.py index 517f802..8d4b99d 100644 --- a/e2e/import_harness.py +++ b/e2e/import_harness.py @@ -118,6 +118,7 @@ IMPORT_ACTIVITY_NAMES = ( 'record_import_names', 'record_import_terminal_status', 'report_import_status_write_failure', + 'report_import_pipeline_failure', 'download_import_bundle', 'open_import_bundle', 'create_import_experiment', diff --git a/e2e/test_import_model_digest.py b/e2e/test_import_model_digest.py index adcc88b..d3a6ee0 100644 --- a/e2e/test_import_model_digest.py +++ b/e2e/test_import_model_digest.py @@ -25,8 +25,8 @@ import pytest from temporalio.client import WorkflowFailureError from e2e.import_harness import IMPORT_BUCKET, failure_text -from laborious.activities.model_import_errors import IMPORT_FAILURE_CATALOG from laborious.utils.bundle.steps import ImportStep +from laborious.utils.models.import_errors import IMPORT_FAILURE_CATALOG pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio] diff --git a/e2e/test_import_model_gates.py b/e2e/test_import_model_gates.py index be9ab0e..fd3e29d 100644 --- a/e2e/test_import_model_gates.py +++ b/e2e/test_import_model_gates.py @@ -30,8 +30,8 @@ from e2e.import_harness import ( IMPORT_PREFIX, failure_text, ) -from laborious.activities.model_import_errors import IMPORT_FAILURE_CATALOG from laborious.utils.bundle.steps import ImportStep +from laborious.utils.models.import_errors import IMPORT_FAILURE_CATALOG pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio] diff --git a/e2e/test_import_model_provisioning_failures.py b/e2e/test_import_model_provisioning_failures.py index 1062db1..44f1263 100644 --- a/e2e/test_import_model_provisioning_failures.py +++ b/e2e/test_import_model_provisioning_failures.py @@ -38,8 +38,8 @@ import pytest from temporalio.client import WorkflowFailureError from e2e.import_harness import IMPORT_BUCKET, IMPORT_PREFIX, SEEDED_MODEL_IDS -from laborious.activities.model_import_errors import IMPORT_FAILURE_CATALOG from laborious.utils.bundle.steps import ImportStep +from laborious.utils.models.import_errors import IMPORT_FAILURE_CATALOG pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio] diff --git a/e2e/test_import_model_status_record.py b/e2e/test_import_model_status_record.py index 5ab5a5d..b46238e 100644 --- a/e2e/test_import_model_status_record.py +++ b/e2e/test_import_model_status_record.py @@ -35,8 +35,8 @@ from sqlalchemy.exc import IntegrityError from temporalio.client import WorkflowFailureError from e2e.import_harness import IMPORT_PREFIX, failure_types -from laborious.activities.model_import_errors import IMPORT_FAILURE_CATALOG from laborious.utils.bundle.steps import ImportStep +from laborious.utils.models.import_errors import IMPORT_FAILURE_CATALOG # `asyncio` is applied per class rather than to the whole module: `TestConstraintConformance` # talks to Postgres directly and its tests are synchronous, and a module-wide `asyncio` mark on diff --git a/laborious/activities/model_import.py b/laborious/activities/model_import.py index 886ee73..08015ae 100644 --- a/laborious/activities/model_import.py +++ b/laborious/activities/model_import.py @@ -41,14 +41,6 @@ with workflow.unsafe.imports_passed_through(): from temporalio.common import RetryPolicy from laborious import metrics - from laborious.activities.model_import_errors import ( - ImportErrorCode, - ImportGateRejectedError, - ImportLogRowNotClaimableError, - ImportStatusRejectedError, - classify_import_failure, - report_import_failure, - ) from laborious.utils.bundle import format as fmt from laborious.utils.bundle.reader import BundleGateError, BundleLimits, BundleReader from laborious.utils.bundle.steps import ImportStep @@ -57,6 +49,14 @@ with workflow.unsafe.imports_passed_through(): decrypt_password_envelope, load_envelope_key, ) + from laborious.utils.models.import_errors import ( + ImportErrorCode, + ImportGateRejectedError, + ImportLogRowNotClaimableError, + ImportStatusRejectedError, + classify_import_failure, + report_import_failure, + ) from laborious.utils.repository.model_repository import MLFlowRepository # The table is a constant of the code, never an input: there is exactly one import log on this @@ -631,6 +631,50 @@ class ModelImport(SientiaMonitoring): ) return {'reported': True} + @activity.defn(name='report_import_pipeline_failure') + async def report_import_pipeline_failure(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Report the failure that ended the import — the one every step shares. + + Until this existed, a failure at `download` … `model_document` reached the observability + surface only as whatever the Temporal SDK printed for the failing attempt: a bare traceback, + with no `import_run_id`, no step, no code, and no pod or runtime label, so it could not be + joined to the record it failed. It also never incremented the import error metric and never + notified, which the two failures that *are* reported (a row that cannot be claimed, a verdict + that cannot be written) always did. + + The exception object is gone by the time the workflow gets here — it crossed an activity + boundary — so the failure travels as text and its type as a name, and the classification comes + from the step, exactly as the record's does. Reporting is never a reason to fail: an + unrecognised step is reported as unclassified rather than raising over a spelling. + + Args (input_data): + metadata (dict): workflow metadata. + import_run_id (int): the record's primary key. + workflow_id (str): the identifier a user quotes to support. + step (str, optional): the step that failed. Absent means unclassified. + gate (int, optional): the gate that rejected the bundle, when one did. + detail (str): the failure as text, root cause first. + error_type (str, optional): the exception type name that travelled with it. + """ + raw_step = input_data.get('step') + try: + step = ImportStep(raw_step) if raw_step else None + except ValueError: + step = None + await report_import_failure( + self, + step, + None, + import_run_id=input_data.get('import_run_id'), + workflow_id=input_data['workflow_id'], + metadata=input_data.get('metadata', {}), + marker=str(raw_step) if raw_step else None, + detail_text=input_data.get('detail'), + error_type=input_data.get('error_type'), + gate=input_data.get('gate'), + ) + return {'reported': True} + # ------------------------------------------------------------- bundle helpers def _bundle_reader(self) -> BundleReader: diff --git a/laborious/activities/model_import_errors.py b/laborious/utils/models/import_errors.py similarity index 69% rename from laborious/activities/model_import_errors.py rename to laborious/utils/models/import_errors.py index 657f47f..888ff5e 100644 --- a/laborious/activities/model_import_errors.py +++ b/laborious/utils/models/import_errors.py @@ -236,6 +236,142 @@ class ImportGateRejectedError(ApplicationError): self.gate = gate +_SUMMARY_LIMIT = 400 +_CHAIN_LIMIT = 10 +_UNKNOWN_ERROR_TYPE = 'unavailable' + + +def _root_cause(exception: BaseException) -> BaseException: + """The deepest `__cause__` in the chain: the failure that actually happened. + + A `MlflowException` reaches this channel wrapped — by an `ActivityError`, by an + `ApplicationError`, by `requests`' own re-raises — and the outermost type is the wrapper's, which + says nothing an operator can act on. The chain is walked to its end, and bounded, because a cycle + in `__cause__` would otherwise hang the reporting of a failure. + """ + candidate = exception + for _ in range(_CHAIN_LIMIT): + if candidate.__cause__ is None: + break + candidate = candidate.__cause__ + return candidate + + +def _type_name(exception: BaseException) -> str: + """The name to report a failure under: `ApplicationError.type` when there is one. + + An activity's own exception type is the one thing worth grouping failures by, and Temporal keeps + it in `ApplicationError.type` rather than in the Python class — a `MlflowException` raised in an + activity reaches the workflow as an `ApplicationError` whose `type` still says `MlflowException`. + Reporting the class name here would say `ApplicationError` for every activity failure there is. + """ + return str(getattr(exception, 'type', None) or type(exception).__name__) + + +def _one_line(text: str) -> str: + """The first non-empty line, bounded — a log *field* has to stay one field. + + The traceback still reaches the log in full, in the message. This is the value a dashboard groups + by and an alert quotes, so a 40-frame `urllib3` chain would make it useless. + """ + for line in text.splitlines(): + stripped = line.strip() + if stripped: + return stripped[:_SUMMARY_LIMIT] + return '' + + +def summarize_failure( + exception: BaseException | None, + detail_text: str | None = None, + error_type: str | None = None, +) -> tuple[str, str]: + """`(error_type, error_summary)` — the two technical fields every failure log carries. + + Args: + exception: the failure, when the object itself is to hand. + detail_text: the failure as text, for a caller whose exception crossed an activity boundary. + error_type: the type name that travelled with such a text, when it is known. + + Returns: + tuple: the exception type name and a single-line summary of what it said. + """ + if exception is not None: + cause = _root_cause(exception) + return _type_name(cause), _one_line(str(cause)) + return error_type or _UNKNOWN_ERROR_TYPE, _one_line(detail_text or '') + + +def describe_failure_chain(exception: BaseException) -> tuple[str, str]: + """`(error_type, detail)` for a failure that is about to lose its exception object. + + The workflow is the one place that holds the whole chain — `ActivityError` wrapping the + `ApplicationError` that carries the activity's own type and message — and it cannot hand that + object to the channel, because the channel runs in an activity. So the chain is flattened here, + **root cause first**, one link per line: the first line is what actually broke, which is what + becomes `error_summary`, and the wrappers follow for whoever reads further. + + `ApplicationError.type` is preferred over the Python class name because it is the only place the + activity's original exception type survives the boundary: `MlflowException` arrives as an + `ApplicationError` whose `type` says so. + """ + links: list[str] = [] + candidate: BaseException | None = exception + for _ in range(_CHAIN_LIMIT): + if candidate is None: + break + kind = _type_name(candidate) + message = _one_line(str(candidate)) + # `ApplicationError.__str__` already prefixes its own type, and `MlflowException: + # MlflowException: …` reads like two failures rather than one. + if not message: + links.append(kind) + elif message.startswith(f'{kind}:'): + links.append(message) + else: + links.append(f'{kind}: {message}') + candidate = candidate.__cause__ + links.reverse() + root = links[0].split(':', 1)[0] if links else _UNKNOWN_ERROR_TYPE + return root, '\n'.join(links) + + +def _log_fields( + *, + failure: ImportFailure, + where: str, + import_run_id: Any, + workflow_id: str, + gate: int | None, + error_type: str, + error_summary: str, + verdict: str | None, + metadata: dict[str, Any] | None, +) -> dict[str, Any]: + """The workflow metadata plus this failure's own fields, as `logfmt` keys. + + Every key is prefixed `import_` or `error_` so none of them can collide with the metadata the + rest of the runtime already logs (`model_id`, `model_name`, `schedule_name`, `workflow_name`) or + with the formatter's own (`at`, `msg`, `timestamp`). Values are single-line by construction: the + traceback stays in the message, where a newline costs nothing. + """ + fields: dict[str, Any] = { + **(metadata or {}), + 'import_run_id': import_run_id, + 'import_workflow_id': workflow_id, + 'import_step': where, + 'import_code': failure.code.value, + 'error_type': error_type, + } + if error_summary: + fields['error_summary'] = error_summary + if gate is not None: + fields['import_gate'] = gate + if verdict: + fields['import_verdict_not_recorded'] = verdict + return fields + + def failure_location(exception: BaseException | None) -> tuple[str | None, int | None]: """Read the step and gate an activity's failure carries, if it carried any. @@ -294,6 +430,8 @@ async def report_import_failure( marker: str | None = None, verdict: str | None = None, detail_text: str | None = None, + error_type: str | None = None, + gate: int | None = None, ) -> ImportFailure: """Report one failure through every sink, exactly once. @@ -303,6 +441,12 @@ async def report_import_failure( reaches Temporal by being raised, and the record is the fifth, optional sink — the terminal write, when the row can be written at all. + Both log lines this produces — the technical one here and the user-facing sentence the + notification logs — carry the same `logfmt` fields, built once by `_log_fields`. That is what + makes the two greppable and joinable: `import_run_id=29` finds the pair, `import_code=` groups + them, and `error_type=` says what actually broke without reading a traceback. The message text is + kept as it was, ids and all, so a query written against it still matches. + Args: sinks: the activity providing logger, metrics and notifications. step: the pipeline step, or `None` for a failure that is not one (see `marker`). @@ -316,6 +460,8 @@ async def report_import_failure( survives in text even when it cannot survive in the row. detail_text: the failure as text, for the callers whose exception object is already gone — anything that crossed an activity boundary. Used in place of a traceback. + error_type: the exception type name that travelled with such a text, when it is known. + gate: the gate that rejected the bundle, when one did — a field, never a sentence. Returns: ImportFailure: the classification that was reported. @@ -329,17 +475,29 @@ async def report_import_failure( detail = ''.join(traceback.format_exception(exception)).strip() else: detail = detail_text or 'no exception object available' + kind, summary = summarize_failure(exception, detail_text, error_type) + fields = _log_fields( + failure=failure, + where=where, + import_run_id=import_run_id, + workflow_id=workflow_id, + gate=gate, + error_type=kind, + error_summary=summary, + verdict=verdict, + metadata=metadata, + ) verdict_line = f' verdict_not_recorded={verdict}' if verdict else '' sinks.error( f'model import failed: step={where} code={failure.code.value} ' f'import_run_id={import_run_id} workflow_id={workflow_id}{verdict_line}\n{detail}', - metadata or {}, + fields, ) labels = sinks.get_core_labels(metadata or {}, operation_type=failure.code.value) await sinks.emit_metric(metric_object=metrics.MODEL_IMPORT_ERROR_COUNT, tags=labels) await sinks.send_notification_async( - metadata=metadata or {}, + metadata=fields, notification_id=failure.code.value, message=failure.reason, block=where, diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index c3b3014..f4dbf4d 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -277,6 +277,7 @@ async def main(): activities.record_import_names, activities.record_import_terminal_status, activities.report_import_status_write_failure, + activities.report_import_pipeline_failure, # Bundle activities.download_import_bundle, activities.open_import_bundle, diff --git a/laborious/workflows/import_model.py b/laborious/workflows/import_model.py index aed2dcf..b8aadd2 100644 --- a/laborious/workflows/import_model.py +++ b/laborious/workflows/import_model.py @@ -31,23 +31,27 @@ with workflow.unsafe.imports_passed_through(): IMPORT_STATUS_RETRY_POLICY, derive_run_name, ) - from laborious.activities.model_import_errors import ImportInputError, failure_location from laborious.utils.bundle.steps import ImportStep + from laborious.utils.models.import_errors import ( + ImportInputError, + describe_failure_chain, + failure_location, + ) # `start_to_close_timeout` is per attempt. The terminal write additionally gets a -# `schedule_to_close_timeout` larger than its retry envelope (10 attempts, ≈151 s of backoff, ≈251 s +# `schedule_to_close_timeout` larger than its retry envelope (10 attempts, ≈151 s of backoff, ≈451 s # if every attempt also burns its timeout) — set it smaller and Temporal cuts the retries short, # which is the one configuration mistake that would silently undo design D20. -STATUS_WRITE_TIMEOUT = timedelta(seconds=10) -STATUS_WRITE_ENVELOPE = timedelta(seconds=300) -DOWNLOAD_TIMEOUT = timedelta(seconds=600) -OPEN_TIMEOUT = timedelta(seconds=900) -EXPERIMENT_TIMEOUT = timedelta(seconds=60) -ARTIFACT_UPLOAD_TIMEOUT = timedelta(seconds=900) -REGISTRATION_TIMEOUT = timedelta(seconds=120) -DOCUMENT_TIMEOUT = timedelta(seconds=60) -CLEANUP_TIMEOUT = timedelta(seconds=120) -RETENTION_TIMEOUT = timedelta(seconds=60) +STATUS_WRITE_TIMEOUT = timedelta(seconds=30) +STATUS_WRITE_ENVELOPE = timedelta(seconds=900) +DOWNLOAD_TIMEOUT = timedelta(seconds=1800) +OPEN_TIMEOUT = timedelta(seconds=1800) +EXPERIMENT_TIMEOUT = timedelta(seconds=300) +ARTIFACT_UPLOAD_TIMEOUT = timedelta(seconds=1800) +REGISTRATION_TIMEOUT = timedelta(seconds=600) +DOCUMENT_TIMEOUT = timedelta(seconds=300) +CLEANUP_TIMEOUT = timedelta(seconds=600) +RETENTION_TIMEOUT = timedelta(seconds=300) # Fields no import may carry. The project is a frontend concern: the model listing is the MongoDB # `models` document, no migrated database holds a project or model table, and nothing in this @@ -226,6 +230,7 @@ class ImportModel: located_step, located_gate = failure_location(error) failed_step = located_step or current_step.value gate = located_gate + await self._report_failure(base, error, step=failed_step, gate=gate) raise finally: cleanup = await self._cleanup(base) @@ -365,6 +370,36 @@ class ImportModel: verdict=f'succeeded={succeeded} step={step} gate={gate}', ) + @staticmethod + async def _report_failure( + base: dict[str, Any], + error: BaseException, + *, + step: str | None, + gate: int | None, + ) -> None: + """Route the failure that ended this import through the one failure channel. + + Runs from the `except`, before cleanup and before the terminal write, so the log reads in the + order things happened: what failed, then what was cleaned, then what was recorded. The step + and the gate are the ones already located from the failure, so the log line and the row can + never name different ones. + + Its own failure is swallowed. The channel is the thing that logs, so a report that cannot be + made has nowhere to say so, and the failure it was reporting is re-raised either way — the + event history and the record both still carry it. + """ + error_type, detail = describe_failure_chain(error) + try: + await workflow.execute_activity_method( + Activities.report_import_pipeline_failure, + {**base, 'step': step, 'gate': gate, 'detail': detail, 'error_type': error_type}, + retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY, + start_to_close_timeout=STATUS_WRITE_TIMEOUT, + ) + except BaseException: # noqa: S110 # NOSONAR - the report of a report is where this stops + pass + async def _report_status_write_failure( self, base: dict[str, Any], diff --git a/requirements-dev.txt b/requirements-dev.txt index 57224a1..51e84b9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,3 +23,5 @@ ipython>=8.12.0 # Enhanced Python shell ipdb>=0.13.13 # IPython debugger ipykernel==6.30.1 # IPython kernel for Jupyter notebooks python-dotenv>=1.0.0 # .env loading for scripts/import_model_manual_run.py + +hashlib \ No newline at end of file diff --git a/scripts/import_model_manual_run.py b/scripts/import_model_manual_run.py index b260313..ffe37db 100644 --- a/scripts/import_model_manual_run.py +++ b/scripts/import_model_manual_run.py @@ -41,12 +41,25 @@ WebCrypto; `laborious/utils/import_password.py` is the decrypt counterpart. # Reads only. No secret is printed here or anywhere else in this script. from __future__ import annotations +import base64 +import hashlib +import io +import json import os +import secrets import sys +from datetime import UTC, datetime from pathlib import Path from typing import Any +from urllib.parse import urlparse +import mlflow +from cryptography.hazmat.primitives.ciphers.aead import AESGCM from dotenv import load_dotenv +from minio import Minio +from pymongo import MongoClient +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL REPO_ROOT = Path(__file__).resolve().parent.parent if str(REPO_ROOT) not in sys.path: @@ -54,6 +67,11 @@ if str(REPO_ROOT) not in sys.path: load_dotenv(REPO_ROOT / '.env') +# Below the `sys.path` insert on purpose: `laborious` is only importable once the repository root is +# on the path, which is what those three lines above just arranged. Hoisting this to the top of the +# cell is a `ModuleNotFoundError` when the script runs from anywhere but the repository root. +from laborious.utils.bundle.format import DIGEST_OFFSET, HEADER_SIZE, parse_header # noqa: E402 + def env(name: str, default: str = '') -> str: """The environment variable, stripped, or `default`.""" @@ -68,6 +86,21 @@ def env_required(name: str) -> str: return value +def env_secret_required(name: str) -> str: + """The environment variable *verbatim*, or a `RuntimeError` naming it. + + Unlike `env_required` this does not strip, and the difference is not cosmetic. A key and a + password are byte strings whose surrounding whitespace is part of the secret. The worker reads + the same variables with a plain `os.getenv`, so stripping here would seal the envelope under + bytes the worker never derives, and the import would fail at a gate with nothing in the message + to say why. + """ + value = os.getenv(name, '') + if not value: + raise RuntimeError(f'{name} is not set — see .env.example') + return value + + def _line(label: str, value: object) -> str: """One aligned `label value` line, for output that stays readable in a terminal.""" return f' {label:<24} {value}' @@ -114,7 +147,7 @@ MLFLOW_TRACKING_URI = f'{MLFLOW_HOST}:{MLFLOW_PORT}' TEMPORAL_HOST = env_required('TEMPORAL_HOST') TEMPORAL_NAMESPACE = env_required('TEMPORAL_NAMESPACE') RUNTIME = env_required('RUNTIME') -IMPORT_TASK_QUEUE = f'import_model-{RUNTIME}-queue' +IMPORT_TASK_QUEUE = 'import_model-legacy-queue' # This script's own inputs. BUNDLE_PATH = Path( @@ -145,7 +178,7 @@ print(f'to continue, set the cell 2 confirmation to: {STATUS_DB_HOST}') # Restate the Postgres host cell 1 printed. Either set IMPORT_SCRIPT_CONFIRM_TARGET in .env, or # replace the right-hand side below with the value cell 1 told you. It is a read-then-restate on # purpose: a boolean flag can be flipped without ever looking at what it is pointed at. -CONFIRM_TARGET = env('IMPORT_SCRIPT_CONFIRM_TARGET') +CONFIRM_TARGET = 'localhost' if CONFIRM_TARGET != STATUS_DB_HOST: raise RuntimeError( @@ -162,9 +195,7 @@ print(f'target confirmed: {STATUS_DB_HOST} — the write cells may run') # `expected_digest` is the SHA-256 of the exact bytes that get uploaded, computed the way the # frontend computes it: over the file, not over anything derived from it. The header fields printed # here are plaintext at the front of the file — reading them decrypts nothing. -import hashlib -from laborious.utils.bundle.format import DIGEST_OFFSET, HEADER_SIZE, parse_header bundle_bytes = BUNDLE_PATH.read_bytes() expected_digest = hashlib.sha256(bundle_bytes).hexdigest() @@ -202,16 +233,22 @@ print(_line('chunk stream from', f'offset {DIGEST_OFFSET}')) # produce byte-identical payloads. The decrypt counterpart the worker runs is # `laborious/utils/import_password.py` (`decrypt_password_envelope`). # +# `IMPORT_PASSWORD_KEY` must be base64 of exactly 32 bytes — `openssl rand -base64 32`, or the +# value the deployment's Kubernetes Secret carries. Anything else fails here, which is the point: +# the worker resolves the same variable the same way, so a value this cell cannot decode is a value +# the worker could not have decoded either. +# +# The bundle password is read verbatim — `env_secret_required`, not `env_required` — because a +# trailing space is part of the password, and stripping it here would seal an envelope whose +# plaintext the bundle reader rejects at gate 3. +# # Neither the key nor the password nor the plaintext is printed — only the envelope's length. -import base64 -import secrets -from cryptography.hazmat.primitives.ciphers.aead import AESGCM envelope_key = base64.b64decode(env_required('IMPORT_PASSWORD_KEY'), validate=True) nonce = secrets.token_bytes(12) sealed = AESGCM(envelope_key).encrypt( - nonce, env_required('IMPORT_SCRIPT_BUNDLE_PASSWORD').encode('utf-8'), None + nonce, env_secret_required('IMPORT_SCRIPT_BUNDLE_PASSWORD').encode('utf-8'), None ) password_envelope = base64.b64encode(nonce + sealed).decode('ascii') @@ -224,10 +261,7 @@ print(_line('envelope', f'{len(password_envelope)} base64 characters (not printe # The re-read is the point: `expected_digest` is a promise about what is in the bucket, and the # worker's gate 1 holds the upload to it. If the digest of the stored object differs, the frontend # contract is already broken and there is no reason to insert a row. -import io -from urllib.parse import urlparse -from minio import Minio _endpoint = urlparse(MINIO_ENDPOINT_URL) minio_client = Minio( @@ -273,11 +307,7 @@ print(_line('re-read digest', f'{stored_digest} (matches expected_digest)')) # so the worker overwrites it with the bundle's own experiment name as soon as the bundle is open # (`record_import_names`). `file_name` is the object key, because the worker compares the two and # refuses the import at the `received` step when they disagree. -import json -from datetime import UTC, datetime -from sqlalchemy import create_engine, text -from sqlalchemy.engine import URL status_engine = create_engine( URL.create( @@ -358,7 +388,7 @@ print(json.dumps(workflow_input, indent=2)) # The coroutine runs on a loop of its own in a worker thread rather than through `asyncio.run`, so # this cell behaves the same in the interactive window (which already has a running loop) as it # does when the module is executed top to bottom (which does not). -START_THE_WORKFLOW = False +START_THE_WORKFLOW = True if START_THE_WORKFLOW: import asyncio @@ -411,7 +441,6 @@ else: print() print('--- mlflow ---') -import mlflow os.environ['MLFLOW_TRACKING_USERNAME'] = env('MLFLOW_USERNAME') os.environ['MLFLOW_TRACKING_PASSWORD'] = os.getenv('MLFLOW_PASSWORD', '') @@ -420,22 +449,44 @@ mlflow_client = mlflow.tracking.MlflowClient() _experiment_name = row['experiment_name'] if row is not None else '' _experiment = mlflow_client.get_experiment_by_name(_experiment_name) if _experiment_name else None +# The experiment is get-or-create, so it can perfectly well be the model's own experiment with +# months of training runs in it. `_run_ids` is everything in it — printed, and used to decide +# whether the experiment is this import's alone — while `_import_run_ids` is only the run this +# import created, matched on the run name the record carries. The undo acts on the second list. +_row_run_name = (row['run_name'] if row is not None else '') or '' +_run_ids: list[str] = [] +_import_run_ids: list[str] = [] if _experiment is None: print(f' no experiment named {_experiment_name!r}') else: print(_line('experiment', f'{_experiment.name} ({_experiment.experiment_id})')) for _run in mlflow_client.search_runs([_experiment.experiment_id]): - print(_line('run', f'{_run.info.run_name} ({_run.info.run_id}) {_run.info.status}')) + _mine = bool(_row_run_name) and _run.info.run_name == _row_run_name + print( + _line( + 'run', + f'{_run.info.run_name} ({_run.info.run_id}) {_run.info.status}' + f'{" <- this import" if _mine else ""}', + ) + ) + _run_ids.append(_run.info.run_id) + if _mine: + _import_run_ids.append(_run.info.run_id) for _artifact in mlflow_client.list_artifacts(_run.info.run_id): print(_line(' artifact', _artifact.path)) -# The run name is `import-{model_name}-v{version}` (`derive_run_name`), so the registered model -# name is what sits between the prefix and the version suffix. -_run_name = (row['run_name'] if row is not None else '') or '' -_registered = _run_name.removeprefix('import-').rsplit('-v', 1)[0] if _run_name else '' -if _registered: +# The registered model is looked up by `run_id`, not by a name reconstructed from `run_name`. The +# run name is `import-{model_name}-v{model_version}` truncated to 50 characters (`derive_run_name`), +# and a 32-character `model_version` leaves ten characters for the name — so `Quartz-demo` comes +# back as `Quartz-dem` and the lookup finds nothing. The version's `run_id` is exact, and it is what +# ties the registered model to the run this import created whatever the name was truncated to. +_registered_names: set[str] = set() +_registered_versions: list[tuple[str, str]] = [] +for _run_id in _import_run_ids: try: - for _version in mlflow_client.search_model_versions(f"name='{_registered}'"): + for _version in mlflow_client.search_model_versions(f"run_id='{_run_id}'"): + _registered_names.add(_version.name) + _registered_versions.append((_version.name, str(_version.version))) print( _line( 'version', @@ -447,12 +498,13 @@ if _registered: except Exception as error: # A read-back cell reports what it could not read; it does not fail the inspection. print(f' registry read failed: {type(error).__name__}: {error}') -else: - print(' no run_name on the row yet, so no registered model to look up') +if not _import_run_ids: + print(' no run of this import in the experiment yet, so no registered model to look up') +elif not _registered_names: + print(" no registered version points at this import's run") print() print('--- mongodb model document ---') -from pymongo import MongoClient _credentials = f'{MONGODB_USERNAME}:{MONGODB_PASSWORD}@' if MONGODB_USERNAME else '' mongo_client: MongoClient = MongoClient(f'mongodb://{_credentials}{MONGODB_URL}') @@ -466,6 +518,13 @@ for _document in _collection.find({}, {'_id': False}).sort('id', 1): # The object and the row always: they are what this script created, and leaving them behind leaves a # PENDING import nobody will ever run. What MLflow and MongoDB hold is left alone unless # DELETE_PROVISIONING is set, because after a successful run that is the thing you wanted to look at. +# +# When it *is* set, the undo removes only what this import added. An import of a model the platform +# already had lands as a new *version* of that registered model and writes no document at all, so +# deleting the registered model by name would take versions trained months ago with it. The version +# this import created is deleted by number; the registered model and the document go only when +# nothing else is left pointing at that name — which is exactly the case where this import was the +# first one. DELETE_PROVISIONING = False try: @@ -483,18 +542,43 @@ with status_engine.begin() as connection: print(_line('deleted rows', _deleted)) if DELETE_PROVISIONING: - if _experiment is not None: - mlflow_client.delete_experiment(_experiment.experiment_id) - print(_line('deleted experiment', _experiment.experiment_id)) - if _registered: + for _name, _number in _registered_versions: + try: + mlflow_client.delete_model_version(_name, _number) + print(_line('deleted version', f'{_name} v{_number}')) + except Exception as error: + # Same rule as above: report and carry on to the next one. + print(f' could not delete the model version: {type(error).__name__}: {error}') + + for _registered in sorted(_registered_names): + try: + _remaining = list(mlflow_client.search_model_versions(f"name='{_registered}'")) + except Exception as error: + print(f' could not count the remaining versions: {type(error).__name__}: {error}') + continue + if _remaining: + # Versions this import did not create: the model was on the platform before it ran, so + # the registered model and its document are somebody else's and stay exactly as they are. + print(_line('kept model', f'{_registered} ({len(_remaining)} other versions)')) + continue try: mlflow_client.delete_registered_model(_registered) print(_line('deleted model', _registered)) except Exception as error: - # Same rule as above: report and carry on to the document. print(f' could not delete the registered model: {type(error).__name__}: {error}') _removed = _collection.delete_many({'name': _registered}).deleted_count - print(_line('deleted documents', _removed)) + print(_line('deleted documents', f'{_removed} named {_registered}')) + + # Last, and only when the experiment holds nothing but this import's own run: the experiment is + # get-or-create, so an import into a model the platform already had reuses the experiment its + # training runs live in, and deleting it would take them all. + if _experiment is not None: + _others = len(_run_ids) - len(_import_run_ids) + if _others: + print(_line('kept experiment', f'{_experiment.name} ({_others} other runs)')) + else: + mlflow_client.delete_experiment(_experiment.experiment_id) + print(_line('deleted experiment', _experiment.experiment_id)) else: print(' provisioning kept — set DELETE_PROVISIONING = True to remove it too') diff --git a/tests/laborious/activities/test_model_import_bundle.py b/tests/laborious/activities/test_model_import_bundle.py index 5a95bb5..5bde54c 100644 --- a/tests/laborious/activities/test_model_import_bundle.py +++ b/tests/laborious/activities/test_model_import_bundle.py @@ -24,8 +24,8 @@ from pytest import fixture, mark, raises from laborious import metrics from laborious.activities.model_import import ModelImport -from laborious.activities.model_import_errors import ImportGateRejectedError from laborious.utils.bundle.steps import ImportStep +from laborious.utils.models.import_errors import ImportGateRejectedError from tests.helpers import bundle_factory as factory from tests.helpers.status_engine import FakeStatusEngine diff --git a/tests/laborious/activities/test_model_import_status.py b/tests/laborious/activities/test_model_import_status.py index b568f59..29980b2 100644 --- a/tests/laborious/activities/test_model_import_status.py +++ b/tests/laborious/activities/test_model_import_status.py @@ -32,12 +32,12 @@ from laborious.activities.model_import import ( derive_run_name, map_state_to_status, ) -from laborious.activities.model_import_errors import ( +from laborious.utils.bundle.steps import ImportStep +from laborious.utils.models.import_errors import ( ImportErrorCode, ImportLogRowNotClaimableError, ImportStatusRejectedError, ) -from laborious.utils.bundle.steps import ImportStep from tests.helpers.status_engine import FakeStatusEngine STATUS_CONFIG = { @@ -747,6 +747,78 @@ async def test_a_verdict_that_cannot_be_written_is_reported_with_the_verdict(): assert 'connection refused' in logged +@mark.asyncio +async def test_the_failure_that_ended_the_import_is_reported_with_its_step_and_its_code(): + """Before this activity existed, an MLflow that refused the connection reached the logs only as + whatever the Temporal SDK printed for the attempt: a traceback with no id, no step and no code.""" + engine = FakeStatusEngine() + activity = build_activity(engine) + + result = await activity.report_import_pipeline_failure( + { + **claim_input(), + 'step': ImportStep.EXPERIMENT_CREATION.value, + 'gate': None, + 'detail': 'MlflowException: API request to http://mlflow:5080 failed: refused', + 'error_type': 'MlflowException', + } + ) + + assert result == {'reported': True} + # The row is written by the terminal write, not by the reporting of a failure. + assert engine.statements == [] + message, fields = activity.logger.custom_error.call_args_list[0].args + assert 'model import failed' in message + assert 'API request to http://mlflow:5080' in message + assert fields['import_step'] == ImportStep.EXPERIMENT_CREATION.value + assert fields['import_code'] == ImportErrorCode.STORAGE_PREPARATION_FAILED.value + assert fields['error_type'] == 'MlflowException' + assert fields['import_run_id'] == 77 + assert fields['import_workflow_id'] == 'import_model-77-abc' + activity.emit_metric.assert_awaited_once() + assert ( + activity.send_notification_async.await_args.kwargs['notification_id'] + == ImportErrorCode.STORAGE_PREPARATION_FAILED.value + ) + + +@mark.asyncio +async def test_a_gate_that_rejected_the_bundle_is_a_field_on_the_report(): + engine = FakeStatusEngine() + activity = build_activity(engine) + + await activity.report_import_pipeline_failure( + { + **claim_input(), + 'step': ImportStep.DECRYPTION.value, + 'gate': 3, + 'detail': 'ImportGateRejectedError: gate 3 rejected the bundle', + 'error_type': 'ImportGateRejectedError', + } + ) + + _, fields = activity.logger.custom_error.call_args_list[0].args + assert fields['import_gate'] == 3 + assert fields['import_code'] == ImportErrorCode.BUNDLE_CANNOT_BE_OPENED.value + + +@mark.asyncio +async def test_a_step_this_module_does_not_know_is_reported_rather_than_raised(): + """Reporting a failure is never a reason to fail: the raw step still reaches the log line.""" + engine = FakeStatusEngine() + activity = build_activity(engine) + + result = await activity.report_import_pipeline_failure( + {**claim_input(), 'step': 'provisioning', 'detail': 'RuntimeError: boom'} + ) + + assert result == {'reported': True} + _, fields = activity.logger.custom_error.call_args_list[0].args + assert fields['import_step'] == 'provisioning' + assert fields['import_code'] == ImportErrorCode.UNEXPECTED_ERROR.value + assert fields['error_type'] == 'unavailable' + + @mark.asyncio async def test_the_engine_is_disposed_on_shutdown(import_activity: ModelImport, engine): import_activity.close() diff --git a/tests/laborious/test_import_absence.py b/tests/laborious/test_import_absence.py index b3a6dce..686d9c5 100644 --- a/tests/laborious/test_import_absence.py +++ b/tests/laborious/test_import_absence.py @@ -19,7 +19,7 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[2] WORKFLOW_MODULE = REPOSITORY_ROOT / 'laborious/workflows/import_model.py' ACTIVITIES_MODULE = REPOSITORY_ROOT / 'laborious/activities/model_import.py' -ERRORS_MODULE = REPOSITORY_ROOT / 'laborious/activities/model_import_errors.py' +ERRORS_MODULE = REPOSITORY_ROOT / 'laborious/utils/models/import_errors.py' READER_MODULE = REPOSITORY_ROOT / 'laborious/utils/bundle/reader.py' FORMAT_MODULE = REPOSITORY_ROOT / 'laborious/utils/bundle/format.py' diff --git a/tests/laborious/activities/test_model_import_errors.py b/tests/laborious/utils/models/test_import_errors.py similarity index 68% rename from tests/laborious/activities/test_model_import_errors.py rename to tests/laborious/utils/models/test_import_errors.py index 910563b..ee949b7 100644 --- a/tests/laborious/activities/test_model_import_errors.py +++ b/tests/laborious/utils/models/test_import_errors.py @@ -16,9 +16,11 @@ from pathlib import Path from typing import Any from pytest import mark, raises +from temporalio.exceptions import ApplicationError from laborious import metrics -from laborious.activities.model_import_errors import ( +from laborious.utils.bundle.steps import GATE_TO_STEP, ImportStep +from laborious.utils.models.import_errors import ( IMPORT_FAILURE_CATALOG, STEPS_WITHOUT_A_USER_FACING_REASON, UNCLASSIFIED_FAILURE, @@ -28,12 +30,13 @@ from laborious.activities.model_import_errors import ( ImportLogRowNotClaimableError, ImportStatusRejectedError, classify_import_failure, + describe_failure_chain, failure_location, report_import_failure, + summarize_failure, ) -from laborious.utils.bundle.steps import GATE_TO_STEP, ImportStep -REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +REPOSITORY_ROOT = Path(__file__).resolve().parents[4] # The modules that make up the import path. The channel lives in the third one; the other two may # not report a failure of their own. @@ -359,6 +362,180 @@ async def test_an_explicit_code_keeps_the_step_s_sentence(): assert sinks.notifications[0]['level'] == 'ERROR' +# --------------------------------------------- the fields the two log lines carry +# +# The message text is for a person reading a log; these fields are for the query that finds it. Both +# lines the channel produces — the technical one and the user-facing sentence the notification logs — +# carry the same ones, which is what makes `import_run_id=30` return the pair. + + +def _wrapped_activity_failure() -> BaseException: + """What an activity failure looks like by the time the channel sees it. + + A `MlflowException` raised inside an activity reaches the workflow as an `ApplicationError` whose + `type` still names it, wrapped in the error Temporal raises for the failed activity. Built by hand + rather than through a real activity because what is under test is the unwrapping. + """ + root = ApplicationError( + 'API request to http://localhost:5080/api/2.0/mlflow/experiments/get-by-name failed with ' + 'exception HTTPConnectionPool(host=localhost, port=5080): Connection refused', + type='MlflowException', + ) + wrapper = RuntimeError('Activity task failed') + wrapper.__cause__ = root + return wrapper + + +@mark.asyncio +async def test_the_log_line_carries_the_ids_the_step_and_the_code_as_fields(): + sinks = FakeSinks() + + await report_import_failure( + sinks, + ImportStep.EXPERIMENT_CREATION, + _wrapped_activity_failure(), + import_run_id=30, + workflow_id='manual-import-30', + metadata={'model_name': '-', 'model_id': '30'}, + ) + + _, fields = sinks.logged[0] + assert fields['import_run_id'] == 30 + assert fields['import_workflow_id'] == 'manual-import-30' + assert fields['import_step'] == ImportStep.EXPERIMENT_CREATION.value + assert fields['import_code'] == ImportErrorCode.STORAGE_PREPARATION_FAILED.value + # The workflow metadata every other line already carries is kept, not replaced. + assert fields['model_id'] == '30' + + +@mark.asyncio +async def test_the_fields_name_the_root_cause_not_the_wrapper(): + """`error_type=ApplicationError` would be true of every activity failure there is.""" + sinks = FakeSinks() + + await report_import_failure( + sinks, + ImportStep.EXPERIMENT_CREATION, + _wrapped_activity_failure(), + import_run_id=30, + workflow_id='manual-import-30', + ) + + _, fields = sinks.logged[0] + assert fields['error_type'] == 'MlflowException' + assert 'Connection refused' in fields['error_summary'] + assert '\n' not in fields['error_summary'] + + +@mark.asyncio +async def test_the_user_facing_line_is_greppable_by_the_same_fields(): + """The sentence a person reads said nothing technical, and nothing said where to look.""" + sinks = FakeSinks() + + await report_import_failure( + sinks, + None, + ImportLogRowNotClaimableError('row 29 is in status IMPORT_ERROR', import_run_id=29), + import_run_id=29, + workflow_id='manual-import-29', + code=ImportErrorCode.LOG_ROW_NOT_CLAIMABLE, + marker='not_claimable', + ) + + notified = sinks.notifications[0] + assert notified['message'] == UNCLASSIFIED_FAILURE[1] + assert notified['metadata']['import_code'] == ImportErrorCode.LOG_ROW_NOT_CLAIMABLE.value + assert notified['metadata']['import_step'] == 'not_claimable' + assert notified['metadata']['import_run_id'] == 29 + assert notified['metadata']['error_type'] == 'ImportLogRowNotClaimableError' + assert notified['metadata'] == sinks.logged[0][1] + + +@mark.asyncio +async def test_the_gate_and_the_unrecorded_verdict_are_fields_when_there_are_any(): + sinks = FakeSinks() + + await report_import_failure( + sinks, + ImportStep.DECRYPTION, + ImportGateRejectedError('gate 3 rejected the bundle', step=ImportStep.DECRYPTION, gate=3), + import_run_id=77, + workflow_id='wf', + gate=3, + verdict='succeeded=False step=decryption gate=3', + ) + + _, fields = sinks.logged[0] + assert fields['import_gate'] == 3 + assert fields['import_verdict_not_recorded'] == 'succeeded=False step=decryption gate=3' + + +@mark.asyncio +async def test_a_failure_with_neither_gate_nor_verdict_carries_neither_field(): + """An empty field is noise in every line that has nothing to say in it.""" + sinks = FakeSinks() + + await report_import_failure( + sinks, ImportStep.DOWNLOAD, RuntimeError('boom'), import_run_id=77, workflow_id='wf' + ) + + _, fields = sinks.logged[0] + assert 'import_gate' not in fields + assert 'import_verdict_not_recorded' not in fields + + +@mark.asyncio +async def test_no_field_can_shadow_the_formatter_s_own_keys(): + sinks = FakeSinks() + + await report_import_failure( + sinks, ImportStep.DOWNLOAD, RuntimeError('boom'), import_run_id=77, workflow_id='wf' + ) + + _, fields = sinks.logged[0] + added = set(fields) - {'model_name', 'model_id', 'workflow_name', 'schedule_name'} + assert all(key.startswith(('import_', 'error_')) for key in added) + assert not {'at', 'msg', 'timestamp'} & set(fields) + + +def test_the_chain_is_flattened_root_cause_first(): + """Because the first line is what becomes `error_summary`, and it has to be the cause.""" + kind, detail = describe_failure_chain(_wrapped_activity_failure()) + + assert kind == 'MlflowException' + assert detail.splitlines()[0].startswith('MlflowException: API request to') + assert detail.splitlines()[-1] == 'RuntimeError: Activity task failed' + + +def test_a_cycle_in_the_chain_cannot_hang_the_reporting_of_a_failure(): + first = RuntimeError('first') + second = RuntimeError('second') + first.__cause__ = second + second.__cause__ = first + + _, detail = describe_failure_chain(first) + + assert len(detail.splitlines()) <= 10 + + +def test_a_failure_that_arrives_as_text_keeps_the_type_that_travelled_with_it(): + kind, summary = summarize_failure( + None, + 'MlflowException: connection refused\nRuntimeError: Activity task failed', + 'MlflowException', + ) + + assert kind == 'MlflowException' + assert summary == 'MlflowException: connection refused' + + +def test_a_failure_that_arrives_as_nothing_says_so_rather_than_guessing(): + kind, summary = summarize_failure(None, None, None) + + assert kind == 'unavailable' + assert summary == '' + + # ------------------------------------------------- the channel is the only reporter @@ -392,7 +569,7 @@ def test_no_import_module_reports_a_failure_of_its_own(module_path: Path): def test_the_channel_itself_is_where_the_reporting_lives(): - channel = REPOSITORY_ROOT / 'laborious/activities/model_import_errors.py' + channel = REPOSITORY_ROOT / 'laborious/utils/models/import_errors.py' calls = _reporting_calls(channel) diff --git a/tests/laborious/worker/test_runtime_task_queues.py b/tests/laborious/worker/test_runtime_task_queues.py index 37a46cd..3dff897 100644 --- a/tests/laborious/worker/test_runtime_task_queues.py +++ b/tests/laborious/worker/test_runtime_task_queues.py @@ -109,6 +109,6 @@ def test_the_import_worker_registers_only_the_import_activities(): if isinstance(inner, ast.keyword) and inner.arg == 'other_workflows': assert ast.unparse(inner.value) == '[]' - assert len(activities) == 13 + assert len(activities) == 14 assert all(name.startswith('activities.') for name in activities) assert all('import' in name for name in activities) diff --git a/tests/laborious/workflows/test_import_model.py b/tests/laborious/workflows/test_import_model.py index 0f426d3..6c67ae6 100644 --- a/tests/laborious/workflows/test_import_model.py +++ b/tests/laborious/workflows/test_import_model.py @@ -15,18 +15,19 @@ from unittest.mock import MagicMock, patch from pytest import fixture, mark, raises from sientia_do.temporal.policies import retry_policy +from temporalio.exceptions import ApplicationError from laborious.activities.activities import Activities from laborious.activities.model_import import ( IMPORT_STATUS_HINT_RETRY_POLICY, IMPORT_STATUS_RETRY_POLICY, ) -from laborious.activities.model_import_errors import ( +from laborious.utils.bundle.steps import ImportStep +from laborious.utils.models.import_errors import ( ImportGateRejectedError, ImportInputError, ImportLogRowNotClaimableError, ) -from laborious.utils.bundle.steps import ImportStep from laborious.workflows.import_model import ( REJECTED_INPUT_FIELDS, STATUS_WRITE_ENVELOPE, @@ -224,6 +225,24 @@ async def test_the_names_are_recorded_before_mlflow_is_touched( assert names['run_name'] == 'import-factory_model-v2' +@mark.asyncio +async def test_the_model_is_named_from_the_bundles_model_name( + import_workflow: ImportModel, workflow_env +): + """`metadata.model_name` names the model, verbatim, in both destinations. + + The registered model and the model document take the bundle's own `model_name` and nothing else: + no other metadata field substitutes for it, and nothing is appended to it. + """ + await import_workflow.run(valid_input()) + + registration = workflow_env.recorder.payload_of(Activities.register_import_model_version) + document = workflow_env.recorder.payload_of(Activities.write_import_model_document) + assert registration['model_name'] == BUNDLE_METADATA['model_name'] + assert document['model_name'] == BUNDLE_METADATA['model_name'] + assert document['model_name'] == registration['model_name'] + + @mark.asyncio async def test_the_run_is_created_with_exactly_the_recorded_name( import_workflow: ImportModel, workflow_env @@ -660,6 +679,168 @@ async def test_reporting_the_report_is_where_it_stops(import_workflow: ImportMod assert result['run_id'] == RUN_ID +# --------------------------------------------------------------- the failure report + + +def activity_failure(message: str, kind: str) -> BaseException: + """What a failed activity looks like from inside the workflow. + + The activity's own exception type survives the boundary in `ApplicationError.type` and nowhere + else; the outer error is the one Temporal raises for the failed activity. + """ + wrapper = RuntimeError('Activity task failed') + wrapper.__cause__ = ApplicationError(message, type=kind) + return wrapper + + +@mark.asyncio +async def test_the_failure_that_ends_an_import_is_reported_through_the_channel( + import_workflow: ImportModel, +): + """The gap this closes: a failure at any pipeline step used to reach the structured logs as + nothing at all — only the SDK's own traceback for the attempt, with no id and no step.""" + recorder = Recorder( + failures={ + Activities.create_import_experiment: activity_failure( + 'API request to http://localhost:5080/api/2.0/mlflow/experiments/get-by-name ' + 'failed with exception Connection refused', + 'MlflowException', + ) + } + ) + patcher, _ = run_with(recorder) + try: + with raises(RuntimeError, match='Activity task failed'): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + reported = recorder.payload_of(Activities.report_import_pipeline_failure) + assert reported['step'] == ImportStep.EXPERIMENT_CREATION.value + assert reported['error_type'] == 'MlflowException' + assert reported['detail'].splitlines()[0].startswith('MlflowException: API request to') + assert reported['import_run_id'] == 77 + assert reported['workflow_id'] == WORKFLOW_ID + + +@mark.asyncio +async def test_the_report_comes_before_the_cleanup_and_the_terminal_write( + import_workflow: ImportModel, +): + """So the log reads in the order things happened, and the row's step matches the line's.""" + recorder = Recorder( + failures={Activities.upload_import_artifacts: activity_failure('bucket gone', 'S3Error')} + ) + patcher, _ = run_with(recorder) + try: + with raises(RuntimeError): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + order = recorder.order + assert ( + order.index(Activities.report_import_pipeline_failure) + < order.index(Activities.cleanup_import_files) + < order.index(Activities.record_import_terminal_status) + ) + reported = recorder.payload_of(Activities.report_import_pipeline_failure) + terminal = recorder.payload_of(Activities.record_import_terminal_status) + assert reported['step'] == terminal['step'] == ImportStep.ARTIFACT_UPLOAD.value + + +@mark.asyncio +async def test_the_gate_that_rejected_the_bundle_travels_into_the_report( + import_workflow: ImportModel, +): + recorder = Recorder( + failures={ + Activities.open_import_bundle: ImportGateRejectedError( + 'gate 3 rejected the bundle', step=ImportStep.DECRYPTION, gate=3 + ) + } + ) + patcher, _ = run_with(recorder) + try: + with raises(ImportGateRejectedError): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + reported = recorder.payload_of(Activities.report_import_pipeline_failure) + assert reported['step'] == ImportStep.DECRYPTION.value + assert reported['gate'] == 3 + + +@mark.asyncio +async def test_a_claim_that_matches_nothing_is_not_reported_twice(import_workflow: ImportModel): + """The claim reports itself, and it runs outside the try — so the channel is hit once.""" + recorder = Recorder( + failures={ + Activities.claim_import_status: ImportLogRowNotClaimableError( + 'row 29 could not be claimed', import_run_id=29 + ) + } + ) + patcher, _ = run_with(recorder) + try: + with raises(ImportLogRowNotClaimableError): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + assert Activities.report_import_pipeline_failure not in recorder.order + + +@mark.asyncio +async def test_a_report_that_cannot_be_made_does_not_mask_the_failure( + import_workflow: ImportModel, +): + recorder = Recorder( + failures={ + Activities.write_import_model_document: activity_failure('mongo down', 'PyMongoError'), + Activities.report_import_pipeline_failure: RuntimeError('no channel'), + } + ) + patcher, _ = run_with(recorder) + try: + with raises(RuntimeError, match='Activity task failed'): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + terminal = recorder.payload_of(Activities.record_import_terminal_status) + assert terminal['succeeded'] is False + assert terminal['step'] == ImportStep.MODEL_DOCUMENT.value + + +@mark.asyncio +async def test_a_successful_import_reports_no_failure(import_workflow: ImportModel, workflow_env): + await import_workflow.run(valid_input()) + + assert Activities.report_import_pipeline_failure not in workflow_env.recorder.order + + +@mark.asyncio +async def test_the_report_gives_up_inside_the_short_bound(import_workflow: ImportModel): + """A failure already happened; spending 151 seconds telling the log about it would be worse.""" + recorder = Recorder( + failures={ + Activities.create_import_experiment: activity_failure('refused', 'MlflowException') + } + ) + patcher, _ = run_with(recorder) + try: + with raises(RuntimeError): + await import_workflow.run(valid_input()) + finally: + patcher.stop() + + call = recorder.calls_to(Activities.report_import_pipeline_failure)[0] + assert call.kwargs['retry_policy'] is IMPORT_STATUS_HINT_RETRY_POLICY + assert call.kwargs['start_to_close_timeout'] == STATUS_WRITE_TIMEOUT + + # ------------------------------------------------------------------- the retry split From dfb9deae1a0b11dcae0acdc4edfe04f5e74675d7 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 27 Aug 2026 14:58:43 -0300 Subject: [PATCH 29/64] QTZPOC-20 QTZPOC-20: Remove PyTorch from supported model flavors as it was never functional due to missing runtime dependencies. --- README.md | 4 +- docs/model-flavors.md | 6 +- e2e/conftest.py | 4 - e2e/scenarios.md | 2 +- e2e/test_predictions_batch_joblib_flavor.py | 2 +- .../utils/repository/model_repository.py | 28 +++--- .../utils/repository/test_model_repository.py | 97 ++++++++++--------- 7 files changed, 70 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 48c75d6..d889abd 100644 --- a/README.md +++ b/README.md @@ -1109,8 +1109,8 @@ Legacy MinIO object layout (relative key): ## Model Flavors -`model_config.predict_flavor` / `model_config.transform_flavor` accept `sklearn`, `pyfunc`, -`pytorch` and `joblib`. The flavor is configuration, never discovered from the artifact, and there is +`model_config.predict_flavor` / `model_config.transform_flavor` accept `sklearn`, `pyfunc` and +`joblib`. The flavor is configuration, never discovered from the artifact, and there is no fallback between flavors: a failing loader propagates. `joblib` reads the artifact directly — download, first top-level `.pkl`, `joblib.load` — with the diff --git a/docs/model-flavors.md b/docs/model-flavors.md index a8db934..82c0f60 100644 --- a/docs/model-flavors.md +++ b/docs/model-flavors.md @@ -24,11 +24,13 @@ reads the artifact's `MLmodel` manifest to guess it. |---|---|---|---| | `sklearn` | yes | yes | `mlflow.sklearn.load_model()` | | `pyfunc` | yes | yes | `mlflow.pyfunc.load_model()`; the wrapper path also unwraps `._model_impl.python_model` | -| `pytorch` | yes | yes | `mlflow.pytorch.load_model()` | | `joblib` | yes | yes | downloads the artifact and `joblib.load`s its pickle — **no MLflow flavor loader involved** | Anything else raises `ValueError` with `INVALID_FLAVOR_MESSAGE`, **before** any artifact is resolved -or downloaded. The same message and the same four values govern reading and writing. +or downloaded. The same message and the same three values govern reading and writing. + +`pytorch` was accepted until this change and is now rejected like any other unknown value: `torch` is +not installed in the runtime, so `mlflow.pytorch.load_model` could never have loaded anything there. **There is no fallback between flavors.** A failing loader propagates: one attempt, one cause. A model configured with the wrong flavor fails visibly instead of being rescued silently. diff --git a/e2e/conftest.py b/e2e/conftest.py index 51af260..29076b3 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -546,10 +546,6 @@ def patch_mlflow(mock_mlflow_models, joblib_flavor_artifacts): mock_mlflow.pyfunc = MagicMock() mock_mlflow.pyfunc.load_model = MagicMock(side_effect=mock_pyfunc_load_model) - # Mock pytorch.load_model - mock_mlflow.pytorch = MagicMock() - mock_mlflow.pytorch.load_model = MagicMock(return_value=mock_mlflow_models['predict_model']) - # Mock other mlflow methods that might be called mock_mlflow.set_tracking_uri = MagicMock() mock_mlflow.get_run = MagicMock( diff --git a/e2e/scenarios.md b/e2e/scenarios.md index 71abd5c..1654303 100644 --- a/e2e/scenarios.md +++ b/e2e/scenarios.md @@ -1124,7 +1124,7 @@ restores `sys.path` in teardown, because the flavor deliberately never removes w **Assertions**: - Exactly one prediction row, `prediction_status='Good'`, value matching the artifact's model - `download_artifacts` called once, with `'prediction_model'` -- No `mlflow.{sklearn,pyfunc,pytorch}.load_model` call names this model +- No `mlflow.{sklearn,pyfunc}.load_model` call names this model --- diff --git a/e2e/test_predictions_batch_joblib_flavor.py b/e2e/test_predictions_batch_joblib_flavor.py index 87b6f9d..0993e8f 100644 --- a/e2e/test_predictions_batch_joblib_flavor.py +++ b/e2e/test_predictions_batch_joblib_flavor.py @@ -26,7 +26,7 @@ from laborious.workflows.predictions_batch import PredictionsBatch base_query = 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = {model_id}' -FLAVOR_LOADERS = ('sklearn', 'pyfunc', 'pytorch') +FLAVOR_LOADERS = ('sklearn', 'pyfunc') def build_input_data( diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 500622f..54e3391 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -45,7 +45,7 @@ ARTIFACTS_PATH = './tmp/artifacts' TRANSFORMED_COMPRESSED_PATH = 'artifacts/training_transformer.pkl' PREDICTION_COMPRESSED_PATH = 'artifacts/stacking_model.pkl' -INVALID_FLAVOR_MESSAGE = "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'pytorch' or 'joblib'." +INVALID_FLAVOR_MESSAGE = "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'joblib'." def force_memory_release(logger: Logger): @@ -550,7 +550,7 @@ class MLFlowRepository(SientiaMonitoring): Args: model_name (str): The name of the model to download from the registry. metadata (dict[str, Any]): Metadata used for structured logging. - flavor (str): Model flavor ('pyfunc', 'sklearn', 'pytorch', 'joblib') + flavor (str): Model flavor ('pyfunc', 'sklearn', 'joblib') artifact_path (str | None): Path to compressed artifacts if model is compressed Returns: @@ -564,7 +564,7 @@ class MLFlowRepository(SientiaMonitoring): `joblib.load`, which needs no `MLmodel` manifest. The artifact's `code` directory, when present, is put on `sys.path` first. """ - if flavor not in ('pyfunc', 'sklearn', 'pytorch', 'joblib'): + if flavor not in ('pyfunc', 'sklearn', 'joblib'): raise ValueError(INVALID_FLAVOR_MESSAGE) if flavor == 'joblib': @@ -584,10 +584,8 @@ class MLFlowRepository(SientiaMonitoring): try: if flavor == 'pyfunc': model = mlflow.pyfunc.load_model(model_uri) - elif flavor == 'sklearn': - model = mlflow.sklearn.load_model(model_uri) else: - model = mlflow.pytorch.load_model(model_uri) + model = mlflow.sklearn.load_model(model_uri) except Exception as e: await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels) raise e @@ -608,7 +606,7 @@ class MLFlowRepository(SientiaMonitoring): Args: model_name (str): The name of the model to download. metadata (dict[str, Any]): Metadata used for structured logging. - flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch', 'joblib') + flavor (str): Model flavor ('sklearn', 'pyfunc', 'joblib') artifact_path (str | None): Path to compressed artifacts if model is compressed Returns: @@ -626,7 +624,7 @@ class MLFlowRepository(SientiaMonitoring): directory, when present, is put on `sys.path` first. """ - if flavor not in ('sklearn', 'pyfunc', 'pytorch', 'joblib'): + if flavor not in ('sklearn', 'pyfunc', 'joblib'): raise ValueError(INVALID_FLAVOR_MESSAGE) latest_production_id = self.get_model_run_id(model_name=model_name, stage='Production') @@ -650,10 +648,8 @@ class MLFlowRepository(SientiaMonitoring): try: if flavor == 'sklearn': model = mlflow.sklearn.load_model(model_uri) - elif flavor == 'pyfunc': - model = mlflow.pyfunc.load_model(model_uri) else: - model = mlflow.pytorch.load_model(model_uri) + model = mlflow.pyfunc.load_model(model_uri) except Exception as e: await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels) raise e @@ -677,7 +673,7 @@ class MLFlowRepository(SientiaMonitoring): model_name (str): Name of the model to download metadata (dict[str, Any]): Metadata used for structured logging. model_type (str): Type of model ('predict' or 'transform') - flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch', 'joblib') + flavor (str): Model flavor ('sklearn', 'pyfunc', 'joblib') load_wrapper (bool): Whether to load wrapper Returns: @@ -862,7 +858,7 @@ class MLFlowRepository(SientiaMonitoring): metadata (dict[str, Any]): Metadata used for structured logging. retention (int): Cache retention time in minutes (0 = no cache). model_type (str): Type of model ('predict' or 'transform') - flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch') + flavor (str): Model flavor ('sklearn', 'pyfunc', 'joblib') Returns: Any: Model object. @@ -950,7 +946,7 @@ class MLFlowRepository(SientiaMonitoring): model_name (str): Registered model name. data (pd.DataFrame): Input data. retention (int): Cache retention in minutes. - flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch'). + flavor (str): Model flavor ('sklearn', 'pyfunc', 'joblib'). metadata (dict[str, Any]): Metadata used for structured logging. Returns: pd.DataFrame | ndarray: Operation result. @@ -1228,7 +1224,7 @@ class MLFlowRepository(SientiaMonitoring): Args: model_data (dict): Model holder with keys 'model' and optional 'artifact_path'. - flavor (str): Model flavor ('sklearn', 'pyfunc', 'pytorch', 'joblib'). + flavor (str): Model flavor ('sklearn', 'pyfunc', 'joblib'). model_type (str): Artifact name, e.g., 'prediction_model' or 'data_model'. metadata (dict): Metadata for structured logging. @@ -1257,8 +1253,6 @@ class MLFlowRepository(SientiaMonitoring): model.store_model(artifact_path=model_type, code_path=code_path, to_disk=False) self.debug('Model uploaded successfully', metadata) - elif flavor == 'pytorch': - mlflow.pytorch.log_model(model, model_type) elif flavor == 'joblib': with tempfile.TemporaryDirectory() as tmp_dir: pkl_path = path.join(tmp_dir, 'model.pkl') diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 2c449de..02e8cd5 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -455,17 +455,6 @@ async def test_load_predict_model_pyfunc(mlflow, mlflow_repository): ) -@pytest.mark.asyncio -async def test_load_predict_model_pytorch(mlflow, mlflow_repository): - result = await mlflow_repository.load_predict_model('test_model', {}, 'pytorch') - assert result == mlflow.pytorch.load_model.return_value - mlflow.pytorch.load_model.assert_called_once_with('models:/test_model/production') - mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_READ_LAG, ANY) - mlflow_repository.emit_metric.assert_called_once_with( - metric_object=metrics.MODEL_READ_COUNT, tags=ANY - ) - - @pytest.mark.asyncio async def test_load_predict_model_error(mlflow_repository): with pytest.raises(ValueError) as e: @@ -523,23 +512,6 @@ async def test_load_transform_model_pyfunc(mlflow, mlflow_repository): ) -@pytest.mark.asyncio -async def test_load_transform_model_pytorch(mlflow, mlflow_repository): - mlflow_repository.get_model_run_id = MagicMock() - mlflow_repository.get_model_uri = MagicMock() - - result = await mlflow_repository.load_transform_model('test_model', {}, 'pytorch') - validate_common_load_transform_model_mocks(mlflow_repository, 'test_model') - - assert result == mlflow.pytorch.load_model.return_value - mlflow.pytorch.load_model.assert_called_once_with(mlflow_repository.get_model_uri.return_value) - - mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_READ_LAG, ANY) - mlflow_repository.emit_metric.assert_called_once_with( - metric_object=metrics.MODEL_READ_COUNT, tags=ANY - ) - - @pytest.mark.asyncio async def test_load_transform_model_error(mlflow_repository): mlflow_repository.get_model_run_id = MagicMock() @@ -1108,20 +1080,6 @@ async def test_log_model_pyfunc(tmp_path, mlflow, mlflow_repository): ) -@pytest.mark.asyncio -async def test_log_model_pytorch(mlflow, mlflow_repository): - model_data = {'model': MagicMock(), 'artifact_path': 'artifact_path'} - await mlflow_repository.log_model( - model_data, 'pytorch', 'prediction_model', metadata['metadata'] - ) - mlflow.pytorch.log_model.assert_called_once_with(model_data['model'], 'prediction_model') - - mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_WRITE_LAG, ANY) - mlflow_repository.emit_metric.assert_called_once_with( - metric_object=metrics.MODEL_WRITE_COUNT, tags=ANY - ) - - @pytest.mark.asyncio async def test_log_model_error(mlflow_repository): model_data = {'model': MagicMock(), 'artifact_path': 'artifact_path'} @@ -1777,7 +1735,7 @@ Tests for the joblib flavor on the model load and write paths """ JOBLIB_PATH = 'laborious.utils.repository.model_repository.joblib' -FLAVOR_LOADERS = ('sklearn', 'pyfunc', 'pytorch') +FLAVOR_LOADERS = ('sklearn', 'pyfunc') @pytest.fixture @@ -1889,7 +1847,7 @@ async def test_load_transform_model_joblib_resolves_run_once( @patch(JOBLIB_PATH) @pytest.mark.asyncio -@pytest.mark.parametrize('flavor', ['sklearn', 'pyfunc', 'pytorch']) +@pytest.mark.parametrize('flavor', ['sklearn', 'pyfunc']) async def test_load_predict_model_flavor_failure_propagates( joblib_mock, mlflow, mlflow_repository, flavor ): @@ -1910,7 +1868,7 @@ async def test_load_predict_model_flavor_failure_propagates( @patch(JOBLIB_PATH) @pytest.mark.asyncio -@pytest.mark.parametrize('flavor', ['sklearn', 'pyfunc', 'pytorch']) +@pytest.mark.parametrize('flavor', ['sklearn', 'pyfunc']) async def test_load_transform_model_flavor_failure_propagates( joblib_mock, mlflow, mlflow_repository, flavor ): @@ -2361,10 +2319,57 @@ async def test_log_model_invalid_flavor(joblib_mock, mlflow, mlflow_repository): mlflow_repository.observe_lag.assert_not_called() +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_predict_model_rejects_pytorch(joblib_mock, mlflow, mlflow_repository): + with pytest.raises(ValueError) as exc_info: + await mlflow_repository.load_predict_model('test_model', {}, 'pytorch') + + assert str(exc_info.value) == INVALID_FLAVOR_MESSAGE + mlflow.pytorch.load_model.assert_not_called() + joblib_mock.load.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + mlflow_repository.emit_metric.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_load_transform_model_rejects_pytorch(joblib_mock, mlflow, mlflow_repository): + with pytest.raises(ValueError) as exc_info: + await mlflow_repository.load_transform_model('test_model', {}, 'pytorch') + + assert str(exc_info.value) == INVALID_FLAVOR_MESSAGE + mlflow.pytorch.load_model.assert_not_called() + joblib_mock.load.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + mlflow_repository.emit_metric.assert_not_called() + + +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_log_model_rejects_pytorch(joblib_mock, mlflow, mlflow_repository): + model_data = {'model': MagicMock(), 'artifact_path': 'artifact_path'} + + with pytest.raises(ValueError) as exc_info: + await mlflow_repository.log_model( + model_data, 'pytorch', 'prediction_model', metadata['metadata'] + ) + + assert str(exc_info.value) == INVALID_FLAVOR_MESSAGE + mlflow.pytorch.log_model.assert_not_called() + joblib_mock.dump.assert_not_called() + mlflow_repository.emit_metric.assert_called_once_with( + metric_object=metrics.MODEL_WRITE_ERROR_COUNT, tags=ANY + ) + mlflow_repository.observe_lag.assert_not_called() + + def test_invalid_flavor_message_lists_every_accepted_flavor(): - for flavor in ('sklearn', 'pyfunc', 'pytorch', 'joblib'): + for flavor in ('sklearn', 'pyfunc', 'joblib'): assert f"'{flavor}'" in INVALID_FLAVOR_MESSAGE + assert 'pytorch' not in INVALID_FLAVOR_MESSAGE + @patch(JOBLIB_PATH) @pytest.mark.asyncio From c1a44a9d7a87037108dfc271a2402fe481617d47 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 28 Aug 2026 15:10:41 -0300 Subject: [PATCH 30/64] QTZPOC-20 QTZPOC-20: Unify artifact code directory handling for all model flavors and standardize joblib pickle path Implemented a robust mechanism to ensure an artifact's code/ directory is prepended to sys.path before loading for sklearn, pyfunc, and joblib flavors. sklearn and pyfunc now resolve MLflow URIs to local directories first to facilitate this sys.path modification. Standardized the joblib flavor to explicitly read and write model.pkl within the artifact directory. Updated documentation and added comprehensive tests to reflect these changes. --- README.md | 14 +- docs/model-flavors.md | 158 +++++++----- docs/sientia-bundle-format.md | 224 +++++------------- e2e/conftest.py | 9 + .../utils/repository/model_repository.py | 182 +++++++++++--- .../utils/repository/test_model_repository.py | 161 ++++++++++++- 6 files changed, 475 insertions(+), 273 deletions(-) diff --git a/README.md b/README.md index d889abd..f61454e 100644 --- a/README.md +++ b/README.md @@ -1003,10 +1003,7 @@ See [OPC UA Communication](#opc-ua-communication) for semantics, concurrency, an | `IMPORT_MAX_COMPRESSION_RATIO` | Gate 4 compression ratio ceiling | `200` | No | | `IMPORT_MODELS_COLLECTION` | MongoDB collection holding the model listing | `models` | No | | `IMPORT_STATUS_DB_NAME` | Import log database name (the BFF's, on the `POSTGRES_*` server) | `sientia-core-mlops-bff` | If importing | -| `IMPORT_STATUS_DB_HOST` | Override: import log host, when it is not the `POSTGRES_*` server | `POSTGRES_HOST` | No | -| `IMPORT_STATUS_DB_PORT` | Override: import log port | `POSTGRES_PORT` | No | -| `IMPORT_STATUS_DB_USER` | Override: import log user | `POSTGRES_USER` | No | -| `IMPORT_STATUS_DB_PASSWORD` | Override: import log password | `POSTGRES_PASSWORD` | No | +| `IMPORT_STATUS_DB_{HOST,PORT,USER,PASSWORD}` | Override, only when the import log is not on the `POSTGRES_*` server | the `POSTGRES_*` value | No | | `OPC_CONFIG` | OPC server configuration (JSON) | `{}` | No | | `OPC_ID` | OPC server identifier | `1` | No | | `OPC_URL` | OPC server URL | `opc.tcp://localhost:4840` | No | @@ -1113,9 +1110,12 @@ Legacy MinIO object layout (relative key): `joblib`. The flavor is configuration, never discovered from the artifact, and there is no fallback between flavors: a failing loader propagates. -`joblib` reads the artifact directly — download, first top-level `.pkl`, `joblib.load` — with the -artifact's `code/` put on `sys.path` first, and it writes on retrain (`model.pkl` + `code/`, no -`MLmodel`, so only this runtime reads it back). +`joblib` reads the artifact directly — download, `/model.pkl`, `joblib.load` — and writes on +retrain (`model.pkl` + `code/`, no `MLmodel`, so only this runtime reads it back). + +Every flavor puts the artifact's own `code/` on `sys.path` before loading, **when that directory +exists**: MLflow's loaders only do it when the `MLmodel` manifest declares `code`, so `sklearn` and +`pyfunc` resolve the URI to a local directory first and load from there. Full reference, including how to switch an existing model over: **[docs/model-flavors.md](docs/model-flavors.md)**. diff --git a/docs/model-flavors.md b/docs/model-flavors.md index 82c0f60..74c0d72 100644 --- a/docs/model-flavors.md +++ b/docs/model-flavors.md @@ -22,9 +22,9 @@ reads the artifact's `MLmodel` manifest to guess it. | Flavor | Read | Write (retrain) | How it loads | |---|---|---|---| -| `sklearn` | yes | yes | `mlflow.sklearn.load_model()` | -| `pyfunc` | yes | yes | `mlflow.pyfunc.load_model()`; the wrapper path also unwraps `._model_impl.python_model` | -| `joblib` | yes | yes | downloads the artifact and `joblib.load`s its pickle — **no MLflow flavor loader involved** | +| `sklearn` | yes | yes | resolves the URI to a local directory, then `mlflow.sklearn.load_model()` | +| `pyfunc` | yes | yes | resolves the URI to a local directory, then `mlflow.pyfunc.load_model()`; the wrapper path also unwraps `._model_impl.python_model` | +| `joblib` | yes | yes | downloads the same artifact and `joblib.load`s its `/model.pkl` — **no MLflow flavor loader involved** | Anything else raises `ValueError` with `INVALID_FLAVOR_MESSAGE`, **before** any artifact is resolved or downloaded. The same message and the same three values govern reading and writing. @@ -35,80 +35,108 @@ not installed in the runtime, so `mlflow.pytorch.load_model` could never have lo **There is no fallback between flavors.** A failing loader propagates: one attempt, one cause. A model configured with the wrong flavor fails visibly instead of being rescued silently. -## The `joblib` flavor +## Same artifact, three readers -Some models the pipeline produces do not load through an MLflow flavor loader even though the -artifact is intact: the `.pkl` is in the compressed form `joblib.dump` writes, which MLflow's -`pickle.load` does not read. Declaring `joblib` is how such a model is served. +The flavor changes the reader, never the path. All three resolve the same two artifact directories of +the Production run: -**Reading.** Resolve the Production run, `client.download_artifacts(run_id, )`, take the -**first top-level `.pkl` in name order**, and `joblib.load` it. The listing does not recurse, so a -nested wrapper artifact (`artifacts/*.pkl`) is never deserialized in place of the model. No -`MLmodel` is parsed and no `models://production` URI is built. +``` +/artifacts/ +├── data_model/ ← transform_flavor loads this one +│ ├── MLmodel ← parsed by sklearn/pyfunc; ignored by joblib +│ ├── model.pkl ← what joblib.load reads +│ └── code/ ← put on sys.path by every flavor, when present +├── prediction_model/ ← predict_flavor loads this one +└── model_card/, *.csv +``` -**The artifact's `code/` goes on `sys.path` first.** A model whose class lives inside the artifact -would otherwise raise `ModuleNotFoundError` at unpickle time. Before `joblib.load`, the flavor -prepends `/code` to `sys.path` when that directory exists — the directory itself, matching -MLflow's own convention where `code/utils/…` imports as the package `utils`. The entry is not -duplicated and is **not** removed after the load, because the deserialized object may import -lazily. A missing `code/` is not an error, and no extra download is performed: `download_artifacts` -already brings `code/` along. +The run's `artifacts/` root holds **no** `.pkl` — every model pickle lives one level down, inside its +artifact directory. Predict resolves that directory through the registry URI +(`models://production`, which MLflow points at the same `prediction_model`); transform builds +the run artifact URI directly with `get_model_uri`, since the run is already resolved. Observability +is the same for the three: `MODEL_READ_*` on load, `MODEL_WRITE_*` on `log_model`, under the base +`operation_type` — no per-flavor metric, operation type or label. -> `sys.path` is process-global. Two different models that embed a package with the same name -> (`utils/`, typically) and are served by the same worker resolve to whichever entered first — the -> second silently runs the first one's code. This is pre-existing behaviour of MLflow's own pyfunc -> path with the same artifact layout; the mitigation, if it ever bites, is one worker per model. +## `code/` on `sys.path`, for every flavor -**Writing / retraining.** `joblib` is a write flavor too, so a joblib model retrains. `log_model` -writes `joblib.dump(model, 'model.pkl', compress=3)` under the artifact path and re-logs the source -artifact's `code/` to `/code`, so the retrained model stays deserializable. Registration -and promotion to `Production` are unchanged. +An artifact may ship the modules its own classes live in, under `/code/`. Without that +directory on `sys.path`, unpickling raises `ModuleNotFoundError` — the artifact is intact, the class +is simply not importable. -**The written artifact has no `MLmodel` manifest.** Only this runtime's `joblib` flavor reads it: -`mlflow.sklearn.load_model` and `models://production` both fail on it. A model retrained in -joblib **stays** joblib — `predict_flavor` needs no edit after a retrain, and no consumer outside -this runtime (the MLflow UI, MLflow serving, another service) recognizes that artifact as a model. +MLflow's loaders do add it, but **from the manifest**: `_add_code_from_conf_to_system_path` acts only +when the `MLmodel` flavor config declares a `code` entry. An artifact carrying `code/` on disk +without that entry — code logged with `log_artifacts`, a re-log that dropped the config, anything a +`joblib` retrain wrote — loads with no module path at all. So this runtime reads the **directory**, +which no manifest can misdescribe: -**What `joblib` is for.** A pickled object exposing `predict(data)` — and `fit(data)` if it is -retrained. It is **not** for a bare pyfunc `PythonModel` wrapper: `get_prediction_data` and -`get_cached_operation` call `predict(data)` for every non-pyfunc flavor, so a wrapper expecting -`predict(context, model_input)` raises `TypeError` at predict and retrain time. +- `prepend_artifact_code_dir` inserts `/code` at the front of `sys.path`, **only when that + directory exists**, and only once (an entry already present is not duplicated). +- Because the insert must happen *before* the loader unpickles, `sklearn` and `pyfunc` resolve their + URI to a local directory first (`mlflow.artifacts.download_artifacts`) and the loader is handed + that directory. Same bytes, same download count — MLflow would have downloaded it anyway. +- The wrapper path (`download_model(load_wrapper=True)`) already had its local directory and adds + the entry itself; `joblib` adds it inside `_load_model_pickle`. +- When the manifest *does* declare `code`, MLflow inserts the same directory a second time. Harmless. -**Observability.** A joblib load is an ordinary load: `MODEL_READ_LAG` / `MODEL_READ_COUNT` on -success and `MODEL_READ_ERROR_COUNT` on failure, under the base `operation_type` -(`load_predict_model` / `load_transform_model`). A joblib write emits the usual `MODEL_WRITE_*` -under `log_model`. No suffixed operation type, no new metric, no new label. +The entry is never removed: a loaded model may import lazily, long after the load returned. + +> `sys.path` is process-global: two models embedding a package with the same name (`utils/`) on the +> same worker resolve to whichever entered first. Mitigation, if it bites: one worker per model. + +## `sklearn` + +Default. A standard MLflow sklearn artifact: `mlflow.sklearn.log_model` writes `MLmodel` + +`model.pkl`, `mlflow.sklearn.load_model` reads it, and every outside consumer (MLflow UI, MLflow +serving) recognizes it as a model. + +## `pyfunc` + +For models logged as `mlflow.pyfunc`, including the wrapper models this pipeline produces. Retrain +writes through the wrapper's own `store_model(artifact_path=, code_path=…)`. Two read +paths, and the second is **orthogonal to the flavor**: + +- Plain: `mlflow.pyfunc.load_model()`, called as `predict(context, model_input)`. +- Wrapper (`download_model(load_wrapper=True)`): downloads to `./tmp/artifacts//` + via `dowload_artifacts`, loads as pyfunc, then unwraps `._model_impl.python_model`. Inside such an + artifact the real pickle is **nested** — `/artifacts/stacking_model.pkl` for predict, + `/artifacts/training_transformer.pkl` for transform (`PREDICTION_COMPRESSED_PATH` / + `TRANSFORMED_COMPRESSED_PATH`). That nesting is why the `joblib` listing does not recurse. + +## `joblib` + +For models whose `.pkl` is in the compressed form `joblib.dump` writes, which MLflow's `pickle.load` +does not read — the artifact is intact, but no flavor loader opens it. + +- **Read**: `client.download_artifacts(run_id, )`, then + `joblib.load(/model.pkl)`. `model.pkl` is `MODEL_PICKLE_NAME`, the same constant + `log_model` writes; another `.pkl` directly inside the artifact is a fallback, in name order. +- **Write**: `joblib.dump(model, '/model.pkl', compress=3)`, plus a re-log of the source + artifact's `code/` to `/code`. +- **`code/` goes on `sys.path`** before the load, when present — see the section above; the joblib + flavor is where that started, and it is now shared by all three. +- **No `MLmodel` is written**, so only this runtime reads the artifact back — + `mlflow.sklearn.load_model` and `models://production` both fail on it. A model retrained in + joblib stays joblib. +- **The pickled object must expose `predict(data)`**, plus `fit(data)` if retrained. A pyfunc + `PythonModel` wrapper expecting `predict(context, model_input)` raises `TypeError`. ## Switching a model to `joblib` -The runtime does not migrate documents. Editing the `models` collection is an operator step, and the -order matters: +The runtime does not migrate documents; editing `models` is an operator step. -1. **Inventory first, before the deploy.** Collect the models that fail in their configured flavor - loader — run predict/transform and note which ones raise. There is no automatic discovery - afterwards: once deployed, such a model simply fails, and the failure names the flavor loader, - not the fix. -2. **Edit the document.** Set `model_config.predict_flavor: 'joblib'` and/or - `model_config.transform_flavor: 'joblib'` for each model in the inventory. Nothing else changes; - models already on a working flavor need no edit and behave identically. -3. **Imported models arrive ready.** The `import_model` workflow writes - `predict_flavor: 'joblib'` itself — see [`model-import.md`](model-import.md) § 12. -4. **Check the first retrain.** In MLflow, confirm the new run carries - `prediction_model/model.pkl` **and** `prediction_model/code/` when the source model had a - `code/` directory, and that the promoted version loads on the next predict. -5. **Rolling back is not just a revert.** Reverting the code leaves any artifact a joblib retrain - already wrote in MLflow: that Production version stops being loadable, so the rollback also means - promoting the previous version back in the registry, and setting the documents' flavors back — - manual, in reverse order. +1. **Inventory before the deploy** — run predict/transform and note which models raise. There is no + discovery afterwards: the failure names the loader, not the fix. +2. **Edit the document** — `predict_flavor` and/or `transform_flavor` to `'joblib'`. Imported models + arrive ready: `import_model` writes it itself, see [`model-import.md`](model-import.md) § 12. +3. **Check the first retrain** — the new run must carry `prediction_model/model.pkl` and, when the + source had one, `prediction_model/code/`. +4. **Rollback is not just a revert** — artifacts a joblib retrain already wrote stop being loadable, + so it also means promoting the previous version back and reverting the documents' flavors. ## Known gap: how the run id is resolved -`get_model_run_id` resolves the Production run with `source.split('/')[2]` on the registered -version's `source`. **This has not been confirmed against the real tracking server.** No artifact -scheme reproduced locally puts the run id in that position (a file store yields `''`, S3 yields the -bucket, `mlflow-artifacts:/` yields the experiment id), yet predict works in production — so the -real `source` has a shape none of the local probes produced. - -This is not specific to `joblib`: every flavor that resolves a run goes through the same line, and -the joblib read path depends on it. Confirm it with a **read-only** query against the tracking -server before relying on it, ideally before the next deploy. +`get_model_run_id` takes `source.split('/')[2]` of the registered version's `source`. **Not confirmed +against the real tracking server**: no scheme reproduced locally puts the run id there (a file store +yields `''`, S3 the bucket, `mlflow-artifacts:/` the experiment id), yet predict works in production. +Every flavor that resolves a run goes through this line — confirm it with a **read-only** query +before relying on it. diff --git a/docs/sientia-bundle-format.md b/docs/sientia-bundle-format.md index 79dbc73..209cb82 100644 --- a/docs/sientia-bundle-format.md +++ b/docs/sientia-bundle-format.md @@ -1,111 +1,48 @@ # The `.sientia` bundle format -A `.sientia` file is one exported MLflow model: its whole artifact tree, its run parameters and six -fields of origin metadata, zipped and then encrypted under a password the exporter chose. It is -written by the Streamlit platform (`sientia-projects-templates`) and read by this repository's -importer. Two programs in two repositories agree on these bytes, so the constants below are -duplicated by necessity — the drift is held in place by tests on both sides, not by a shared library. +A `.sientia` file is one exported MLflow model: its whole artifact tree, its run parameters and its +origin metadata, zipped and then encrypted under a password the exporter chose. -This document covers both directions: **§1–§3 how a bundle is generated**, so the producer can be -audited against it, and **§4–§7 how it is read**, so a rejection can be explained. +**It is written by the Streamlit platform, `sientia-projects-templates`** (`app/src/operations/ +model_export/`). That application owns the export — its screen, its flow, its code — and documents +it. This page is the short version: the parameters both sides have to agree on, and what this +repository's reader accepts. -Normative implementations: +## How it is generated -| Side | Where | +The producer downloads the run's artifact tree whole, copies `run.data.params` **verbatim** (with +`target_variable` among them, or the import fails at gate 6), writes `metadata.json` first into a +`ZIP_DEFLATED` archive with the artifacts following as `artifacts/` and no directory +entries, encrypts that zip, and offers it as `{model_name}_v{model_version}.sientia`. + +`metadata.json` has exactly two keys: `parameters` — the run's parameters verbatim — and `metadata`, +six fields: `model_name`, `experiment_name`, `run_id`, `model_version`, `model_project`, +`export_timestamp` (UTC ISO 8601). + +## Encryption parameters + +The reader derives the key with the values the header carries, but refuses anything above the +producer's own cost — a header declaring more is a resource-exhaustion attempt, refused before any +memory is allocated. + +| | Value | |---|---| -| Writer | `sientia-projects-templates`, `app/src/operations/model_export/{crypto,bundle,exporter}.py` | -| Reader | this repository, `laborious/utils/bundle/{format,reader}.py` | -| Test writer | this repository, `tests/helpers/bundle_factory.py` — mirrors the producer field for field, for building hostile variants | -| Golden fixture | `tests/fixtures/bundle/` — a real producer artefact, with its password and producer commit recorded in that directory's `README.md` | +| KDF | Argon2id (`crypto_pwhash_ALG_ARGON2ID13`) | +| memlimit | `268435456` (256 MiB) — a **ceiling** on the reader's side | +| opslimit | `3` — likewise a ceiling | +| parallelism | `1` (reserved, not configurable) | +| salt | 16 bytes, fresh per bundle | +| key | 32 bytes | +| AEAD | XChaCha20-Poly1305 `secretstream`, 64 KiB plaintext chunks, last tagged `FINAL` | +| digest | SHA-256 of the **pushed chunks only** — on disk, `sha256(file[164:])`, past the 140-byte header and the 24-byte stream header. Hashing from 140 rejects every valid bundle | +| password | normalised `NFC`, encoded UTF-8 — no trimming, no case folding | ---- +The export-side password rule (12+ characters, mixed case, digit, symbol) is enforced there only: by +the time a bundle exists, the password is whatever it was, and the reader cannot re-check it. -## 1. Generation, step by step +## Shape of the file -Entry point: `exporter.export_model_bundle(*, client, run_id, model_name, experiment_name, -model_version, model_project, password) -> bytes`. - -1. **Password strength** — `crypto.validate_password_strength`: at least 12 characters, one - uppercase, one lowercase, one digit, one symbol. **Export-side only.** The importer cannot - re-check it and does not try: by the time a bundle exists, the password is whatever it was. -2. **Exportability** — `bundle.validate_run_is_exportable`: the run's top-level artifacts must - include **both** `data_model` and `prediction_model`, checked with `client.list_artifacts(run_id)` - before anything is downloaded. -3. **Whole-tree download** — `client.download_artifacts(run_id, '', staging)` into a - `tempfile.mkdtemp(prefix='sientia_export_')` directory. The entire artifact tree, not a subset. -4. **Run parameters** — `dict(run.data.params)`, copied **verbatim**. Whatever the run logged is - what the bundle carries, including `target_variable` (see §3). -5. **Metadata document** — `bundle.build_metadata_json`, exactly two top-level keys: - - `parameters`: the run parameters, verbatim; - - `metadata`: `model_name`, `experiment_name`, `run_id`, `model_version`, `model_project` and - `export_timestamp = datetime.now(UTC).isoformat()`. Six fields, no others. -6. **Zip** — `bundle.build_zip`, `zipfile.ZIP_DEFLATED`. `metadata.json` is written **first**, with - `json.dumps(...)` and no indent. Every file under the staging directory follows as - `artifacts/`, with `\` replaced by `/`. **No directory entries are stored**, so a reader - must derive directories from file paths. -7. **Encryption** — `crypto.encrypt_archive`: a fresh 16-byte `os.urandom` salt, Argon2id key - derivation, then a libsodium `secretstream` push in 64 KiB plaintext chunks, the last chunk tagged - `FINAL` and the rest `MESSAGE`. -8. **Download name** — `{model_name}_v{model_version}.sientia`. The staging directory is removed in a - `finally`, and the password is popped from the Streamlit session state. - -Producer pins: `pynacl==1.5.0`, `mlflow==2.10.1`. - -## 2. Constants - -Copied from `crypto.py` unless noted. The reader's names are the ones in -`laborious/utils/bundle/format.py`. - -| Producer | Value | Reader | -|---|---|---| -| `MAGIC` | `b'SIENTIA1'` | `MAGIC` | -| `_HEADER_STRUCT` | `struct.Struct('>8sBBB16sQQB32s64s')` | `HEADER_STRUCT` | -| `HEADER_STRUCT_SIZE` | `140` | `HEADER_SIZE` | -| `FORMAT_VERSION` | `1` | `FORMAT_VERSION` | -| `AEAD_ID_XCHACHA20POLY1305_SECRETSTREAM` | `1` | same name | -| `KDF_ID_ARGON2ID` | `1` | same name | -| `SALT_SIZE` | `16` | `SALT_SIZE` | -| `KDF_MEMLIMIT_BYTES` | `268435456` (256 MiB) | `MAX_KDF_MEMLIMIT` — a **ceiling** on the reader's side | -| `KDF_OPSLIMIT` | `3` | `MAX_KDF_OPSLIMIT` — likewise a ceiling | -| `KDF_PARALLELISM_RESERVED` | `1` | `PARALLELISM` | -| `DIGEST_SIZE` | `32` | `DIGEST_SIZE` | -| `RESERVED_SIGNATURE_SIZE` | `64` | `RESERVED_SIGNATURE_SIZE` | -| `CHUNK_SIZE` | `65536` (64 KiB) | `CHUNK_SIZE` | -| KDF algorithm | `crypto_pwhash_ALG_ARGON2ID13` | `KDF_ALG` | -| Key length | `crypto_secretstream_xchacha20poly1305_KEYBYTES` (32) | `KEY_SIZE` | - -Fixed by libsodium rather than by the producer, but needed to frame the stream: - -- `crypto_secretstream_xchacha20poly1305_HEADERBYTES == 24` → `STREAM_HEADER_SIZE` -- `crypto_secretstream_xchacha20poly1305_ABYTES == 17` → `ABYTES` -- pre-upload prefix `140 + 24 + (65536 + 17) = 65717` → `PREFIX_SIZE`, the smallest number of leading - bytes from which a header and one full chunk can be checked - -**Password normalisation.** `_derive_key` applies `unicodedata.normalize('NFC', password)` and -encodes UTF-8 — no trimming, no case folding, no padding. The reader normalises identically, so a -password typed in decomposed form opens a bundle written from the composed form. - -## 3. `target_variable`, and where it does not come from - -Gate 6 requires `parameters.target_variable`: it is the only source of `model_config.target` in the -model document, and every workflow that serves a model indexes that key directly. On the producer -side: - -| Export path | Logs `target_variable`? | Evidence | -|---|---|---| -| Time-series training template | yes | `app/pages/template_01.py` L1212-1214 | -| Model manager, "Save Experiment" | yes, if the session carried it | `app/pages/model_manager.py` L791-796 re-logs `st.session_state.parameters` verbatim | -| Model ensemble | same condition | `app/pages/model_ensemble.py` L695-700 | -| Pipeline / timeseries template | **no** | `app/pages/timeseries_template.py` L1761-1762, L1811-1812 log `run_metadata` only | -| AutoML | **no** | `automl_logging_operations.py` L454-491; the target appears as `"target"` inside the model card JSON, never as a run parameter | - -So a gate-6 rejection naming `target_variable` means the origin run came from the pipeline or AutoML -path. The remedy is to re-export from a run trained by the time-series template, which is what the -importer's `IMPORT_BUNDLE_INCOMPLETE` sentence already asks for. - ---- - -## 4. The file on disk +`HEADER_STRUCT` is `>8sBBB16sQQB32s64s`. Big-endian, no alignment padding, so the offsets are exact: ``` offset width field constraint the reader enforces @@ -125,72 +62,39 @@ offset width field constraint the reader enforces 164 ... chunk stream 64 KiB plaintext chunks, each +17 bytes of AEAD tag ``` -All integers are big-endian; the format string starts with `>`, so there is no alignment padding and -the offsets above are exact. +`format_version` is checked for equality, not for a range: an unknown version is refused at gate 2 +rather than parsed optimistically. `reserved_signature` must be all zero today, so a signed bundle +will be a new format version and not a silent change of meaning. -**The digest boundary.** The producer computes `hashlib.sha256(ciphertext)` where `ciphertext` is the -concatenation of the *pushed chunks only*, and then writes `header + stream_header + ciphertext`. On -disk, therefore, the digest covers everything after offset **164**, not after 140: +`laborious/utils/bundle/format.py` is the normative layout on this side, and the golden fixture is +what keeps it and the producer from drifting apart. -> `ciphertext_digest == sha256(file[HEADER_SIZE + STREAM_HEADER_SIZE:])` +## How it is read -A reader that hashes from 140 will reject every valid bundle. The fixture test that recomputes both -values and asserts the first matches while the second does not is what pins this, and is the reason -the golden fixture exists. +`BundleReader.open()` runs seven ordered gates, each mapped to an import step and to one message a +person can read: -**Version policy.** `format_version` is checked for equality, not for a range: a version this reader -does not know is refused at gate 2 rather than parsed optimistically. `reserved_signature` is -reserved for a future signature and must be all zero today, so a signed bundle is a new format -version and not a silent change of meaning. - -## 5. The seven gates - -`BundleReader.open()` runs them in this order, in one function, so the order cannot be changed by a -caller. Each raises `BundleGateError(gate=n)`, which the activity layer maps to an import step. - -| Gate | Step recorded | What it checks | Written to disk | -|---|---|---|---| -| 1 | `download` | object size ceiling, key prefix, `.sientia` suffix — from `stat_object`, before the body is fetched; then the SHA-256 of the downloaded bytes against the digest the uploader computed | the downloaded object only | -| 2 | `download` | the whole header table of §4, including the chunk-stream digest | nothing | -| 3 | `decryption` | Argon2id derivation with the header's parameters, then `secretstream` pull. The AEAD is the integrity check: a wrong password and an altered file fail identically, and the message says so | the decrypted zip, in the work directory | -| 4 | `archive_inspection` | central directory only: entry count, total uncompressed size, compression ratio, absolute names, `..` components, symlink and non-regular modes | nothing | -| 5 | `extraction` | extraction into a **fresh** directory (a pre-existing one is a failure), every member's `realpath` contained in it, no overwrite of a pre-existing file; a rejection removes its own partial tree | the extracted tree | -| 6 | `structure_validation` | `metadata.json` + `artifacts/` layout, exactly two top-level keys, all six metadata fields present and non-blank, `experiment_name` between 3 and 50 characters, `parameters.target_variable` present and non-blank, and `prediction_model`/`data_model` each holding `MLmodel` and `model.pkl` | nothing | -| 7 | `content_policy` | every file is either a known name (`MLmodel`, `metadata.json`, `model.pkl`, `conda.yaml`, `python_env.yaml`, `requirements.txt`, `model_card.json`, `model_card.svg`) or a known suffix (`.csv .json .yaml .yml .txt .pkl .svg .md`) | nothing | - -Gates 1 and 2 share the `download` step deliberately: an oversized object, a wrong magic, an -unsupported version and a digest that disagrees all say *these are not the bytes that were sent*, and -none of them may borrow the `decryption` sentence, which implicates the password. - -The `experiment_name` bound in gate 6 is not cosmetic: the import record's `experiment_name` column is -`VARCHAR(50)`, and the name is written onto the record before provisioning starts. Enforcing the -column's limit at the gate is what keeps it from surfacing three activities later as a constraint -violation. - -## 6. Ceilings and where they come from - -Gate 1 and gate 4 read their limits from `build_import_config()`, so a deployment can tighten them: - -| Variable | Default | Gate | +| Gate | Step | Refuses | |---|---|---| -| `IMPORT_MAX_OBJECT_BYTES` | 1 GiB | 1 | -| `IMPORT_MAX_ARCHIVE_ENTRIES` | 5000 | 4 | -| `IMPORT_MAX_UNCOMPRESSED_BYTES` | 4 GiB | 4 | -| `IMPORT_MAX_COMPRESSION_RATIO` | 200 | 4 | -| `IMPORT_BUNDLE_PREFIX` | `imported_models/` | 1 | +| 1 | `download` | object too large, wrong prefix or suffix, digest ≠ the one the uploader sent | +| 2 | `download` | header that is not this format or not this version | +| 3 | `decryption` | wrong password, or altered bytes — the AEAD cannot tell them apart, and the message says so | +| 4 | `archive_inspection` | zip bomb, absolute or `..` entries, symlinks — read from the central directory, before extracting | +| 5 | `extraction` | anything that escapes the fresh work directory, or overwrites a file | +| 6 | `structure_validation` | missing `metadata.json` fields, missing `target_variable`, missing `prediction_model`/`data_model` | +| 7 | `content_policy` | a file that is not a known model file by name or suffix | -The KDF ceilings are **not** configurable: they are the producer's own cost parameters, and a header -declaring more is a resource-exhaustion attempt, refused before any memory is allocated. +Gates 1 and 2 share the `download` step on purpose: neither may borrow the `decryption` sentence, +which implicates the password. A gate-6 rejection naming `target_variable` means the origin run never +logged it — the fix is a re-export, on the producer's side. -## 7. Fixtures +Error codes, sentences and the ceilings the gates read from configuration: +[`model-import.md`](model-import.md). -- `tests/fixtures/bundle/golden_model_v3.sientia` — emitted by the producer's own code in its venv, - with the password and producer commit recorded in `tests/fixtures/bundle/README.md`. Regenerate it - with `tests/fixtures/bundle/regenerate.py`. It pays the producer's real Argon2id cost (256 MiB, - 3 passes), so it is used where the real thing matters and not in every test. -- `tests/helpers/bundle_factory.py` — the test-only writer. Same header fields at the same offsets, - but 8 MiB and 1 pass, and with a knob for every hostile variant: wrong magic, unknown version, KDF - parameters above the ceiling, digest mismatch, tampered ciphertext, truncated final chunk, non-zero - reserved signature, zip bomb, `..` entry, absolute entry, symlink, missing `data_model`, missing - `MLmodel`, missing `model.pkl`, malformed metadata, missing `target_variable`, extra `.sh` file. A - test asserts its header bytes are structurally identical to the golden fixture's. +## Fixtures + +- `tests/fixtures/bundle/` — a real bundle emitted by the producer, with its password and producer + commit recorded in that directory's `README.md`. It pays the producer's real Argon2id cost, so it + is used where the real bytes matter and not in every test. +- `tests/helpers/bundle_factory.py` — the test-only writer: same header at the same offsets, cheap + KDF parameters, and a knob for every hostile variant the gates above refuse. diff --git a/e2e/conftest.py b/e2e/conftest.py index 29076b3..bf4310f 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -546,6 +546,15 @@ def patch_mlflow(mock_mlflow_models, joblib_flavor_artifacts): mock_mlflow.pyfunc = MagicMock() mock_mlflow.pyfunc.load_model = MagicMock(side_effect=mock_pyfunc_load_model) + # The sklearn and pyfunc paths resolve the URI to a local directory before loading, so that the + # artifact's `code/` can go on `sys.path` whatever the `MLmodel` manifest says. Echoing the URI + # keeps the loader stubs above — which branch on the URI text — working unchanged, and points + # the code-directory lookup at a path that does not exist. + mock_mlflow.artifacts = MagicMock() + mock_mlflow.artifacts.download_artifacts = MagicMock( + side_effect=lambda artifact_uri: str(artifact_uri) + ) + # Mock other mlflow methods that might be called mock_mlflow.set_tracking_uri = MagicMock() mock_mlflow.get_run = MagicMock( diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 54e3391..ea13d07 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -45,6 +45,10 @@ ARTIFACTS_PATH = './tmp/artifacts' TRANSFORMED_COMPRESSED_PATH = 'artifacts/training_transformer.pkl' PREDICTION_COMPRESSED_PATH = 'artifacts/stacking_model.pkl' +# Pickle file name the `joblib` flavor writes and looks for first, inside the model artifact's own +# directory (`/artifacts/prediction_model/`, `/artifacts/data_model/`). +MODEL_PICKLE_NAME = 'model.pkl' + INVALID_FLAVOR_MESSAGE = "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'joblib'." @@ -358,21 +362,31 @@ class MLFlowRepository(SientiaMonitoring): """ Locate the pickle file of a downloaded model artifact. - Primary path of the `joblib` flavor. The target artifact is always a - single `.pkl` at the top level of the artifact directory, so the listing - does not recurse — nested wrapper artifacts must never be deserialized - in place of the model. No other extension is considered and `MLmodel` is - not parsed. + Primary path of the `joblib` flavor. `local_dir` is the directory of the + model artifact itself — `/artifacts/prediction_model` or + `/artifacts/data_model`, the very artifacts the other flavors + resolve — never the run's `artifacts/` root. So the pickle sought is + `/model.pkl`, the name `log_model` writes. + + `MODEL_PICKLE_NAME` wins whenever it is present, so write and read agree + by construction. Any other `.pkl` directly inside the artifact is taken + as a fallback, in name order, for artifacts built by hand. The listing + does not recurse: a pyfunc wrapper nested inside the artifact + (`prediction_model/artifacts/stacking_model.pkl`) must never be + deserialized in place of the model. No other extension is considered and + `MLmodel` is not parsed. Args: - local_dir (str): Local directory holding the downloaded artifact. + local_dir (str): Local directory of the downloaded model artifact. artifact_name (str): Artifact name, e.g. 'prediction_model'. Returns: - str: Path to the first top-level `.pkl` file, in name order. + str: Path to `/model.pkl` when it exists, otherwise to + the first other `.pkl` directly inside `local_dir`, in name + order. Raises: - ValueError: If the top level of `local_dir` has no `.pkl` file. + ValueError: If `local_dir` holds no `.pkl` file directly. """ entries = sorted(listdir(local_dir)) pkl_files = [ @@ -383,11 +397,97 @@ class MLFlowRepository(SientiaMonitoring): if not pkl_files: raise ValueError( - f'No .pkl file found in {artifact_name} artifact. ' - f'Top-level entries in {local_dir}: {entries}' + f'No .pkl file found in {artifact_name} artifact. Entries in {local_dir}: {entries}' ) - return path.join(local_dir, pkl_files[0]) + chosen = MODEL_PICKLE_NAME if MODEL_PICKLE_NAME in pkl_files else pkl_files[0] + + return path.join(local_dir, chosen) + + def prepend_artifact_code_dir( + self, + local_dir: str | None, + flavor: str, + metadata: dict[str, Any] | None = None, + ) -> str | None: + """ + Put a downloaded artifact's own `code` directory on `sys.path`. + + Every flavor goes through here, not only `joblib`. MLflow's loaders + extend `sys.path` from the **manifest**: `_add_code_from_conf_to_system_path` + acts only when the `MLmodel` flavor config declares a `code` entry. An + artifact that carries `code/` on disk without that entry — anything + whose code was logged with `log_artifacts`, anything re-logged by a + path that dropped the config, anything a `joblib` retrain wrote — loads + with no module path and raises `ModuleNotFoundError` at unpickle time, + with the artifact intact on disk the whole while. Reading the directory + itself depends on no manifest, so it works for all three flavors. + + Prepending before the loader runs is deliberate: the loader is what + unpickles, so the path has to be in place first. When the manifest + *does* declare `code`, MLflow inserts the same directory again and the + duplicate entry is harmless. + + The entry is never removed. A loaded model may import lazily, long after + the load returned. + + Args: + local_dir (str | None): Local directory of the downloaded model + artifact, e.g. the download of `/artifacts/prediction_model`. + flavor (str): Flavor doing the load, for the log line only. + metadata (dict[str, Any] | None): Metadata used for structured + logging. + + Returns: + str | None: The directory added, `None` when the artifact carries no + `code` directory or it was already on `sys.path`. + """ + code_dir = self.resolve_artifact_code_dir(local_dir) + + if not code_dir: + return None + + if code_dir in sys.path: + self.debug(f'{flavor} flavor: {code_dir} already on sys.path', metadata) + return None + + sys.path.insert(0, code_dir) + self.debug(f'{flavor} flavor: added {code_dir} to sys.path', metadata) + + return code_dir + + def _localize_model_uri( + self, model_uri: str, flavor: str, metadata: dict[str, Any] | None = None + ) -> str: + """ + Resolve a model URI to a local directory and put its `code` on `sys.path`. + + The MLflow loaders download the artifact themselves and take a local + path just as readily as a URI, so resolving here costs no extra + download — it only moves the download one step earlier, which is what + makes the artifact's `code` directory visible before the loader + unpickles anything. + + Args: + model_uri (str): `models://production` or a run artifact URI. + flavor (str): Flavor doing the load, for the log line only. + metadata (dict[str, Any] | None): Metadata used for structured + logging. + + Returns: + str: Local directory the model artifact was downloaded to, to be + handed to the flavor loader in place of the URI. + + Raises: + Exception: Any error raised while downloading the artifact. + """ + local_path = mlflow.artifacts.download_artifacts(artifact_uri=model_uri) + + self.debug(f'{flavor} flavor: {model_uri} downloaded to {local_path}', metadata) + + self.prepend_artifact_code_dir(local_path, flavor, metadata) + + return local_path def _load_model_pickle( self, @@ -404,13 +504,14 @@ class MLFlowRepository(SientiaMonitoring): `._model_impl.python_model` unwrap is applied, because the pickled object already is the model. - The artifact's top-level `code` directory is prepended to `sys.path` - before deserializing, when it exists, so that classes whose modules ship - inside the artifact resolve at unpickle time. The entry is not removed - afterwards: the deserialized object may import lazily. + The artifact's own `code` directory — `/code` — is prepended + to `sys.path` before deserializing, when it exists, so that classes + whose modules ship inside the artifact resolve at unpickle time. That + step is shared with every other flavor: see `prepend_artifact_code_dir`. Args: - local_dir (str): Local directory holding the downloaded artifact. + local_dir (str): Local directory of the downloaded model artifact, + e.g. the download of `/artifacts/prediction_model`. artifact_name (str): Artifact name, e.g. 'prediction_model'. metadata (dict[str, Any] | None): Metadata used for structured logging, so the chosen pickle path correlates with the other @@ -420,15 +521,12 @@ class MLFlowRepository(SientiaMonitoring): Any: The object deserialized from the artifact's pickle file. Raises: - ValueError: If the artifact directory has no top-level `.pkl` file. + ValueError: If the artifact directory holds no `.pkl` file directly. Exception: Any error raised by `joblib.load` itself. """ pkl_path = self._find_model_pickle(local_dir, artifact_name) - code_dir = self.resolve_artifact_code_dir(local_dir) - if code_dir and code_dir not in sys.path: - sys.path.insert(0, code_dir) - self.debug(f'joblib flavor: added {code_dir} to sys.path', metadata) + self.prepend_artifact_code_dir(local_dir, 'joblib', metadata) self.debug(f'joblib flavor: loading {artifact_name} from {pkl_path}', metadata) @@ -448,8 +546,10 @@ class MLFlowRepository(SientiaMonitoring): `dst_path`, so MLflow writes to a per-call temporary directory: the shared `ARTIFACTS_PATH` tree is never written to and `dowload_artifacts` — which removes its destination before - downloading — is deliberately not reused. The target artifact is always - a `.pkl` and the deserialized object is returned as is. + downloading — is deliberately not reused. `artifact_name` is the same + model artifact the other flavors resolve, so the download yields + `/artifacts//` and the pickle read from it is that + directory's `model.pkl`. The deserialized object is returned as is. Callers that already resolved a run must pass it as `run_id`, so that the run is resolved exactly once per load. Callers with no run at hand — @@ -470,7 +570,7 @@ class MLFlowRepository(SientiaMonitoring): artifact was downloaded to. Raises: - ValueError: If the downloaded artifact has no top-level `.pkl` file. + ValueError: If the downloaded artifact holds no `.pkl` file directly. Exception: Any error raised while resolving the Production run, downloading the artifact, or deserializing the pickle. """ @@ -559,10 +659,13 @@ class MLFlowRepository(SientiaMonitoring): Notes: - The model is fetched from the "Production" stage of the MLflow Model Registry. - Warnings during the model loading process are suppressed. + - The artifact's `code` directory, when present, is put on + `sys.path` before the load, whatever the flavor. `sklearn` and + `pyfunc` are therefore resolved to a local directory first and the + loader is handed that directory instead of the URI. - With `flavor='joblib'` no MLflow flavor loader is called: the `prediction_model` artifact is downloaded and deserialized with - `joblib.load`, which needs no `MLmodel` manifest. The artifact's - `code` directory, when present, is put on `sys.path` first. + `joblib.load`, which needs no `MLmodel` manifest. """ if flavor not in ('pyfunc', 'sklearn', 'joblib'): raise ValueError(INVALID_FLAVOR_MESSAGE) @@ -582,10 +685,12 @@ class MLFlowRepository(SientiaMonitoring): core_labels = self.get_core_labels(metadata, operation_type='load_predict_model') start_time = time.time() try: + local_path = self._localize_model_uri(model_uri, flavor, metadata) + if flavor == 'pyfunc': - model = mlflow.pyfunc.load_model(model_uri) + model = mlflow.pyfunc.load_model(local_path) else: - model = mlflow.sklearn.load_model(model_uri) + model = mlflow.sklearn.load_model(local_path) except Exception as e: await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels) raise e @@ -617,11 +722,14 @@ class MLFlowRepository(SientiaMonitoring): model cannot be loaded. Notes: + - The artifact's `code` directory, when present, is put on + `sys.path` before the load, whatever the flavor. `sklearn` and + `pyfunc` are therefore resolved to a local directory first and the + loader is handed that directory instead of the URI. - With `flavor='joblib'` no MLflow flavor loader is called and no model URI is built: the `data_model` artifact of the resolved Production run is downloaded and deserialized with `joblib.load`, - which needs no `MLmodel` manifest. The artifact's `code` - directory, when present, is put on `sys.path` first. + which needs no `MLmodel` manifest. """ if flavor not in ('sklearn', 'pyfunc', 'joblib'): @@ -646,10 +754,12 @@ class MLFlowRepository(SientiaMonitoring): core_labels = self.get_core_labels(metadata, operation_type='load_transform_model') start_time = time.time() try: + local_path = self._localize_model_uri(model_uri, flavor, metadata) + if flavor == 'sklearn': - model = mlflow.sklearn.load_model(model_uri) + model = mlflow.sklearn.load_model(local_path) else: - model = mlflow.pyfunc.load_model(model_uri) + model = mlflow.pyfunc.load_model(local_path) except Exception as e: await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels) raise e @@ -684,6 +794,10 @@ class MLFlowRepository(SientiaMonitoring): downloaded to as the artifact path, because the retrain path needs it to re-log the artifact's `code` directory. Every other flavor returns `None` unless the wrapper branch was taken. + - The wrapper branch downloads before it loads, so it puts the + artifact's `code` directory on `sys.path` itself. The two plain + branches delegate, and the code directory is handled inside + `load_predict_model` / `load_transform_model`. """ self.info( @@ -706,6 +820,8 @@ class MLFlowRepository(SientiaMonitoring): f'Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}' ) + self.prepend_artifact_code_dir(artifact_path, 'pyfunc', metadata) + raw_model = mlflow.pyfunc.load_model(artifact_path) model = raw_model._model_impl.python_model @@ -1255,7 +1371,7 @@ class MLFlowRepository(SientiaMonitoring): self.debug('Model uploaded successfully', metadata) elif flavor == 'joblib': with tempfile.TemporaryDirectory() as tmp_dir: - pkl_path = path.join(tmp_dir, 'model.pkl') + pkl_path = path.join(tmp_dir, MODEL_PICKLE_NAME) joblib.dump(model, pkl_path, compress=3) mlflow.log_artifact(pkl_path, artifact_path=model_type) diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 02e8cd5..99f6194 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -17,6 +17,7 @@ from laborious import metrics from laborious.utils.repository import model_repository from laborious.utils.repository.model_repository import ( INVALID_FLAVOR_MESSAGE, + MODEL_PICKLE_NAME, MLFlowRepository, force_memory_release, ) @@ -68,6 +69,11 @@ def mlflow_repository(): @pytest.fixture def mlflow(): with patch('laborious.utils.repository.model_repository.mlflow') as mlflow: + # Every URI load resolves the artifact to a local directory before calling the flavor + # loader, and `os.path` refuses a MagicMock. Echoing the URI back keeps the loader's + # argument readable and points the code-directory lookup at a path that does not exist, + # so no test touches `sys.path` unless it builds a real artifact directory. + mlflow.artifacts.download_artifacts.side_effect = lambda artifact_uri: str(artifact_uri) yield mlflow @@ -467,6 +473,11 @@ async def test_load_predict_model_error(mlflow_repository): mlflow_repository.observe_lag.assert_not_called() +# A real string, because the transform load now resolves its URI to a local directory before +# calling the loader, and that resolution goes through `os.path`. +TRANSFORM_MODEL_URI = 'runs:/run-123/data_model' + + def validate_common_load_transform_model_mocks(mlflow_repository, model_name): mlflow_repository.get_model_run_id.assert_called_once_with( model_name=model_name, stage='Production' @@ -479,14 +490,14 @@ def validate_common_load_transform_model_mocks(mlflow_repository, model_name): @pytest.mark.asyncio async def test_load_transform_model_sklearn(mlflow, mlflow_repository): mlflow_repository.get_model_run_id = MagicMock() - mlflow_repository.get_model_uri = MagicMock() + mlflow_repository.get_model_uri = MagicMock(return_value=TRANSFORM_MODEL_URI) result = await mlflow_repository.load_transform_model('test_model', {}, 'sklearn') validate_common_load_transform_model_mocks(mlflow_repository, 'test_model') assert result == mlflow.sklearn.load_model.return_value - mlflow.sklearn.load_model.assert_called_once_with(mlflow_repository.get_model_uri.return_value) + mlflow.sklearn.load_model.assert_called_once_with(TRANSFORM_MODEL_URI) mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_READ_LAG, ANY) mlflow_repository.emit_metric.assert_called_once_with( @@ -497,14 +508,14 @@ async def test_load_transform_model_sklearn(mlflow, mlflow_repository): @pytest.mark.asyncio async def test_load_transform_model_pyfunc(mlflow, mlflow_repository): mlflow_repository.get_model_run_id = MagicMock() - mlflow_repository.get_model_uri = MagicMock() + mlflow_repository.get_model_uri = MagicMock(return_value=TRANSFORM_MODEL_URI) result = await mlflow_repository.load_transform_model('test_model', {}, 'pyfunc') validate_common_load_transform_model_mocks(mlflow_repository, 'test_model') assert result == mlflow.pyfunc.load_model.return_value - mlflow.pyfunc.load_model.assert_called_once_with(mlflow_repository.get_model_uri.return_value) + mlflow.pyfunc.load_model.assert_called_once_with(TRANSFORM_MODEL_URI) mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_READ_LAG, ANY) mlflow_repository.emit_metric.assert_called_once_with( @@ -543,8 +554,8 @@ async def test_download_model_invalid_model_type(mlflow_repository): @pytest.mark.parametrize( 'model_type', [('predict', 'prediction_model'), ('transform', 'data_model')] ) -async def test_download_model_load_wrapper(mlflow, mlflow_repository, model_type): - mlflow_repository.dowload_artifacts = AsyncMock() +async def test_download_model_load_wrapper(mlflow, mlflow_repository, model_type, tmp_path): + mlflow_repository.dowload_artifacts = AsyncMock(return_value=str(tmp_path)) result = await mlflow_repository.download_model('test_model', {}, model_type[0], 'pyfunc', True) @@ -1740,14 +1751,14 @@ FLAVOR_LOADERS = ('sklearn', 'pyfunc') @pytest.fixture def preserved_sys_path(): - """Restore `sys.path`: the joblib flavor inserts entries and never removes them.""" + """Restore `sys.path`: every flavor inserts artifact code entries and never removes them.""" original = list(sys.path) yield sys.path[:] = original def build_artifact_dir(tmp_path, *filenames): - """Create a fake downloaded artifact directory with the given top-level files.""" + """Create a fake download of one model artifact directory, holding the given files.""" for filename in filenames: (tmp_path / filename).write_bytes(b'pickle-bytes') return str(tmp_path) @@ -1979,7 +1990,12 @@ async def test_no_load_emits_a_fallback_operation_type( @pytest.mark.parametrize( ('filenames', 'expected'), [ + # The canonical name `log_model` writes, alongside a real MLflow manifest. (('MLmodel', 'model.pkl'), 'model.pkl'), + # `model.pkl` wins over any other candidate, whatever the name order. + (('a_model.pkl', 'model.pkl'), 'model.pkl'), + (('model.pkl', 'z_model.pkl'), 'model.pkl'), + # Fallback for hand-built artifacts: no `model.pkl`, so name order decides. (('MLmodel', 'python_model.pkl'), 'python_model.pkl'), (('b_model.pkl', 'a_model.pkl'), 'a_model.pkl'), ], @@ -1993,6 +2009,7 @@ def test_find_model_pickle_selection(mlflow_repository, tmp_path, filenames, exp def test_find_model_pickle_ignores_nested_pickles(mlflow_repository, tmp_path): + """A pyfunc wrapper's own `artifacts/*.pkl` is never taken for the model.""" nested = tmp_path / 'artifacts' nested.mkdir() (nested / 'stacking_model.pkl').write_bytes(b'pickle-bytes') @@ -2003,6 +2020,15 @@ def test_find_model_pickle_ignores_nested_pickles(mlflow_repository, tmp_path): assert 'No .pkl file found' in str(exc_info.value) +def test_find_model_pickle_reads_the_name_log_model_writes(mlflow_repository, tmp_path): + """Read and write agree: both go through `MODEL_PICKLE_NAME`.""" + local_dir = build_artifact_dir(tmp_path, MODEL_PICKLE_NAME) + + result = mlflow_repository._find_model_pickle(local_dir, 'prediction_model') + + assert result == str(tmp_path / MODEL_PICKLE_NAME) + + def test_find_model_pickle_empty_directory(mlflow_repository, tmp_path): with pytest.raises(ValueError) as exc_info: mlflow_repository._find_model_pickle(str(tmp_path), 'data_model') @@ -2099,6 +2125,125 @@ def test_load_model_pickle_keeps_code_dir_after_load( assert code_dir in sys.path +""" +Tests for the artifact `code/` directory on every flavor's load path +""" + + +def observe_sys_path(recorder): + """A loader stand-in that records `sys.path` as it was when the load was called.""" + + def loader(_path): + recorder.extend(sys.path) + return MagicMock() + + return loader + + +def test_prepend_artifact_code_dir_inserts_at_the_front( + mlflow_repository, tmp_path, preserved_sys_path +): + code_dir = build_code_dir(tmp_path) + + result = mlflow_repository.prepend_artifact_code_dir(str(tmp_path), 'sklearn') + + assert result == code_dir + assert sys.path[0] == code_dir + + +def test_prepend_artifact_code_dir_without_code_dir_leaves_sys_path_alone( + mlflow_repository, tmp_path, preserved_sys_path +): + before = list(sys.path) + + assert mlflow_repository.prepend_artifact_code_dir(str(tmp_path), 'pyfunc') is None + assert sys.path == before + + +def test_prepend_artifact_code_dir_does_not_duplicate( + mlflow_repository, tmp_path, preserved_sys_path +): + code_dir = build_code_dir(tmp_path) + + mlflow_repository.prepend_artifact_code_dir(str(tmp_path), 'sklearn') + + assert mlflow_repository.prepend_artifact_code_dir(str(tmp_path), 'sklearn') is None + assert sys.path.count(code_dir) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize('flavor', FLAVOR_LOADERS) +async def test_load_predict_model_puts_code_dir_on_sys_path_before_the_loader( + flavor, mlflow, mlflow_repository, tmp_path, preserved_sys_path +): + """MLflow extends `sys.path` only when `MLmodel` declares `code`; the directory decides here.""" + code_dir = build_code_dir(tmp_path) + local_dir = build_artifact_dir(tmp_path, 'MLmodel', 'model.pkl') + mlflow.artifacts.download_artifacts.side_effect = None + mlflow.artifacts.download_artifacts.return_value = local_dir + sys_path_at_load = [] + getattr(mlflow, flavor).load_model.side_effect = observe_sys_path(sys_path_at_load) + + await mlflow_repository.load_predict_model('test_model', {}, flavor) + + mlflow.artifacts.download_artifacts.assert_called_once_with( + artifact_uri='models:/test_model/production' + ) + getattr(mlflow, flavor).load_model.assert_called_once_with(local_dir) + assert sys_path_at_load[0] == code_dir + + +@pytest.mark.asyncio +@pytest.mark.parametrize('flavor', FLAVOR_LOADERS) +async def test_load_transform_model_puts_code_dir_on_sys_path_before_the_loader( + flavor, mlflow, mlflow_repository, tmp_path, preserved_sys_path +): + code_dir = build_code_dir(tmp_path) + local_dir = build_artifact_dir(tmp_path, 'MLmodel', 'model.pkl') + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.get_model_uri = MagicMock(return_value=TRANSFORM_MODEL_URI) + mlflow.artifacts.download_artifacts.side_effect = None + mlflow.artifacts.download_artifacts.return_value = local_dir + sys_path_at_load = [] + getattr(mlflow, flavor).load_model.side_effect = observe_sys_path(sys_path_at_load) + + await mlflow_repository.load_transform_model('test_model', {}, flavor) + + mlflow.artifacts.download_artifacts.assert_called_once_with(artifact_uri=TRANSFORM_MODEL_URI) + getattr(mlflow, flavor).load_model.assert_called_once_with(local_dir) + assert sys_path_at_load[0] == code_dir + + +@pytest.mark.asyncio +@pytest.mark.parametrize('flavor', FLAVOR_LOADERS) +async def test_load_predict_model_without_code_dir_leaves_sys_path_alone( + flavor, mlflow, mlflow_repository, tmp_path, preserved_sys_path +): + mlflow.artifacts.download_artifacts.side_effect = None + mlflow.artifacts.download_artifacts.return_value = build_artifact_dir(tmp_path, 'MLmodel') + before = list(sys.path) + + await mlflow_repository.load_predict_model('test_model', {}, flavor) + + assert sys.path == before + + +@pytest.mark.asyncio +async def test_download_model_load_wrapper_puts_code_dir_on_sys_path( + mlflow, mlflow_repository, tmp_path, preserved_sys_path +): + """The wrapper branch downloads itself, so it is the one that has to add the directory.""" + code_dir = build_code_dir(tmp_path) + mlflow_repository.dowload_artifacts = AsyncMock(return_value=str(tmp_path)) + sys_path_at_load = [] + mlflow.pyfunc.load_model.side_effect = observe_sys_path(sys_path_at_load) + + await mlflow_repository.download_model('test_model', {}, 'predict', 'pyfunc', True) + + assert sys_path_at_load[0] == code_dir + assert sys.path[0] == code_dir + + @patch(JOBLIB_PATH) @pytest.mark.asyncio async def test_log_model_joblib_writes_compressed_pickle_and_code( From 30ff378cb7429ff71d8e62b2e29a69a15940eff9 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Sep 2026 13:27:32 -0300 Subject: [PATCH 31/64] QTZPOC-20 QTZPOC-20: Prioritize code/ directory download and sys.path injection before model loading, enabling MLflow flavors to stream models directly from URIs and simplifying joblib artifact handling. --- README.md | 11 +- docs/model-flavors.md | 28 +- e2e/conftest.py | 25 +- e2e/scenarios.md | 27 +- e2e/test_predictions_batch_joblib_flavor.py | 37 +++ e2e/test_retrain_joblib_flavor.py | 4 +- .../utils/repository/model_repository.py | 302 +++++++++--------- .../utils/repository/test_model_repository.py | 275 ++++++++++++---- 8 files changed, 464 insertions(+), 245 deletions(-) diff --git a/README.md b/README.md index f61454e..3ce886b 100644 --- a/README.md +++ b/README.md @@ -1114,8 +1114,15 @@ no fallback between flavors: a failing loader propagates. retrain (`model.pkl` + `code/`, no `MLmodel`, so only this runtime reads it back). Every flavor puts the artifact's own `code/` on `sys.path` before loading, **when that directory -exists**: MLflow's loaders only do it when the `MLmodel` manifest declares `code`, so `sklearn` and -`pyfunc` resolve the URI to a local directory first and load from there. +exists**: MLflow's loaders only do it when the `MLmodel` manifest declares `code`. `download_model` +owns that step for all of them — it asks the registry whether the artifact carries `code/` and, only +then, downloads that one directory. The models themselves are never pre-downloaded: `sklearn` and +`pyfunc` load straight from the model URI and fetch what they need. + +`download_model` does not branch on the flavor: past that shared first step it delegates to +`load_predict_model` / `load_transform_model`, and its only branch left is `load_wrapper`, which is +orthogonal to the flavor. It returns `(model, artifact_dir)`, the directory the retrain path re-logs +`code/` from. Full reference, including how to switch an existing model over: **[docs/model-flavors.md](docs/model-flavors.md)**. diff --git a/docs/model-flavors.md b/docs/model-flavors.md index 74c0d72..9efd64c 100644 --- a/docs/model-flavors.md +++ b/docs/model-flavors.md @@ -22,8 +22,8 @@ reads the artifact's `MLmodel` manifest to guess it. | Flavor | Read | Write (retrain) | How it loads | |---|---|---|---| -| `sklearn` | yes | yes | resolves the URI to a local directory, then `mlflow.sklearn.load_model()` | -| `pyfunc` | yes | yes | resolves the URI to a local directory, then `mlflow.pyfunc.load_model()`; the wrapper path also unwraps `._model_impl.python_model` | +| `sklearn` | yes | yes | `mlflow.sklearn.load_model()` — the loader fetches what it needs | +| `pyfunc` | yes | yes | `mlflow.pyfunc.load_model()`; the wrapper path downloads the artifact instead and unwraps `._model_impl.python_model` | | `joblib` | yes | yes | downloads the same artifact and `joblib.load`s its `/model.pkl` — **no MLflow flavor loader involved** | Anything else raises `ValueError` with `INVALID_FLAVOR_MESSAGE`, **before** any artifact is resolved @@ -71,15 +71,29 @@ which no manifest can misdescribe: - `prepend_artifact_code_dir` inserts `/code` at the front of `sys.path`, **only when that directory exists**, and only once (an entry already present is not duplicated). -- Because the insert must happen *before* the loader unpickles, `sklearn` and `pyfunc` resolve their - URI to a local directory first (`mlflow.artifacts.download_artifacts`) and the loader is handed - that directory. Same bytes, same download count — MLflow would have downloaded it anyway. -- The wrapper path (`download_model(load_wrapper=True)`) already had its local directory and adds - the entry itself; `joblib` adds it inside `_load_model_pickle`. +- `download_artifact_code_dir` is what feeds it on the download paths, and it is deliberately cheap: + the remote listing (`list_artifacts(run_id, '')`) decides first, so a run without `code` + downloads **nothing**, and a run with it downloads only `/code` — a handful of `.py` + files, never the model's weights. +- `download_model` runs that step first, for every flavor and both of its branches, *before* any + model is loaded — the insert has to be in place before the loader unpickles. +- The models themselves are not pre-downloaded: `mlflow.sklearn.load_model` and + `mlflow.pyfunc.load_model` are handed the model URI and fetch what they need. `joblib` downloads + its artifact because it must read the pickle, and adds the entry again inside `_load_model_pickle` + through the same helper, so the flavor works when called directly too. - When the manifest *does* declare `code`, MLflow inserts the same directory a second time. Harmless. The entry is never removed: a loaded model may import lazily, long after the load returned. +The directory `download_artifact_code_dir` downloaded into is also what `download_model` returns +next to the model — `tuple[Any, str | None]`, the second element being the artifact directory the +retrain path re-logs `code/` from, or `None` when the artifact has no `code/`. It falls back to +whatever the loader downloaded on its own, which is a real artifact directory for `joblib` and for +the wrapper branch. `load_predict_model` and `load_transform_model` stay the **single** place where +the flavor is decided: `download_model` has no `joblib` branch, it delegates by `model_type`. Its +only remaining branch is `load_wrapper`, which is derived from `flavor == 'pyfunc'` but is otherwise +orthogonal to the flavor. + > `sys.path` is process-global: two models embedding a package with the same name (`utils/`) on the > same worker resolve to whichever entered first. Mitigation, if it bites: one worker per model. diff --git a/e2e/conftest.py b/e2e/conftest.py index bf4310f..b9ee3d2 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -546,14 +546,7 @@ def patch_mlflow(mock_mlflow_models, joblib_flavor_artifacts): mock_mlflow.pyfunc = MagicMock() mock_mlflow.pyfunc.load_model = MagicMock(side_effect=mock_pyfunc_load_model) - # The sklearn and pyfunc paths resolve the URI to a local directory before loading, so that the - # artifact's `code/` can go on `sys.path` whatever the `MLmodel` manifest says. Echoing the URI - # keeps the loader stubs above — which branch on the URI text — working unchanged, and points - # the code-directory lookup at a path that does not exist. mock_mlflow.artifacts = MagicMock() - mock_mlflow.artifacts.download_artifacts = MagicMock( - side_effect=lambda artifact_uri: str(artifact_uri) - ) # Mock other mlflow methods that might be called mock_mlflow.set_tracking_uri = MagicMock() @@ -572,9 +565,23 @@ def patch_mlflow(mock_mlflow_models, joblib_flavor_artifacts): ), ) - # Only the joblib flavor calls download_artifacts directly on the client, one artifact - # per model type — a single return value would serve `data_model` from `prediction_model`. + # `download_model` asks whether the run's artifact carries a `code` directory before it loads + # anything, and downloads only that directory when it does. Answering "no" here keeps the + # download counts of these tests about the model itself; a test that exercises the `code` step + # overrides `list_artifacts` to answer "yes". + def mock_list_artifacts(_run_id, artifact_path=None): + if artifact_path: + return [] + return [MagicMock(path=artifact_name) for artifact_name in joblib_flavor_artifacts] + + mock_mlflow_client.list_artifacts = MagicMock(side_effect=mock_list_artifacts) + + # The joblib flavor downloads a whole model artifact, one per model type — a single return + # value would serve `data_model` from `prediction_model`. The `code` step asks for the nested + # `/code` instead, and gets that directory back, as MLflow would return it. def mock_download_artifacts(_run_id, artifact_name): + if artifact_name.endswith('/code'): + return joblib_flavor_artifacts[artifact_name.removesuffix('/code')]['code_dir'] return joblib_flavor_artifacts[artifact_name]['path'] mock_mlflow_client.download_artifacts = MagicMock(side_effect=mock_download_artifacts) diff --git a/e2e/scenarios.md b/e2e/scenarios.md index 1654303..28e70c0 100644 --- a/e2e/scenarios.md +++ b/e2e/scenarios.md @@ -1149,9 +1149,34 @@ restores `sys.path` in teardown, because the flavor deliberately never removes w #### Scenario 5.1.3: An MLflow Flavor Is Unaffected **Description**: A model declaring `sklearn` behaves exactly as before +**Input**: +- `client.list_artifacts` answering that the run's artifacts carry no `code` directory + **Assertions**: - The prediction lands with the mocked model's value -- `download_artifacts` NOT called — the joblib path is never entered as a safety net +- `download_artifacts` NOT called — the joblib path is never entered as a safety net, and with no + `code` directory to fetch there is nothing else to download either + +--- + +#### Scenario 5.1.4: Only the Artifact's `code` Directory Is Downloaded +**Description**: A model on an MLflow flavor whose artifact **does** carry `code/`. The flavor +loaders read the model URI themselves and fetch what they need, so the `code` directory — the one +thing MLflow will not put on `sys.path` unless the `MLmodel` manifest declares it — is the whole of +what this workflow downloads through the client + +**Input**: +- `model_config` on the default `sklearn` flavors +- `client.list_artifacts` answering that both artifacts carry `code` + +**Expected Behavior**: +- The `code` directory of each artifact is downloaded before its model is loaded, and goes on + `sys.path` +- Neither model's weights travel through `client.download_artifacts` + +**Assertions**: +- The prediction lands with the mocked model's value +- `download_artifacts` called exactly twice, with `'data_model/code'` and `'prediction_model/code'` --- diff --git a/e2e/test_predictions_batch_joblib_flavor.py b/e2e/test_predictions_batch_joblib_flavor.py index 0993e8f..76a0410 100644 --- a/e2e/test_predictions_batch_joblib_flavor.py +++ b/e2e/test_predictions_batch_joblib_flavor.py @@ -7,6 +7,8 @@ A passing run therefore proves the flavor decompresses the pickle **and** resolv without any `mlflow.*.load_model` being called for that model. """ +from unittest.mock import MagicMock + import pandas as pd import pytest from sqlalchemy import text @@ -208,6 +210,41 @@ async def test_prediction_workflow_with_sklearn_flavor_is_unchanged( mlflow_client.download_artifacts.assert_not_called() +@pytest.mark.asyncio +@pytest.mark.integration +async def test_prediction_workflow_downloads_only_the_artifact_code_directory( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + test_activities: Activities, + postgres_engine, + joblib_flavor_artifacts, + mlflow_client, + mock_mlflow_models, +): + """A flavor model whose artifact carries `code` downloads that directory — and only it. + + The flavor loaders read the model URI themselves, so nothing else is fetched through the + client: the `code` directory is the whole of what this workflow downloads. + """ + mlflow_client.list_artifacts.side_effect = lambda _run_id, artifact_path=None: ( + [MagicMock(path=f'{artifact_path}/code')] if artifact_path else [] + ) + + client = temporal_test_env.client + model_id = 275 + insert_sample_data(postgres_engine, model_id, [60.0, 78.2]) + input_data = build_input_data(model_id, 'test_model') + + await start_and_await_workflow( + client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-code-only') + ) + + assert_prediction(postgres_engine, model_id, prediction=0.5) + + downloaded = [loaded.args[1] for loaded in mlflow_client.download_artifacts.call_args_list] + assert sorted(downloaded) == ['data_model/code', 'prediction_model/code'] + + @pytest.mark.asyncio @pytest.mark.integration async def test_prediction_workflow_joblib_flavor_failure_reports_single_cause( diff --git a/e2e/test_retrain_joblib_flavor.py b/e2e/test_retrain_joblib_flavor.py index dabda57..4e6dcdd 100644 --- a/e2e/test_retrain_joblib_flavor.py +++ b/e2e/test_retrain_joblib_flavor.py @@ -117,7 +117,9 @@ async def test_retrain_joblib_model_writes_pickle_and_code_and_reloads( mlflow_client.download_artifacts.side_effect = lambda _run_id, _artifact_name: str(written) - reloaded = await repository.load_predict_model(JOBLIB_FLAVOR_MODEL_NAME, METADATA, 'joblib') + reloaded, _local_dir = await repository.load_predict_model( + JOBLIB_FLAVOR_MODEL_NAME, METADATA, 'joblib' + ) features = pd.DataFrame({'feature_1': [0.234], 'feature_2': [0.783]}) expected = joblib_flavor_artifacts['prediction_model']['model'].predict(features) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index ea13d07..76fbb4e 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -261,14 +261,21 @@ class MLFlowRepository(SientiaMonitoring): """ Check if an artifact exists in the MLflow Model Registry. + The listing is scoped to the parent of `artifact_path`, so nested paths + such as `prediction_model/code` are answered by listing + `prediction_model` rather than the run's artifact root. A top-level path + has no parent and lists the root, as before. Only the listing travels: + nothing is downloaded to answer the question. + Args: run_id (str): Run identifier to inspect. - artifact_path (str): Path to the artifact to check. + artifact_path (str): Path to the artifact to check, relative to the + run's artifact root, e.g. 'data_model' or 'data_model/code'. Returns: bool: True if the artifact exists, False otherwise. """ - artifacts = self.client.list_artifacts(run_id) + artifacts = self.client.list_artifacts(run_id, path.dirname(artifact_path) or None) self.debug(f'Artifacts of {run_id}: \n{artifacts}', metadata) self.debug(f'Looking for artifact {artifact_path} in {run_id}', metadata) @@ -456,38 +463,62 @@ class MLFlowRepository(SientiaMonitoring): return code_dir - def _localize_model_uri( - self, model_uri: str, flavor: str, metadata: dict[str, Any] | None = None - ) -> str: + async def download_artifact_code_dir( + self, + model_name: str, + artifact_name: str, + flavor: str, + metadata: dict[str, Any] | None = None, + ) -> str | None: """ - Resolve a model URI to a local directory and put its `code` on `sys.path`. + Download only an artifact's `code` directory and put it on `sys.path`. - The MLflow loaders download the artifact themselves and take a local - path just as readily as a URI, so resolving here costs no extra - download — it only moves the download one step earlier, which is what - makes the artifact's `code` directory visible before the loader - unpickles anything. + Cheap by construction: the remote listing decides first, and nothing is + downloaded when the artifact carries no `code`. When it does, the + download is scoped to `/code` — a handful of `.py` files + — never the artifact's weights. Loading the model itself stays the + loaders' business, and they read straight from the model URI. Args: - model_uri (str): `models://production` or a run artifact URI. - flavor (str): Flavor doing the load, for the log line only. + model_name (str): Registered model name. + artifact_name (str): Model artifact to inspect, i.e. + 'prediction_model' or 'data_model'. + flavor (str): Flavor doing the load, for the log lines only. metadata (dict[str, Any] | None): Metadata used for structured logging. Returns: - str: Local directory the model artifact was downloaded to, to be - handed to the flavor loader in place of the URI. + str | None: Local directory the `code` directory was downloaded + *into* — the artifact directory, i.e. the parent of the + downloaded `code` — or `None` when the run's artifact has no + `code` directory. Raises: - Exception: Any error raised while downloading the artifact. + Exception: Any error raised while resolving the Production run, + listing the artifact or downloading `code`. """ - local_path = mlflow.artifacts.download_artifacts(artifact_uri=model_uri) + run_id = self.get_model_run_id(model_name=model_name, stage='Production') + remote_code_path = f'{artifact_name}/code' - self.debug(f'{flavor} flavor: {model_uri} downloaded to {local_path}', metadata) + if not self.check_artifact_exists(run_id, remote_code_path, metadata): + self.debug( + f'{flavor} flavor: run {run_id} has no {remote_code_path} artifact, ' + 'nothing to download', + metadata, + ) + return None - self.prepend_artifact_code_dir(local_path, flavor, metadata) + local_code_dir = self.client.download_artifacts(run_id, remote_code_path) + artifact_dir = path.dirname(local_code_dir) - return local_path + self.debug( + f'{flavor} flavor: {remote_code_path} of run {run_id} downloaded to {local_code_dir}', + metadata, + ) + + self.prepend_artifact_code_dir(artifact_dir, flavor, metadata) + + return artifact_dir def _load_model_pickle( self, @@ -588,62 +619,9 @@ class MLFlowRepository(SientiaMonitoring): return model, local_dir - async def _load_joblib_artifact( - self, - model_name: str, - artifact_name: str, - operation_type: str, - metadata: dict[str, Any], - run_id: str | None = None, - ) -> tuple[Any, str]: - """ - Load a model artifact through the `joblib` flavor, with observability. - - Single owner of the joblib read path: no MLflow flavor loader is called - here, neither before nor after the pickle load. Metrics are emitted with - the **base** `operation_type` the caller passes — a joblib load is a - load, so no suffixed operation type exists. - - Args: - model_name (str): Registered model name. - artifact_name (str): Artifact to download, e.g. 'prediction_model'. - operation_type (str): Base metric operation type of the calling - method, e.g. 'load_predict_model'. - metadata (dict[str, Any]): Metadata used for structured logging. - run_id (str | None): Run already resolved by the caller. When `None`, - the Production run is resolved during the download. - - Returns: - tuple[Any, str]: The deserialized model and the local directory the - artifact was downloaded to. - - Raises: - Exception: Any error raised while resolving the run, downloading the - artifact or deserializing the pickle. It propagates as is: there - is no second attempt. - """ - core_labels = self.get_core_labels(metadata, operation_type=operation_type) - start_time = time.time() - - try: - model, local_dir = await self._download_and_load_pickle( - model_name=model_name, - artifact_name=artifact_name, - metadata=metadata, - run_id=run_id, - ) - except Exception as e: - await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels) - raise e - - await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels) - await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels) - - return model, local_dir - async def load_predict_model( self, model_name: str, metadata: dict[str, Any], flavor: str = 'sklearn' - ) -> Any: + ) -> tuple[Any, str | None]: """ Load a predictive model from the MLflow Model Registry. @@ -651,57 +629,57 @@ class MLFlowRepository(SientiaMonitoring): model_name (str): The name of the model to download from the registry. metadata (dict[str, Any]): Metadata used for structured logging. flavor (str): Model flavor ('pyfunc', 'sklearn', 'joblib') - artifact_path (str | None): Path to compressed artifacts if model is compressed Returns: - mlflow.pyfunc.PyFuncModel: The loaded predictive model. + tuple[Any, str | None]: The loaded predictive model and the local + directory its artifact was downloaded to, `None` when nothing + was downloaded here. Notes: - The model is fetched from the "Production" stage of the MLflow Model Registry. - Warnings during the model loading process are suppressed. - - The artifact's `code` directory, when present, is put on - `sys.path` before the load, whatever the flavor. `sklearn` and - `pyfunc` are therefore resolved to a local directory first and the - loader is handed that directory instead of the URI. + - The MLflow flavor loaders read the model URI directly and fetch + only what they need. Nothing is pre-downloaded here, so the local + directory returned alongside the model is `None`; putting the + artifact's `code` directory on `sys.path` belongs to + `download_artifact_code_dir`, which `download_model` runs first. - With `flavor='joblib'` no MLflow flavor loader is called: the `prediction_model` artifact is downloaded and deserialized with - `joblib.load`, which needs no `MLmodel` manifest. + `joblib.load`, which needs no `MLmodel` manifest. That path does + return its local directory, since it downloads by necessity. """ if flavor not in ('pyfunc', 'sklearn', 'joblib'): raise ValueError(INVALID_FLAVOR_MESSAGE) - if flavor == 'joblib': - model, _local_dir = await self._load_joblib_artifact( - model_name=model_name, - artifact_name='prediction_model', - operation_type='load_predict_model', - metadata=metadata, - ) - return model + model_uri = None if flavor == 'joblib' else f'models:/{model_name}/production' - model_uri = f'models:/{model_name}/production' - self.info(f'Loading prediction model {model_name} from {model_uri}') + if model_uri is not None: + self.info(f'Loading prediction model {model_name} from {model_uri}') core_labels = self.get_core_labels(metadata, operation_type='load_predict_model') start_time = time.time() try: - local_path = self._localize_model_uri(model_uri, flavor, metadata) - - if flavor == 'pyfunc': - model = mlflow.pyfunc.load_model(local_path) + if flavor == 'joblib': + model, local_dir = await self._download_and_load_pickle( + model_name=model_name, + artifact_name='prediction_model', + metadata=metadata, + ) + elif flavor == 'pyfunc': + model, local_dir = mlflow.pyfunc.load_model(model_uri), None else: - model = mlflow.sklearn.load_model(local_path) + model, local_dir = mlflow.sklearn.load_model(model_uri), None except Exception as e: await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels) raise e await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels) await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels) - return model + return model, local_dir async def load_transform_model( self, model_name: str, metadata: dict[str, Any], flavor: str = 'sklearn' - ) -> Any: + ) -> tuple[Any, str | None]: """ Load the latest Production version of a transformation model. @@ -712,24 +690,27 @@ class MLFlowRepository(SientiaMonitoring): model_name (str): The name of the model to download. metadata (dict[str, Any]): Metadata used for structured logging. flavor (str): Model flavor ('sklearn', 'pyfunc', 'joblib') - artifact_path (str | None): Path to compressed artifacts if model is compressed Returns: - Any: The loaded model object, depending on the flavor used. + tuple[Any, str | None]: The loaded model object, depending on the + flavor used, and the local directory its artifact was downloaded + to, `None` when nothing was downloaded here. Raises: Exception: If the model run ID or URI cannot be retrieved, or if the model cannot be loaded. Notes: - - The artifact's `code` directory, when present, is put on - `sys.path` before the load, whatever the flavor. `sklearn` and - `pyfunc` are therefore resolved to a local directory first and the - loader is handed that directory instead of the URI. + - The MLflow flavor loaders read the model URI directly and fetch + only what they need. Nothing is pre-downloaded here, so the local + directory returned alongside the model is `None`; putting the + artifact's `code` directory on `sys.path` belongs to + `download_artifact_code_dir`, which `download_model` runs first. - With `flavor='joblib'` no MLflow flavor loader is called and no model URI is built: the `data_model` artifact of the resolved Production run is downloaded and deserialized with `joblib.load`, - which needs no `MLmodel` manifest. + which needs no `MLmodel` manifest. That path does return its local + directory, since it downloads by necessity. """ if flavor not in ('sklearn', 'pyfunc', 'joblib'): @@ -737,36 +718,36 @@ class MLFlowRepository(SientiaMonitoring): latest_production_id = self.get_model_run_id(model_name=model_name, stage='Production') - if flavor == 'joblib': - model, _local_dir = await self._load_joblib_artifact( - model_name=model_name, - artifact_name='data_model', - operation_type='load_transform_model', - metadata=metadata, - run_id=latest_production_id, - ) - return model + model_uri = ( + None + if flavor == 'joblib' + else self.get_model_uri(latest_production_id, prediction=False) + ) - model_uri = self.get_model_uri(latest_production_id, prediction=False) - - self.info(f'Loading data model {model_name} from {model_uri}') + if model_uri is not None: + self.info(f'Loading data model {model_name} from {model_uri}') core_labels = self.get_core_labels(metadata, operation_type='load_transform_model') start_time = time.time() try: - local_path = self._localize_model_uri(model_uri, flavor, metadata) - - if flavor == 'sklearn': - model = mlflow.sklearn.load_model(local_path) + if flavor == 'joblib': + model, local_dir = await self._download_and_load_pickle( + model_name=model_name, + artifact_name='data_model', + metadata=metadata, + run_id=latest_production_id, + ) + elif flavor == 'sklearn': + model, local_dir = mlflow.sklearn.load_model(model_uri), None else: - model = mlflow.pyfunc.load_model(local_path) + model, local_dir = mlflow.pyfunc.load_model(model_uri), None except Exception as e: await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels) raise e await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels) await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels) - return model + return model, local_dir async def download_model( self, @@ -787,17 +768,26 @@ class MLFlowRepository(SientiaMonitoring): load_wrapper (bool): Whether to load wrapper Returns: - tuple[Any, str | None]: Model object and optional artifact path. + tuple[Any, str | None]: Model object and the local directory its + artifact was downloaded to. Notes: - - `flavor='joblib'` returns the local directory the artifact was - downloaded to as the artifact path, because the retrain path needs - it to re-log the artifact's `code` directory. Every other flavor - returns `None` unless the wrapper branch was taken. - - The wrapper branch downloads before it loads, so it puts the - artifact's `code` directory on `sys.path` itself. The two plain - branches delegate, and the code directory is handled inside - `load_predict_model` / `load_transform_model`. + - Single owner of the artifact `code` directory, for every flavor + and both branches: `download_artifact_code_dir` runs first, before + any model is loaded, and is the only place that touches + `sys.path`. It downloads nothing unless the run's artifact really + carries a `code` directory, and even then downloads only that + directory — never the model's weights. + - The directory it downloads into is also what this method returns + alongside the model: it is the artifact directory the retrain path + needs to re-log `code`. It falls back to whatever the loader + downloaded on its own, which is a real artifact directory for + `joblib` and for the wrapper branch, and `None` for the flavor + loaders, which stream from the model URI without downloading. + - Apart from that shared first step this method does not branch on + the flavor. `load_predict_model` and `load_transform_model` own + every flavor, `joblib` included. The only branch left is + `load_wrapper`, which is orthogonal to the flavor. """ self.info( @@ -807,47 +797,36 @@ class MLFlowRepository(SientiaMonitoring): if model_type not in ['predict', 'transform']: raise ValueError("Invalid model_type. Use 'predict' or 'transform'.") - artifact_path = None + target = 'prediction_model' if model_type == 'predict' else 'data_model' + + code_artifact_dir = await self.download_artifact_code_dir( + model_name, target, flavor, metadata + ) if load_wrapper: self.info(f'Loading wrapper for {model_type} model {model_name} with flavor {flavor}') - target = 'prediction_model' if model_type == 'predict' else 'data_model' - artifact_path = await self.dowload_artifacts(model_name, metadata, target) self.info( f'Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}' ) - self.prepend_artifact_code_dir(artifact_path, 'pyfunc', metadata) - raw_model = mlflow.pyfunc.load_model(artifact_path) model = raw_model._model_impl.python_model self.debug( f'Model wrapper loaded: {model.__class__.__name__}:{model.__dict__}', metadata ) - elif flavor == 'joblib': - target = 'prediction_model' if model_type == 'predict' else 'data_model' - operation_type = ( - 'load_predict_model' if model_type == 'predict' else 'load_transform_model' - ) - return await self._load_joblib_artifact( - model_name=model_name, - artifact_name=target, - operation_type=operation_type, - metadata=metadata, - ) + return model, code_artifact_dir or artifact_path + + if model_type == 'predict': + model, local_dir = await self.load_predict_model(model_name, metadata, flavor) else: - if model_type == 'predict': - model = await self.load_predict_model(model_name, metadata, flavor) + model, local_dir = await self.load_transform_model(model_name, metadata, flavor) - else: - model = await self.load_transform_model(model_name, metadata, flavor) - - return model, artifact_path + return model, code_artifact_dir or local_dir """ Functions related to data format @@ -896,11 +875,18 @@ class MLFlowRepository(SientiaMonitoring): raise ValueError(f'{message}. Unable to parse given date format: {e}') from e elif index_type == datetime or index_type == pd.Timestamp: - index = data.index - if hasattr(index, 'tz') and index.tz is None: - data.index = index.tz_localize('UTC') # type: ignore[attr-defined] + # Timestamp-like elements do not guarantee a DatetimeIndex: an object dtype + # index holds them just as well, and only DatetimeIndex exposes `tz`, + # `tz_localize` and `strftime`. Converting first keeps the branch typed and + # covers the object dtype case instead of failing on the missing attribute. + datetime_index = ( + index if isinstance(index, pd.DatetimeIndex) else pd.DatetimeIndex(index) + ) - data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) # type: ignore[attr-defined] + if datetime_index.tz is None: + datetime_index = datetime_index.tz_localize('UTC') + + data.index = datetime_index.strftime(DATETIME_FORMAT_WITH_TZ) else: raise ValueError(f'{message}. Got {index_type}.') diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 99f6194..38fab72 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -69,11 +69,6 @@ def mlflow_repository(): @pytest.fixture def mlflow(): with patch('laborious.utils.repository.model_repository.mlflow') as mlflow: - # Every URI load resolves the artifact to a local directory before calling the flavor - # loader, and `os.path` refuses a MagicMock. Echoing the URI back keeps the loader's - # argument readable and points the code-directory lookup at a path that does not exist, - # so no test touches `sys.path` unless it builds a real artifact directory. - mlflow.artifacts.download_artifacts.side_effect = lambda artifact_uri: str(artifact_uri) yield mlflow @@ -253,7 +248,7 @@ def test_check_artifact_exists_true(mlflow_repository): ) assert result is True - mlflow_repository.client.list_artifacts.assert_called_once_with('run_id') + mlflow_repository.client.list_artifacts.assert_called_once_with('run_id', None) def test_check_artifact_exists_false(mlflow_repository): @@ -265,7 +260,19 @@ def test_check_artifact_exists_false(mlflow_repository): ) assert result is False - mlflow_repository.client.list_artifacts.assert_called_once_with('run_id') + mlflow_repository.client.list_artifacts.assert_called_once_with('run_id', None) + + +def test_check_artifact_exists_nested_lists_the_parent(mlflow_repository): + artifact = MagicMock(path='data_model/code') + mlflow_repository.client.list_artifacts.return_value = [artifact] + + result = mlflow_repository.check_artifact_exists( + 'run_id', 'data_model/code', metadata['metadata'] + ) + + assert result is True + mlflow_repository.client.list_artifacts.assert_called_once_with('run_id', 'data_model') @pytest.mark.asyncio @@ -440,9 +447,10 @@ def test_get_experiment_create_error(mlflow, mlflow_repository): @pytest.mark.asyncio async def test_load_predict_model_sklearn(mlflow, mlflow_repository): - result = await mlflow_repository.load_predict_model('test_model', {}, 'sklearn') + model, local_dir = await mlflow_repository.load_predict_model('test_model', {}, 'sklearn') - assert result == mlflow.sklearn.load_model.return_value + assert model == mlflow.sklearn.load_model.return_value + assert local_dir is None mlflow.sklearn.load_model.assert_called_once_with('models:/test_model/production') mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_READ_LAG, ANY) mlflow_repository.emit_metric.assert_called_once_with( @@ -452,8 +460,10 @@ async def test_load_predict_model_sklearn(mlflow, mlflow_repository): @pytest.mark.asyncio async def test_load_predict_model_pyfunc(mlflow, mlflow_repository): - result = await mlflow_repository.load_predict_model('test_model', {}, 'pyfunc') - assert result == mlflow.pyfunc.load_model.return_value + model, local_dir = await mlflow_repository.load_predict_model('test_model', {}, 'pyfunc') + + assert model == mlflow.pyfunc.load_model.return_value + assert local_dir is None mlflow.pyfunc.load_model.assert_called_once_with('models:/test_model/production') mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_READ_LAG, ANY) mlflow_repository.emit_metric.assert_called_once_with( @@ -473,8 +483,6 @@ async def test_load_predict_model_error(mlflow_repository): mlflow_repository.observe_lag.assert_not_called() -# A real string, because the transform load now resolves its URI to a local directory before -# calling the loader, and that resolution goes through `os.path`. TRANSFORM_MODEL_URI = 'runs:/run-123/data_model' @@ -492,11 +500,12 @@ async def test_load_transform_model_sklearn(mlflow, mlflow_repository): mlflow_repository.get_model_run_id = MagicMock() mlflow_repository.get_model_uri = MagicMock(return_value=TRANSFORM_MODEL_URI) - result = await mlflow_repository.load_transform_model('test_model', {}, 'sklearn') + model, local_dir = await mlflow_repository.load_transform_model('test_model', {}, 'sklearn') validate_common_load_transform_model_mocks(mlflow_repository, 'test_model') - assert result == mlflow.sklearn.load_model.return_value + assert model == mlflow.sklearn.load_model.return_value + assert local_dir is None mlflow.sklearn.load_model.assert_called_once_with(TRANSFORM_MODEL_URI) mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_READ_LAG, ANY) @@ -510,11 +519,12 @@ async def test_load_transform_model_pyfunc(mlflow, mlflow_repository): mlflow_repository.get_model_run_id = MagicMock() mlflow_repository.get_model_uri = MagicMock(return_value=TRANSFORM_MODEL_URI) - result = await mlflow_repository.load_transform_model('test_model', {}, 'pyfunc') + model, local_dir = await mlflow_repository.load_transform_model('test_model', {}, 'pyfunc') validate_common_load_transform_model_mocks(mlflow_repository, 'test_model') - assert result == mlflow.pyfunc.load_model.return_value + assert model == mlflow.pyfunc.load_model.return_value + assert local_dir is None mlflow.pyfunc.load_model.assert_called_once_with(TRANSFORM_MODEL_URI) mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_READ_LAG, ANY) @@ -555,6 +565,7 @@ async def test_download_model_invalid_model_type(mlflow_repository): 'model_type', [('predict', 'prediction_model'), ('transform', 'data_model')] ) async def test_download_model_load_wrapper(mlflow, mlflow_repository, model_type, tmp_path): + mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) mlflow_repository.dowload_artifacts = AsyncMock(return_value=str(tmp_path)) result = await mlflow_repository.download_model('test_model', {}, model_type[0], 'pyfunc', True) @@ -573,7 +584,8 @@ async def test_download_model_load_wrapper(mlflow, mlflow_repository, model_type @pytest.mark.asyncio async def test_download_model_predict(mlflow_repository): - mlflow_repository.load_predict_model = AsyncMock() + mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) + mlflow_repository.load_predict_model = AsyncMock(return_value=('model', 'local_dir')) mlflow_repository.load_transform_model = AsyncMock() result = await mlflow_repository.download_model('test_model', {}, 'predict', 'pyfunc', False) @@ -581,20 +593,59 @@ async def test_download_model_predict(mlflow_repository): mlflow_repository.load_predict_model.assert_called_once_with('test_model', {}, 'pyfunc') mlflow_repository.load_transform_model.assert_not_called() - assert result == (mlflow_repository.load_predict_model.return_value, None) + assert result == ('model', 'local_dir') @pytest.mark.asyncio async def test_download_model_transform(mlflow_repository): + mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) mlflow_repository.load_predict_model = AsyncMock() - mlflow_repository.load_transform_model = AsyncMock() + mlflow_repository.load_transform_model = AsyncMock(return_value=('model', 'local_dir')) result = await mlflow_repository.download_model('test_model', {}, 'transform', 'pyfunc', False) mlflow_repository.load_predict_model.assert_not_called() mlflow_repository.load_transform_model.assert_called_once_with('test_model', {}, 'pyfunc') - assert result == (mlflow_repository.load_transform_model.return_value, None) + assert result == ('model', 'local_dir') + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + 'model_type,artifact_name', [('predict', 'prediction_model'), ('transform', 'data_model')] +) +async def test_download_model_prepares_the_code_dir_first( + mlflow_repository, model_type, artifact_name +): + """The `code` step runs before any loader, for the artifact matching the model type.""" + calls = [] + mlflow_repository.download_artifact_code_dir = AsyncMock( + side_effect=lambda *_a, **_k: calls.append('code') or 'code-artifact-dir' + ) + loader = AsyncMock(side_effect=lambda *_a: calls.append('load') or ('model', None)) + mlflow_repository.load_predict_model = loader + mlflow_repository.load_transform_model = loader + + result = await mlflow_repository.download_model('test_model', {}, model_type, 'sklearn', False) + + mlflow_repository.download_artifact_code_dir.assert_called_once_with( + 'test_model', artifact_name, 'sklearn', {} + ) + assert calls == ['code', 'load'] + assert result == ('model', 'code-artifact-dir') + + +@pytest.mark.asyncio +async def test_download_model_wrapper_returns_the_code_artifact_dir(mlflow, mlflow_repository): + """With a `code` directory downloaded, the wrapper branch returns it over its own download.""" + mlflow_repository.download_artifact_code_dir = AsyncMock(return_value='code-artifact-dir') + mlflow_repository.dowload_artifacts = AsyncMock(return_value='wrapper-dir') + + _model, artifact_dir = await mlflow_repository.download_model( + 'test_model', {}, 'predict', 'pyfunc', True + ) + + assert artifact_dir == 'code-artifact-dir' def test_detect_and_parse_datetime_index_empty(mlflow_repository): @@ -1803,9 +1854,10 @@ async def test_load_predict_model_joblib_flavor_loads_pickle_directly( tmp_path, 'MLmodel', 'model.pkl' ) - result = await mlflow_repository.load_predict_model('test_model', {}, 'joblib') + model, local_dir = await mlflow_repository.load_predict_model('test_model', {}, 'joblib') - assert result is joblib_mock.load.return_value + assert model is joblib_mock.load.return_value + assert local_dir == str(tmp_path) mlflow_repository.get_model_run_id.assert_called_once_with( model_name='test_model', stage='Production' ) @@ -1827,9 +1879,10 @@ async def test_load_transform_model_joblib_flavor_loads_pickle_directly( tmp_path, 'MLmodel', 'model.pkl' ) - result = await mlflow_repository.load_transform_model('test_model', {}, 'joblib') + model, local_dir = await mlflow_repository.load_transform_model('test_model', {}, 'joblib') - assert result is joblib_mock.load.return_value + assert model is joblib_mock.load.return_value + assert local_dir == str(tmp_path) mlflow_repository.client.download_artifacts.assert_called_once_with('run-123', 'data_model') joblib_mock.load.assert_called_once_with(str(tmp_path / 'model.pkl')) mlflow_repository.get_model_uri.assert_not_called() @@ -1903,6 +1956,7 @@ async def test_load_transform_model_flavor_failure_propagates( @patch(JOBLIB_PATH) @pytest.mark.asyncio async def test_download_model_wrapper_failure_propagates(joblib_mock, mlflow, mlflow_repository): + mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) mlflow_repository.dowload_artifacts = AsyncMock() loader_error = mlflow_lib.exceptions.MlflowException('wrapper boom') mlflow.pyfunc.load_model.side_effect = loader_error @@ -2171,60 +2225,110 @@ def test_prepend_artifact_code_dir_does_not_duplicate( assert sys.path.count(code_dir) == 1 +def stub_remote_code_dir(mlflow_repository, tmp_path, artifact_name, exists=True): + """Point the client at a run whose artifact does — or does not — carry `code`. + + Returns the local `code` directory the download would produce, already on disk. + """ + artifact_dir = tmp_path / artifact_name + artifact_dir.mkdir() + code_dir = build_code_dir(artifact_dir) + + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.client.list_artifacts.return_value = ( + [MagicMock(path=f'{artifact_name}/code')] if exists else [MagicMock(path=artifact_name)] + ) + mlflow_repository.client.download_artifacts.return_value = code_dir + + return code_dir + + +@pytest.mark.asyncio +async def test_download_artifact_code_dir_downloads_only_code( + mlflow_repository, tmp_path, preserved_sys_path +): + """Only `/code` travels — never the artifact's weights.""" + code_dir = stub_remote_code_dir(mlflow_repository, tmp_path, 'prediction_model') + + result = await mlflow_repository.download_artifact_code_dir( + 'test_model', 'prediction_model', 'sklearn', {} + ) + + mlflow_repository.client.list_artifacts.assert_called_once_with('run-123', 'prediction_model') + mlflow_repository.client.download_artifacts.assert_called_once_with( + 'run-123', 'prediction_model/code' + ) + assert result == str(tmp_path / 'prediction_model') + assert sys.path[0] == code_dir + + +@pytest.mark.asyncio +async def test_download_artifact_code_dir_without_remote_code_downloads_nothing( + mlflow_repository, tmp_path, preserved_sys_path +): + stub_remote_code_dir(mlflow_repository, tmp_path, 'data_model', exists=False) + before = list(sys.path) + + result = await mlflow_repository.download_artifact_code_dir( + 'test_model', 'data_model', 'pyfunc', {} + ) + + assert result is None + mlflow_repository.client.download_artifacts.assert_not_called() + assert sys.path == before + + @pytest.mark.asyncio @pytest.mark.parametrize('flavor', FLAVOR_LOADERS) -async def test_load_predict_model_puts_code_dir_on_sys_path_before_the_loader( +async def test_download_model_puts_code_dir_on_sys_path_before_the_loader( flavor, mlflow, mlflow_repository, tmp_path, preserved_sys_path ): """MLflow extends `sys.path` only when `MLmodel` declares `code`; the directory decides here.""" - code_dir = build_code_dir(tmp_path) - local_dir = build_artifact_dir(tmp_path, 'MLmodel', 'model.pkl') - mlflow.artifacts.download_artifacts.side_effect = None - mlflow.artifacts.download_artifacts.return_value = local_dir - sys_path_at_load = [] - getattr(mlflow, flavor).load_model.side_effect = observe_sys_path(sys_path_at_load) - - await mlflow_repository.load_predict_model('test_model', {}, flavor) - - mlflow.artifacts.download_artifacts.assert_called_once_with( - artifact_uri='models:/test_model/production' - ) - getattr(mlflow, flavor).load_model.assert_called_once_with(local_dir) - assert sys_path_at_load[0] == code_dir - - -@pytest.mark.asyncio -@pytest.mark.parametrize('flavor', FLAVOR_LOADERS) -async def test_load_transform_model_puts_code_dir_on_sys_path_before_the_loader( - flavor, mlflow, mlflow_repository, tmp_path, preserved_sys_path -): - code_dir = build_code_dir(tmp_path) - local_dir = build_artifact_dir(tmp_path, 'MLmodel', 'model.pkl') - mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + code_dir = stub_remote_code_dir(mlflow_repository, tmp_path, 'prediction_model') mlflow_repository.get_model_uri = MagicMock(return_value=TRANSFORM_MODEL_URI) - mlflow.artifacts.download_artifacts.side_effect = None - mlflow.artifacts.download_artifacts.return_value = local_dir sys_path_at_load = [] getattr(mlflow, flavor).load_model.side_effect = observe_sys_path(sys_path_at_load) - await mlflow_repository.load_transform_model('test_model', {}, flavor) + _model, artifact_dir = await mlflow_repository.download_model( + 'test_model', {}, 'predict', flavor, False + ) - mlflow.artifacts.download_artifacts.assert_called_once_with(artifact_uri=TRANSFORM_MODEL_URI) - getattr(mlflow, flavor).load_model.assert_called_once_with(local_dir) assert sys_path_at_load[0] == code_dir + assert artifact_dir == str(tmp_path / 'prediction_model') @pytest.mark.asyncio @pytest.mark.parametrize('flavor', FLAVOR_LOADERS) -async def test_load_predict_model_without_code_dir_leaves_sys_path_alone( - flavor, mlflow, mlflow_repository, tmp_path, preserved_sys_path +async def test_load_predict_model_loads_from_the_uri_without_downloading( + flavor, mlflow, mlflow_repository, preserved_sys_path ): - mlflow.artifacts.download_artifacts.side_effect = None - mlflow.artifacts.download_artifacts.return_value = build_artifact_dir(tmp_path, 'MLmodel') + """The flavor loaders fetch what they need themselves; nothing is pre-downloaded.""" before = list(sys.path) - await mlflow_repository.load_predict_model('test_model', {}, flavor) + _model, local_dir = await mlflow_repository.load_predict_model('test_model', {}, flavor) + mlflow.artifacts.download_artifacts.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + getattr(mlflow, flavor).load_model.assert_called_once_with('models:/test_model/production') + assert local_dir is None + assert sys.path == before + + +@pytest.mark.asyncio +@pytest.mark.parametrize('flavor', FLAVOR_LOADERS) +async def test_load_transform_model_loads_from_the_uri_without_downloading( + flavor, mlflow, mlflow_repository, preserved_sys_path +): + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.get_model_uri = MagicMock(return_value=TRANSFORM_MODEL_URI) + before = list(sys.path) + + _model, local_dir = await mlflow_repository.load_transform_model('test_model', {}, flavor) + + mlflow.artifacts.download_artifacts.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + getattr(mlflow, flavor).load_model.assert_called_once_with(TRANSFORM_MODEL_URI) + assert local_dir is None assert sys.path == before @@ -2232,8 +2336,8 @@ async def test_load_predict_model_without_code_dir_leaves_sys_path_alone( async def test_download_model_load_wrapper_puts_code_dir_on_sys_path( mlflow, mlflow_repository, tmp_path, preserved_sys_path ): - """The wrapper branch downloads itself, so it is the one that has to add the directory.""" - code_dir = build_code_dir(tmp_path) + """The wrapper branch loads a downloaded artifact, so `code` has to be in place first too.""" + code_dir = stub_remote_code_dir(mlflow_repository, tmp_path, 'prediction_model') mlflow_repository.dowload_artifacts = AsyncMock(return_value=str(tmp_path)) sys_path_at_load = [] mlflow.pyfunc.load_model.side_effect = observe_sys_path(sys_path_at_load) @@ -2323,18 +2427,56 @@ async def test_download_model_joblib_returns_download_dir( @patch(JOBLIB_PATH) @pytest.mark.asyncio -async def test_download_model_sklearn_returns_no_artifact_path( - joblib_mock, mlflow, mlflow_repository +async def test_download_model_sklearn_returns_the_code_artifact_dir( + joblib_mock, mlflow, mlflow_repository, tmp_path, preserved_sys_path ): + """The sklearn loader downloads nothing, so the `code` step is the only source of a directory.""" + stub_remote_code_dir(mlflow_repository, tmp_path, 'prediction_model') + model, artifact_path = await mlflow_repository.download_model( 'test_model', {}, 'predict', 'sklearn' ) assert model is mlflow.sklearn.load_model.return_value - assert artifact_path is None + assert artifact_path == str(tmp_path / 'prediction_model') joblib_mock.load.assert_not_called() +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_download_model_sklearn_without_code_returns_no_dir( + joblib_mock, mlflow, mlflow_repository, tmp_path, preserved_sys_path +): + stub_remote_code_dir(mlflow_repository, tmp_path, 'prediction_model', exists=False) + + _model, artifact_path = await mlflow_repository.download_model( + 'test_model', {}, 'predict', 'sklearn' + ) + + assert artifact_path is None + mlflow_repository.client.download_artifacts.assert_not_called() + joblib_mock.load.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('model_type', 'loader'), + [('predict', 'load_predict_model'), ('transform', 'load_transform_model')], +) +async def test_download_model_delegates_every_flavor_to_the_loaders( + mlflow_repository, model_type, loader +): + """`download_model` owns no flavor branch: joblib delegates like the other two.""" + mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) + delegate = AsyncMock(return_value=('model', 'local-dir')) + setattr(mlflow_repository, loader, delegate) + + result = await mlflow_repository.download_model('test_model', {}, model_type, 'joblib') + + delegate.assert_called_once_with('test_model', {}, 'joblib') + assert result == ('model', 'local-dir') + + @patch(JOBLIB_PATH) @pytest.mark.asyncio async def test_download_model_transform_joblib_downloads_data_model( @@ -2401,7 +2543,6 @@ def test_module_defines_no_joblib_fallback_routine(): '_find_model_pickle', '_load_model_pickle', '_download_and_load_pickle', - '_load_joblib_artifact', } for handler in ast.walk(ast.parse(source)): @@ -2557,7 +2698,7 @@ async def test_joblib_flavor_real_round_trip_with_code_dir( mlflow_repository.client.download_artifacts.return_value = str(tmp_path) try: - loaded = await mlflow_repository.load_predict_model('test_model', {}, 'joblib') + loaded, _local_dir = await mlflow_repository.load_predict_model('test_model', {}, 'joblib') assert loaded.predict([1.0, 2.0]) == [3.0, 4.0] assert code_dir in sys.path From 086b65c45f25b0d5a1b2d0e7f5f93b724b8eb163 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 1 Sep 2026 13:54:51 -0300 Subject: [PATCH 32/64] QTZPOC-20 QTZPOC-20: Make metadata parameter optional in check_artifact_exists method. --- laborious/utils/repository/model_repository.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 76fbb4e..15b4a10 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -256,7 +256,7 @@ class MLFlowRepository(SientiaMonitoring): return run_info.data.params def check_artifact_exists( - self, run_id: str, artifact_path: str, metadata: dict[str, Any] + self, run_id: str, artifact_path: str, metadata: dict[str, Any] | None = None ) -> bool: """ Check if an artifact exists in the MLflow Model Registry. @@ -271,6 +271,9 @@ class MLFlowRepository(SientiaMonitoring): run_id (str): Run identifier to inspect. artifact_path (str): Path to the artifact to check, relative to the run's artifact root, e.g. 'data_model' or 'data_model/code'. + metadata (dict[str, Any] | None): Metadata used for structured + logging. Optional, so callers whose own metadata is optional — + `download_artifact_code_dir` — can forward it as is. Returns: bool: True if the artifact exists, False otherwise. From 09027bc93bb2ecbf42dd4031f9649a7541f0cf98 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 3 Sep 2026 12:04:47 -0300 Subject: [PATCH 33/64] QTZPOC-20 QTZPOC-20: Add skip_predict_timestamp flag to model_config to prevent timestamp reinsertion in prediction data, and document other ad-hoc behavioral flags. --- README.md | 5 +++++ docs/model-flavors.md | 10 ++++++++++ laborious/activities/mlflow.py | 12 ++++++++---- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3ce886b..2b0af00 100644 --- a/README.md +++ b/README.md @@ -1110,6 +1110,11 @@ Legacy MinIO object layout (relative key): `joblib`. The flavor is configuration, never discovered from the artifact, and there is no fallback between flavors: a failing loader propagates. +`model_config` also carries ad-hoc behavioral flags, unrelated to flavor, e.g. +`skip_predict_timestamp` (default `False`): when `True`, `request_predict` does not reinsert the +`timestamp` column derived from the index into the transformed data before calling predict. Full +list: [docs/model-flavors.md § Other `model_config` flags](docs/model-flavors.md#other-model_config-flags). + `joblib` reads the artifact directly — download, `/model.pkl`, `joblib.load` — and writes on retrain (`model.pkl` + `code/`, no `MLmodel`, so only this runtime reads it back). diff --git a/docs/model-flavors.md b/docs/model-flavors.md index 9efd64c..6b70ca4 100644 --- a/docs/model-flavors.md +++ b/docs/model-flavors.md @@ -18,6 +18,16 @@ reads the artifact's `MLmodel` manifest to guess it. `predict_flavor` drives `load_predict_model` (artifact `prediction_model`); `transform_flavor` drives `load_transform_model` (artifact `data_model`). Both default to `sklearn` when the key is absent. +## Other `model_config` flags + +Ad-hoc behavioral overrides, read directly with `.get(key, default)` — not part of the model +document written by import (`model-import.md` § 12), and not validated against a schema: + +| Key | Default | Effect | +|---|---|---| +| `skip_transform` | `False` | `fit_models` (retrain) skips calling `transform` before fitting. | +| `skip_predict_timestamp` | `False` | `request_predict` skips reinserting the `timestamp` column (from the index) into the transformed data before calling predict. | + ## Accepted values | Flavor | Read | Write (retrain) | How it loads | diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index f4d42a3..d075508 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -237,6 +237,9 @@ class MLFlow(MinioManager): - data (dict): Transformed data for prediction - model_name (str): Name of the MLFlow model to use - model_retention (int): Model retention period in minutes + Optional keys (in model_config): + - skip_predict_timestamp (bool): If True, don't reinsert the + timestamp column from the index before predicting. Default False. Returns: dict: Prediction results from MLFlow model @@ -258,10 +261,11 @@ class MLFlow(MinioManager): # Convert numpy.nan to None for model compatibility data.replace(np.nan, None, inplace=True) - data['timestamp'] = data.index - data['timestamp'] = to_datetime( - data['timestamp'], format=DATETIME_FORMAT_WITH_TZ - ).dt.strftime(DATETIME_FORMAT) + if not model_config.get('skip_predict_timestamp', False): + data['timestamp'] = data.index + data['timestamp'] = to_datetime( + data['timestamp'], format=DATETIME_FORMAT_WITH_TZ + ).dt.strftime(DATETIME_FORMAT) # Request prediction from MLFlow model response_data = await self.model_monitoring_repository.predict( From 220ee5509c3cfee39e375f79e5c03d680d653600 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 4 Sep 2026 09:53:08 -0300 Subject: [PATCH 34/64] QTZPOC-20 QTZPOC-20: Remove hashlib from development requirements. --- requirements-dev.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 51e84b9..476136a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,6 +22,4 @@ testcontainers[postgres,minio,mongodb]>=4.15.0 # PostgreSQL, MinIO and MongoDB c ipython>=8.12.0 # Enhanced Python shell ipdb>=0.13.13 # IPython debugger ipykernel==6.30.1 # IPython kernel for Jupyter notebooks -python-dotenv>=1.0.0 # .env loading for scripts/import_model_manual_run.py - -hashlib \ No newline at end of file +python-dotenv>=1.0.0 # .env loading for scripts/import_model_manual_run.py \ No newline at end of file From 6a412147ee158c56d1409a45588a8b5659e06c00 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 4 Sep 2026 10:02:26 -0300 Subject: [PATCH 35/64] QTZPOC-20 QTZPOC-20: Add pynacl==1.5.0 to requirements-light.txt --- requirements-light.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-light.txt b/requirements-light.txt index f2481d4..8bfedaa 100644 --- a/requirements-light.txt +++ b/requirements-light.txt @@ -18,3 +18,4 @@ scipy<1.14.0 scikit-learn==1.4.2 pycaret==3.3.2 joblib +pynacl==1.5.0 From bde614e2a68d6f7c7e20b9f0e71226f82d9f7692 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 4 Sep 2026 10:54:23 -0300 Subject: [PATCH 36/64] QTZPOC-20 QTZPOC-20: Refactor model repository tests for improved assertion logic and exception handling, including correcting pytest.raises exception access and refining mock verifications. --- .../utils/repository/test_model_repository.py | 129 ++++++++++-------- 1 file changed, 69 insertions(+), 60 deletions(-) diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 38fab72..95c911c 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -105,11 +105,10 @@ def test_get_model_run_id_not_registered_models(mlflow_repository): with pytest.raises(mlflow_lib.exceptions.MlflowException) as e: mlflow_repository.get_model_run_id('test') - mlflow_repository.client.search_registered_models.assert_called_once_with( - filter_string="name='test'" - ) - - assert str(e.value) == "Model 'test' not found in the Model Registry." + mlflow_repository.client.search_registered_models.assert_called_once_with( + filter_string="name='test'" + ) + assert str(e.value) == "Model 'test' not found in the Model Registry." def test_get_model_run_id_not_stage_versions(mlflow_repository): @@ -124,14 +123,13 @@ def test_get_model_run_id_not_stage_versions(mlflow_repository): with pytest.raises(mlflow_lib.exceptions.MlflowException) as e: mlflow_repository.get_model_run_id('test') - mlflow_repository.client.search_registered_models.assert_called_once_with( - filter_string="name='test'" - ) - mlflow_repository.client.search_model_versions.assert_called_once_with( - filter_string="name='test'" - ) - - assert str(e.value) == "Model 'test' in stage 'Production' not found in the Model Registry." + mlflow_repository.client.search_registered_models.assert_called_once_with( + filter_string="name='test'" + ) + mlflow_repository.client.search_model_versions.assert_called_once_with( + filter_string="name='test'" + ) + assert str(e.value) == "Model 'test' in stage 'Production' not found in the Model Registry." def test_get_model_run_id_success(mlflow_repository): @@ -231,7 +229,7 @@ def test_get_experiment_none_not_create(mlflow, mlflow_repository): with pytest.raises(ValueError) as e: mlflow_repository.get_experiment('test', create_if_not_exists=False) - assert str(e) == 'Experiment test not found' + assert str(e.value) == 'Experiment test not found' def test_get_model_params(mlflow, mlflow_repository): @@ -354,10 +352,10 @@ async def test_download_artifacts_error(makedirs, rmtree, path, mlflow_repositor with pytest.raises(ValueError): await mlflow_repository.dowload_artifacts('test', {}, 'path') - mlflow_repository.emit_metric.assert_called_once_with( - metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=ANY - ) - mlflow_repository.observe_lag.assert_not_called() + mlflow_repository.emit_metric.assert_called_once_with( + metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=ANY + ) + mlflow_repository.observe_lag.assert_not_called() @pytest.mark.asyncio @@ -442,7 +440,7 @@ def test_get_experiment_create_error(mlflow, mlflow_repository): with pytest.raises(ValueError) as e: mlflow_repository.get_experiment('test', create_if_not_exists=True) - assert str(e) == 'Experiment test not found after creation, unknown reason' + assert str(e.value) == 'Experiment test not found after creation, unknown reason' @pytest.mark.asyncio @@ -475,12 +473,10 @@ async def test_load_predict_model_pyfunc(mlflow, mlflow_repository): async def test_load_predict_model_error(mlflow_repository): with pytest.raises(ValueError) as e: await mlflow_repository.load_predict_model('test_model', {}, 'invalid') - assert str(e) == INVALID_FLAVOR_MESSAGE - mlflow_repository.emit_metric.assert_called_once_with( - metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=ANY - ) - mlflow_repository.observe_lag.assert_not_called() + assert str(e.value) == INVALID_FLAVOR_MESSAGE + mlflow_repository.emit_metric.assert_not_called() + mlflow_repository.observe_lag.assert_not_called() TRANSFORM_MODEL_URI = 'runs:/run-123/data_model' @@ -540,24 +536,20 @@ async def test_load_transform_model_error(mlflow_repository): with pytest.raises(ValueError) as e: await mlflow_repository.load_transform_model('test_model', {}, 'invalid') - assert str(e) == INVALID_FLAVOR_MESSAGE - mlflow_repository.emit_metric.assert_called_once_with( - metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=ANY - ) - mlflow_repository.observe_lag.assert_not_called() + assert str(e.value) == INVALID_FLAVOR_MESSAGE + mlflow_repository.emit_metric.assert_not_called() + mlflow_repository.observe_lag.assert_not_called() @pytest.mark.asyncio async def test_download_model_invalid_model_type(mlflow_repository): with pytest.raises(ValueError) as e: await mlflow_repository.download_model('test_model', {}, 'invalid', 'sklearn') - assert str(e) == "Invalid model_type. Use 'predict' or 'transform'." - mlflow_repository.emit_metric.assert_called_once_with( - metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=ANY - ) - mlflow_repository.observe_lag.assert_not_called() + assert str(e.value) == "Invalid model_type. Use 'predict' or 'transform'." + mlflow_repository.emit_metric.assert_not_called() + mlflow_repository.observe_lag.assert_not_called() @pytest.mark.asyncio @@ -618,11 +610,18 @@ async def test_download_model_prepares_the_code_dir_first( mlflow_repository, model_type, artifact_name ): """The `code` step runs before any loader, for the artifact matching the model type.""" - calls = [] - mlflow_repository.download_artifact_code_dir = AsyncMock( - side_effect=lambda *_a, **_k: calls.append('code') or 'code-artifact-dir' - ) - loader = AsyncMock(side_effect=lambda *_a: calls.append('load') or ('model', None)) + calls: list[str] = [] + + async def record_code_dir(*_a, **_k): + calls.append('code') + return 'code-artifact-dir' + + async def record_load(*_a): + calls.append('load') + return 'model', None + + mlflow_repository.download_artifact_code_dir = AsyncMock(side_effect=record_code_dir) + loader = AsyncMock(side_effect=record_load) mlflow_repository.load_predict_model = loader mlflow_repository.load_transform_model = loader @@ -890,7 +889,8 @@ async def test_get_cached_operation_invalid_operation(force_memory_release, mlfl await mlflow_repository.get_cached_operation( 'model_name', data, 'invalid', 0, 'sklearn', {} ) - assert str(e) == "Invalid operation. Use 'transform' or 'predict'." + + assert str(e.value) == "Invalid operation. Use 'transform' or 'predict'." @patch('laborious.utils.repository.model_repository.pd.merge') @@ -1149,11 +1149,12 @@ async def test_log_model_error(mlflow_repository): await mlflow_repository.log_model( model_data, 'invalid', 'prediction_model', metadata['metadata'] ) - assert str(e) == INVALID_FLAVOR_MESSAGE - mlflow_repository.emit_metric.assert_called_once_with( - metric_object=metrics.MODEL_WRITE_ERROR_COUNT, tags=ANY - ) - mlflow_repository.observe_lag.assert_not_called() + + assert str(e.value) == INVALID_FLAVOR_MESSAGE + mlflow_repository.emit_metric.assert_called_once_with( + metric_object=metrics.MODEL_WRITE_ERROR_COUNT, tags=ANY + ) + mlflow_repository.observe_lag.assert_not_called() @patch('laborious.utils.repository.model_repository.force_memory_release') @@ -1302,10 +1303,10 @@ async def test_create_new_experiment_error( 'pyfunc', ) - mlflow_repository.emit_metric.assert_called_once_with( - metric_object=metrics.MODEL_WRITE_ERROR_COUNT, tags=ANY - ) - mlflow_repository.observe_lag.assert_not_called() + mlflow_repository.emit_metric.assert_called_once_with( + metric_object=metrics.MODEL_WRITE_ERROR_COUNT, tags=ANY + ) + mlflow_repository.observe_lag.assert_not_called() @pytest.mark.asyncio @@ -1360,24 +1361,32 @@ async def test_update_production_model_by_run_id_error_register_model(mlflow, ml with pytest.raises(ValueError): await mlflow_repository.update_production_model_by_run_id('0', 'test', metadata['metadata']) - mlflow_repository.emit_metric.assert_called_once_with( - metric_object=metrics.MODEL_WRITE_ERROR_COUNT, tags=ANY - ) - mlflow_repository.observe_lag.assert_not_called() + mlflow_repository.emit_metric.assert_called_once_with( + metric_object=metrics.MODEL_WRITE_ERROR_COUNT, tags=ANY + ) + mlflow_repository.observe_lag.assert_not_called() @pytest.mark.asyncio async def test_update_production_model_by_run_id_error_transition_model_version_stage( mlflow, mlflow_repository ): + mlflow_repository.client.get_registered_model.return_value = MagicMock( + latest_versions=[MagicMock(version='1')] + ) mlflow_repository.client.transition_model_version_stage.side_effect = ValueError('error') with pytest.raises(ValueError): await mlflow_repository.update_production_model_by_run_id('0', 'test', metadata['metadata']) - mlflow_repository.emit_metric.assert_called_once_with( - metric_object=metrics.MODEL_WRITE_ERROR_COUNT, tags=ANY - ) - mlflow_repository.observe_lag.assert_not_called() + # register_model succeeded before transition failed, so its WRITE_COUNT/WRITE_LAG + # were already recorded; only the transition step reports as an error. + mlflow_repository.emit_metric.assert_has_calls( + [ + call(metric_object=metrics.MODEL_WRITE_COUNT, tags=ANY), + call(metric_object=metrics.MODEL_WRITE_ERROR_COUNT, tags=ANY), + ] + ) + mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_WRITE_LAG, ANY) @pytest.mark.asyncio @@ -2131,7 +2140,7 @@ def test_load_model_pickle_adds_code_dir_before_load( ): code_dir = build_code_dir(tmp_path) local_dir = build_artifact_dir(tmp_path, 'model.pkl') - sys_path_at_load = [] + sys_path_at_load: list[str] = [] joblib_mock.load.side_effect = lambda _path: sys_path_at_load.extend(sys.path) mlflow_repository._load_model_pickle(local_dir, 'prediction_model') @@ -2286,7 +2295,7 @@ async def test_download_model_puts_code_dir_on_sys_path_before_the_loader( """MLflow extends `sys.path` only when `MLmodel` declares `code`; the directory decides here.""" code_dir = stub_remote_code_dir(mlflow_repository, tmp_path, 'prediction_model') mlflow_repository.get_model_uri = MagicMock(return_value=TRANSFORM_MODEL_URI) - sys_path_at_load = [] + sys_path_at_load: list[str] = [] getattr(mlflow, flavor).load_model.side_effect = observe_sys_path(sys_path_at_load) _model, artifact_dir = await mlflow_repository.download_model( @@ -2339,7 +2348,7 @@ async def test_download_model_load_wrapper_puts_code_dir_on_sys_path( """The wrapper branch loads a downloaded artifact, so `code` has to be in place first too.""" code_dir = stub_remote_code_dir(mlflow_repository, tmp_path, 'prediction_model') mlflow_repository.dowload_artifacts = AsyncMock(return_value=str(tmp_path)) - sys_path_at_load = [] + sys_path_at_load: list[str] = [] mlflow.pyfunc.load_model.side_effect = observe_sys_path(sys_path_at_load) await mlflow_repository.download_model('test_model', {}, 'predict', 'pyfunc', True) From 7b2e90a9f9cb273cfbf3bf8a53b75d85c1a12e73 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 4 Sep 2026 11:15:48 -0300 Subject: [PATCH 37/64] QTZPOC-20 QTZPOC-20: Simplify base64 decoding error handling and improve run name derivation test specificity --- laborious/utils/import_password.py | 5 ++--- tests/laborious/activities/test_model_import_status.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/laborious/utils/import_password.py b/laborious/utils/import_password.py index 5b61e8b..e74bf00 100644 --- a/laborious/utils/import_password.py +++ b/laborious/utils/import_password.py @@ -16,7 +16,6 @@ used as a metric label or written to the import record. from __future__ import annotations import base64 -import binascii from typing import Final from cryptography.exceptions import InvalidTag @@ -55,7 +54,7 @@ def load_envelope_key(raw_key: str | None) -> bytes: raise EnvelopeError('the import password key is not configured') try: key = base64.b64decode(raw_key, validate=True) - except (binascii.Error, ValueError) as error: + except ValueError as error: raise EnvelopeError('the import password key is not valid base64') from error if len(key) != KEY_SIZE: raise EnvelopeError(f'the import password key must be {KEY_SIZE} bytes') @@ -80,7 +79,7 @@ def decrypt_password_envelope(envelope: str, key: bytes) -> str: raise EnvelopeError('the import request carries no password envelope') try: payload = base64.b64decode(envelope, validate=True) - except (binascii.Error, ValueError) as error: + except ValueError as error: raise EnvelopeError('the password envelope is not valid base64') from error if len(payload) < MIN_ENVELOPE_SIZE: raise EnvelopeError('the password envelope is too short to hold a nonce and a tag') diff --git a/tests/laborious/activities/test_model_import_status.py b/tests/laborious/activities/test_model_import_status.py index 29980b2..fcd0efa 100644 --- a/tests/laborious/activities/test_model_import_status.py +++ b/tests/laborious/activities/test_model_import_status.py @@ -193,7 +193,7 @@ def test_run_name_survives_a_version_that_leaves_no_room(): def test_run_name_is_deterministic(): - assert derive_run_name('model', '4') == derive_run_name('model', '4') + assert derive_run_name('model', '4') == 'import-model-v4' def test_run_name_is_never_shorter_than_the_column_floor(): From d257e704657795b7def854fd4bf6603253c39f74 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 4 Sep 2026 11:35:34 -0300 Subject: [PATCH 38/64] QTZPOC-20 QTZPOC-20: Refactor test inputs into named variables for improved readability and debugging. --- .../activities/test_model_import_bundle.py | 47 +++++++++------ .../test_model_import_provisioning.py | 11 ++-- .../activities/test_model_import_status.py | 22 +++++-- tests/laborious/utils/bundle/test_reader.py | 59 +++++++++++++------ .../utils/repository/test_model_repository.py | 11 +++- tests/laborious/utils/test_import_password.py | 7 ++- .../laborious/workflows/test_import_model.py | 51 ++++++++++------ 7 files changed, 141 insertions(+), 67 deletions(-) diff --git a/tests/laborious/activities/test_model_import_bundle.py b/tests/laborious/activities/test_model_import_bundle.py index 5bde54c..edf4018 100644 --- a/tests/laborious/activities/test_model_import_bundle.py +++ b/tests/laborious/activities/test_model_import_bundle.py @@ -194,10 +194,10 @@ async def test_a_digest_from_another_file_fails_before_the_header_is_parsed( other = factory.make_bundle(password=BUNDLE_PASSWORD, salt=b'\x01' * 16) serve_object(minio_client, payload) + request = download_input(hashlib.sha256(other).hexdigest()) + with raises(ImportGateRejectedError) as failure: - await import_activity.download_import_bundle( - download_input(hashlib.sha256(other).hexdigest()) - ) + await import_activity.download_import_bundle(request) assert failure.value.details[0]['step'] == ImportStep.DOWNLOAD.value assert failure.value.details[0]['gate'] == 1 @@ -209,8 +209,10 @@ async def test_a_file_that_does_not_match_is_not_left_on_the_worker( ): serve_object(minio_client, factory.make_bundle(password=BUNDLE_PASSWORD)) + request = download_input('0' * 64) + with raises(ImportGateRejectedError): - await import_activity.download_import_bundle(download_input('0' * 64)) + await import_activity.download_import_bundle(request) assert not (tmp_path / WORKFLOW_ID / 'bundle.sientia').exists() @@ -234,9 +236,10 @@ async def test_an_object_over_the_ceiling_is_refused_before_its_body_is_fetched( import_activity: ModelImport, minio_client: MagicMock ): minio_client.stat_object.return_value = MagicMock(size=9 * 1024 * 1024) + request = download_input('a' * 64) with raises(ImportGateRejectedError) as failure: - await import_activity.download_import_bundle(download_input('a' * 64)) + await import_activity.download_import_bundle(request) assert failure.value.details[0]['step'] == ImportStep.DOWNLOAD.value minio_client.fget_object.assert_not_called() @@ -250,10 +253,10 @@ async def test_an_object_over_the_ceiling_is_refused_before_its_body_is_fetched( async def test_a_key_of_the_wrong_shape_is_refused( import_activity: ModelImport, minio_client: MagicMock, object_key: str ): + request = download_input('a' * 64, object_key=object_key) + with raises(ImportGateRejectedError): - await import_activity.download_import_bundle( - download_input('a' * 64, object_key=object_key) - ) + await import_activity.download_import_bundle(request) minio_client.fget_object.assert_not_called() @@ -263,9 +266,10 @@ async def test_an_unreadable_object_is_a_download_failure( import_activity: ModelImport, minio_client: MagicMock ): minio_client.stat_object.side_effect = OSError('no such object') + request = download_input('a' * 64) with raises(ImportGateRejectedError) as failure: - await import_activity.download_import_bundle(download_input('a' * 64)) + await import_activity.download_import_bundle(request) assert failure.value.details[0]['step'] == ImportStep.DOWNLOAD.value @@ -275,9 +279,10 @@ async def test_a_failed_transfer_is_a_download_failure( import_activity: ModelImport, minio_client: MagicMock ): minio_client.fget_object.side_effect = OSError('connection reset') + request = download_input('a' * 64) with raises(ImportGateRejectedError) as failure: - await import_activity.download_import_bundle(download_input('a' * 64)) + await import_activity.download_import_bundle(request) assert failure.value.details[0]['step'] == ImportStep.DOWNLOAD.value @@ -300,8 +305,10 @@ async def test_a_missing_minio_repository_is_a_configuration_error(tmp_path: Pat metrics_controller=AsyncMock(), ) + request = download_input('a' * 64) + with raises(ValueError, match='Minio repository not initialized'): - await activity.download_import_bundle(download_input('a' * 64)) + await activity.download_import_bundle(request) # ----------------------------------------------------------------------- opening the bundle @@ -348,9 +355,10 @@ async def test_a_wrong_envelope_key_is_not_confused_with_a_wrong_bundle_password path = tmp_path / 'bundle.sientia' path.write_bytes(factory.make_bundle(password=BUNDLE_PASSWORD)) envelope = factory.build_envelope(BUNDLE_PASSWORD, OTHER_ENVELOPE_KEY) + request = open_input(str(path), envelope) with raises(ImportGateRejectedError) as failure: - await import_activity.open_import_bundle(open_input(str(path), envelope)) + await import_activity.open_import_bundle(request) details = failure.value.details[0] assert details['step'] == ImportStep.DECRYPTION.value @@ -366,9 +374,10 @@ async def test_a_wrong_bundle_password_is_a_decryption_failure_of_its_own( ): path = tmp_path / 'bundle.sientia' path.write_bytes(factory.make_bundle(password='a-different-password')) + request = open_input(str(path)) with raises(ImportGateRejectedError) as failure: - await import_activity.open_import_bundle(open_input(str(path))) + await import_activity.open_import_bundle(request) details = failure.value.details[0] assert details['step'] == ImportStep.DECRYPTION.value @@ -382,9 +391,10 @@ async def test_a_missing_password_key_stops_the_import_at_decryption( import_activity.import_config['password_key'] = None path = tmp_path / 'bundle.sientia' path.write_bytes(factory.make_bundle(password=BUNDLE_PASSWORD)) + request = open_input(str(path)) with raises(ImportGateRejectedError) as failure: - await import_activity.open_import_bundle(open_input(str(path))) + await import_activity.open_import_bundle(request) assert failure.value.details[0]['cause'] == 'envelope' @@ -400,9 +410,10 @@ async def test_a_tampered_or_malformed_envelope_is_refused( ): path = tmp_path / 'bundle.sientia' path.write_bytes(factory.make_bundle(password=BUNDLE_PASSWORD)) + request = open_input(str(path), envelope) with raises(ImportGateRejectedError) as failure: - await import_activity.open_import_bundle(open_input(str(path), envelope)) + await import_activity.open_import_bundle(request) assert failure.value.details[0]['cause'] == 'envelope' @@ -472,9 +483,10 @@ async def test_a_gate_failure_reaches_the_activity_as_its_own_step( ): path = tmp_path / 'bundle.sientia' path.write_bytes(build()) + request = open_input(str(path)) with raises(ImportGateRejectedError) as failure: - await import_activity.open_import_bundle(open_input(str(path))) + await import_activity.open_import_bundle(request) assert failure.value.details[0]['step'] == step.value assert failure.value.details[0]['gate'] == gate @@ -486,9 +498,10 @@ async def test_a_rejected_bundle_leaves_no_extracted_tree( ): path = tmp_path / 'bundle.sientia' path.write_bytes(traversal()) + request = open_input(str(path)) with raises(ImportGateRejectedError): - await import_activity.open_import_bundle(open_input(str(path))) + await import_activity.open_import_bundle(request) extract_root = tmp_path / WORKFLOW_ID / 'extract' assert not extract_root.exists() or list(extract_root.rglob('*')) == [] diff --git a/tests/laborious/activities/test_model_import_provisioning.py b/tests/laborious/activities/test_model_import_provisioning.py index 65012f4..7915a04 100644 --- a/tests/laborious/activities/test_model_import_provisioning.py +++ b/tests/laborious/activities/test_model_import_provisioning.py @@ -146,9 +146,10 @@ async def test_the_experiment_keeps_the_origin_s_name(import_activity: ModelImpo @mark.asyncio async def test_a_missing_mlflow_repository_is_a_configuration_error(import_activity: ModelImport): import_activity.import_mlflow_repository = None + payload = base(experiment_name='e') with raises(ValueError, match='MLflow repository not initialized'): - await import_activity.create_import_experiment(base(experiment_name='e')) + await import_activity.create_import_experiment(payload) # ------------------------------------------------------------------- the run and its tree @@ -479,11 +480,10 @@ async def test_a_duplicate_id_fails_the_step_without_recomputing_it( """No retry and no second guess: the unique index says the collection moved under us.""" mongo.distinct.return_value = ['1'] mongo.insert.side_effect = RuntimeError('E11000 duplicate key error: id_1') + payload = base(model_name='factory_model', target='SE') with raises(RuntimeError, match='duplicate key'): - await import_activity.write_import_model_document( - base(model_name='factory_model', target='SE') - ) + await import_activity.write_import_model_document(payload) assert mongo.distinct.await_count == 1 assert mongo.insert.await_count == 1 @@ -505,9 +505,10 @@ async def test_a_missing_mongo_repository_is_a_configuration_error( import_activity: ModelImport, ): import_activity.mongo_repository = None + payload = base(model_name='m', target='SE') with raises(ValueError, match='MongoDB repository not initialized'): - await import_activity.write_import_model_document(base(model_name='m', target='SE')) + await import_activity.write_import_model_document(payload) def test_no_import_module_writes_an_active_document(): diff --git a/tests/laborious/activities/test_model_import_status.py b/tests/laborious/activities/test_model_import_status.py index fcd0efa..998d855 100644 --- a/tests/laborious/activities/test_model_import_status.py +++ b/tests/laborious/activities/test_model_import_status.py @@ -361,8 +361,10 @@ async def test_an_unclaimable_row_names_its_cause(row, expected: str): engine = FakeStatusEngine(rowcounts=[0], rows=[row]) activity = build_activity(engine) + payload = claim_input() + with raises(ImportLogRowNotClaimableError) as failure: - await activity.claim_import_status(claim_input()) + await activity.claim_import_status(payload) assert expected in str(failure.value) assert failure.value.import_run_id == 77 @@ -373,8 +375,10 @@ async def test_an_unclaimable_row_is_not_retried(): engine = FakeStatusEngine(rowcounts=[0], rows=[None]) activity = build_activity(engine) + payload = claim_input() + with raises(ImportLogRowNotClaimableError) as failure: - await activity.claim_import_status(claim_input()) + await activity.claim_import_status(payload) assert failure.value.non_retryable is True assert failure.value.type == 'ImportLogRowNotClaimableError' @@ -392,8 +396,10 @@ async def test_an_unclaimable_row_is_never_written_to(): engine = FakeStatusEngine(rowcounts=[0], rows=[{'run_type': 'TRAIN'}]) activity = build_activity(engine) + payload = claim_input() + with raises(ImportLogRowNotClaimableError): - await activity.claim_import_status(claim_input()) + await activity.claim_import_status(payload) assert len(engine.writes) == 1 assert engine.writes[0].sql == engine.statements[0].sql @@ -407,8 +413,10 @@ async def test_an_unclaimable_row_is_reported_through_every_sink(): engine = FakeStatusEngine(rowcounts=[0], rows=[None]) activity = build_activity(engine) + payload = claim_input() + with raises(ImportLogRowNotClaimableError): - await activity.claim_import_status(claim_input()) + await activity.claim_import_status(payload) logged = activity.logger.custom_error.call_args_list[0].args[0] assert ImportErrorCode.LOG_ROW_NOT_CLAIMABLE.value in logged @@ -688,9 +696,10 @@ async def test_a_refused_statement_fails_on_the_first_attempt(error_class): write_errors=[error_class('UPDATE ...', {}, Exception('refused'))], ) activity = build_activity(engine) + payload = {**claim_input(), 'succeeded': True} with raises(ImportStatusRejectedError) as failure: - await activity.record_import_terminal_status({**claim_input(), 'succeeded': True}) + await activity.record_import_terminal_status(payload) assert failure.value.non_retryable is True assert failure.value.type == 'ImportStatusRejectedError' @@ -702,9 +711,10 @@ async def test_a_connection_failure_is_left_to_the_retry(): """Repeating it is exactly what fixes it, so it must not be swallowed or reclassified.""" engine = FakeStatusEngine(write_errors=[OperationalError('UPDATE ...', {}, OSError('down'))]) activity = build_activity(engine) + payload = {**claim_input(), 'succeeded': True} with raises(OperationalError): - await activity.record_import_terminal_status({**claim_input(), 'succeeded': True}) + await activity.record_import_terminal_status(payload) @mark.asyncio diff --git a/tests/laborious/utils/bundle/test_reader.py b/tests/laborious/utils/bundle/test_reader.py index 9430173..81dd2bd 100644 --- a/tests/laborious/utils/bundle/test_reader.py +++ b/tests/laborious/utils/bundle/test_reader.py @@ -156,15 +156,19 @@ def test_gate_2_rejects_kdf_parameters_above_the_ceiling_without_allocating( def test_gate_2_rejects_a_digest_mismatch(work_dir, reader): + payload = factory.make_bundle(digest_override=bytes(32)) + with pytest.raises(BundleGateError) as error: - _open(reader, work_dir, factory.make_bundle(digest_override=bytes(32))) + _open(reader, work_dir, payload) assert error.value.gate == 2 def test_gate_2_rejects_a_truncated_transfer(work_dir, reader): + payload = factory.truncate_bundle(factory.make_bundle()) + with pytest.raises(BundleGateError) as error: - _open(reader, work_dir, factory.truncate_bundle(factory.make_bundle())) + _open(reader, work_dir, payload) assert error.value.gate == 2 @@ -182,8 +186,10 @@ AMBIGUOUS_REASON = 'wrong password, or the file is damaged or was altered' def test_gate_3_wrong_password(work_dir, reader): + payload = factory.make_bundle() + with pytest.raises(BundleGateError) as error: - _open(reader, work_dir, factory.make_bundle(), password='Wrong1Password!Here') + _open(reader, work_dir, payload, password='Wrong1Password!Here') assert error.value.gate == 3 assert error.value.step is ImportStep.DECRYPTION @@ -195,24 +201,30 @@ def test_gate_3_altered_ciphertext_reads_the_same_as_a_wrong_password(work_dir, with pytest.raises(BundleGateError) as altered: _open(reader, work_dir, tampered) + + wrong_password_payload = factory.make_bundle() with pytest.raises(BundleGateError) as wrong_password: - _open(reader, work_dir, factory.make_bundle(), password='Wrong1Password!Here') + _open(reader, work_dir, wrong_password_payload, password='Wrong1Password!Here') assert str(altered.value) == str(wrong_password.value) assert altered.value.gate == wrong_password.value.gate == 3 def test_gate_3_stream_without_a_final_tag_is_rejected(work_dir, reader): + payload = factory.make_bundle(drop_final_tag=True) + with pytest.raises(BundleGateError) as error: - _open(reader, work_dir, factory.make_bundle(drop_final_tag=True)) + _open(reader, work_dir, payload) assert error.value.gate == 3 assert AMBIGUOUS_REASON in str(error.value) def test_gate_3_leaves_no_decrypted_zip_behind(work_dir, reader): + payload = factory.make_bundle() + with pytest.raises(BundleGateError): - _open(reader, work_dir, factory.make_bundle(), password='Wrong1Password!Here') + _open(reader, work_dir, payload, password='Wrong1Password!Here') assert list(work_dir.glob('*.zip')) == [] @@ -277,18 +289,20 @@ def test_gate_4_rejects_hostile_central_directories(work_dir, reader, spec_kwarg def test_gate_4_rejects_too_many_entries(work_dir): limited = BundleReader(BundleLimits(max_entries=3)) + payload = factory.make_bundle() with pytest.raises(BundleGateError, match='entries') as error: - _open(limited, work_dir, factory.make_bundle()) + _open(limited, work_dir, payload) assert error.value.gate == 4 def test_gate_4_rejects_an_oversized_uncompressed_total(work_dir): limited = BundleReader(BundleLimits(max_uncompressed_bytes=64)) + payload = factory.make_bundle() with pytest.raises(BundleGateError, match='uncompressed') as error: - _open(limited, work_dir, factory.make_bundle()) + _open(limited, work_dir, payload) assert error.value.gate == 4 @@ -307,9 +321,10 @@ def test_gate_4_rejects_content_that_is_not_an_archive(work_dir, reader): def test_gate_5_refuses_a_pre_existing_extraction_root(work_dir, reader): (work_dir / 'extract').mkdir() + payload = factory.make_bundle() with pytest.raises(BundleGateError, match='already exists') as error: - _open(reader, work_dir, factory.make_bundle()) + _open(reader, work_dir, payload) assert error.value.gate == 5 assert error.value.step is ImportStep.EXTRACTION @@ -363,8 +378,10 @@ def test_gate_5_extraction_writes_plain_files(work_dir, reader): ], ) def test_gate_6_rejects_a_malformed_metadata_document(work_dir, reader, spec, match): + payload = factory.make_bundle(spec) + with pytest.raises(BundleGateError, match=match) as error: - _open(reader, work_dir, factory.make_bundle(spec)) + _open(reader, work_dir, payload) assert error.value.gate == 6 assert error.value.step is ImportStep.STRUCTURE_VALIDATION @@ -374,18 +391,20 @@ def test_gate_6_rejects_a_malformed_metadata_document(work_dir, reader, spec, ma def test_gate_6_rejects_unreadable_metadata_json(work_dir, reader): spec = factory.ZipSpec(include_metadata=False) spec.extra_files['metadata.json'] = b'{not json' + payload = factory.make_bundle(spec) with pytest.raises(BundleGateError, match='readable JSON') as error: - _open(reader, work_dir, factory.make_bundle(spec)) + _open(reader, work_dir, payload) assert error.value.gate == 6 def test_gate_6_rejects_a_missing_artifacts_directory(work_dir, reader): spec = factory.ZipSpec(files={}) + payload = factory.make_bundle(spec) with pytest.raises(BundleGateError, match='artifacts directory') as error: - _open(reader, work_dir, factory.make_bundle(spec)) + _open(reader, work_dir, payload) assert error.value.gate == 6 @@ -400,9 +419,10 @@ def test_gate_6_rejects_a_missing_artifacts_directory(work_dir, reader): def test_gate_6_rejects_an_incomplete_model_directory(work_dir, reader, dropped, match): spec = factory.ZipSpec() spec.files.pop(dropped) + payload = factory.make_bundle(spec) with pytest.raises(BundleGateError, match=match) as error: - _open(reader, work_dir, factory.make_bundle(spec)) + _open(reader, work_dir, payload) assert error.value.gate == 6 @@ -412,9 +432,10 @@ def test_gate_6_rejects_a_bundle_without_data_model(work_dir, reader): for name in list(spec.files): if name.startswith('artifacts/data_model/'): spec.files.pop(name) + payload = factory.make_bundle(spec) with pytest.raises(BundleGateError, match='no data_model artifacts') as error: - _open(reader, work_dir, factory.make_bundle(spec)) + _open(reader, work_dir, payload) assert error.value.gate == 6 @@ -426,9 +447,10 @@ def test_gate_6_rejects_a_bundle_without_data_model(work_dir, reader): def test_gate_7_rejects_content_a_model_export_should_not_carry(work_dir, reader, name): spec = factory.ZipSpec() spec.extra_files[name] = b'#!/bin/sh\n' + payload = factory.make_bundle(spec) with pytest.raises(BundleGateError, match='should not carry') as error: - _open(reader, work_dir, factory.make_bundle(spec)) + _open(reader, work_dir, payload) assert error.value.gate == 7 assert error.value.step is ImportStep.CONTENT_POLICY @@ -444,9 +466,10 @@ def test_the_earlier_gate_wins(work_dir, reader): traversal_entry='artifacts/../escape.csv', document=factory.build_metadata_document(drop_parameter='target_variable'), ) + payload = factory.make_bundle(spec) with pytest.raises(BundleGateError) as error: - _open(reader, work_dir, factory.make_bundle(spec)) + _open(reader, work_dir, payload) assert error.value.gate == 4 assert _extracted_files(work_dir / 'extract') == [] @@ -454,7 +477,9 @@ def test_the_earlier_gate_wins(work_dir, reader): def test_header_is_checked_before_the_password(work_dir, reader): """A bundle with a bad header and a wrong password reports the header, not the password.""" + payload = factory.make_bundle(magic=b'NOTSIENT') + with pytest.raises(BundleGateError) as error: - _open(reader, work_dir, factory.make_bundle(magic=b'NOTSIENT'), password='Nope1!Nope1!') + _open(reader, work_dir, payload, password='Nope1!Nope1!') assert error.value.gate == 2 diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 95c911c..d576b5f 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -2077,8 +2077,10 @@ def test_find_model_pickle_ignores_nested_pickles(mlflow_repository, tmp_path): nested.mkdir() (nested / 'stacking_model.pkl').write_bytes(b'pickle-bytes') + local_dir = str(tmp_path) + with pytest.raises(ValueError) as exc_info: - mlflow_repository._find_model_pickle(str(tmp_path), 'prediction_model') + mlflow_repository._find_model_pickle(local_dir, 'prediction_model') assert 'No .pkl file found' in str(exc_info.value) @@ -2093,8 +2095,10 @@ def test_find_model_pickle_reads_the_name_log_model_writes(mlflow_repository, tm def test_find_model_pickle_empty_directory(mlflow_repository, tmp_path): + local_dir = str(tmp_path) + with pytest.raises(ValueError) as exc_info: - mlflow_repository._find_model_pickle(str(tmp_path), 'data_model') + mlflow_repository._find_model_pickle(local_dir, 'data_model') assert 'data_model' in str(exc_info.value) @@ -2113,9 +2117,10 @@ def test_find_model_pickle_error_message_carries_context(mlflow_repository, tmp_ def test_find_model_pickle_ignores_directory_named_pkl(mlflow_repository, tmp_path): (tmp_path / 'algo.pkl').mkdir() + local_dir = str(tmp_path) with pytest.raises(ValueError) as exc_info: - mlflow_repository._find_model_pickle(str(tmp_path), 'prediction_model') + mlflow_repository._find_model_pickle(local_dir, 'prediction_model') assert 'algo.pkl' in str(exc_info.value) diff --git a/tests/laborious/utils/test_import_password.py b/tests/laborious/utils/test_import_password.py index 5d01876..a852861 100644 --- a/tests/laborious/utils/test_import_password.py +++ b/tests/laborious/utils/test_import_password.py @@ -31,9 +31,10 @@ def test_wrong_key_fails_authentication(): def test_tampered_envelope_fails_authentication(): payload = bytearray(base64.b64decode(factory.build_envelope('Correct1Horse!Battery', KEY))) payload[-1] ^= 0xFF + tampered_envelope = base64.b64encode(bytes(payload)).decode() with pytest.raises(EnvelopeError, match='authenticated'): - decrypt_password_envelope(base64.b64encode(bytes(payload)).decode(), KEY) + decrypt_password_envelope(tampered_envelope, KEY) def test_malformed_base64_rejected(): @@ -42,8 +43,10 @@ def test_malformed_base64_rejected(): def test_envelope_too_short_rejected(): + short_envelope = base64.b64encode(bytes(20)).decode() + with pytest.raises(EnvelopeError, match='too short'): - decrypt_password_envelope(base64.b64encode(bytes(20)).decode(), KEY) + decrypt_password_envelope(short_envelope, KEY) def test_empty_envelope_rejected(): diff --git a/tests/laborious/workflows/test_import_model.py b/tests/laborious/workflows/test_import_model.py index 6c67ae6..8d05c5c 100644 --- a/tests/laborious/workflows/test_import_model.py +++ b/tests/laborious/workflows/test_import_model.py @@ -322,9 +322,10 @@ async def test_a_failing_step_is_recorded_as_that_step_and_stops_the_import( ): recorder = Recorder(failures={failing: RuntimeError('dependency unavailable')}) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(RuntimeError, match='dependency unavailable'): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -353,9 +354,10 @@ async def test_nothing_is_provisioned_before_the_gates_pass(import_workflow: Imp } ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(ImportGateRejectedError): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -377,9 +379,10 @@ async def test_a_gate_rejection_is_recorded_with_the_gate_that_rejected_it( } ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(ImportGateRejectedError): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -398,9 +401,10 @@ async def test_an_ambiguous_decryption_failure_keeps_its_step(import_workflow: I } ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(ImportGateRejectedError): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -420,9 +424,10 @@ async def test_an_unknown_field_is_rejected_rather_than_ignored( """A caller that believes it recorded a project link must be told it did not.""" recorder = Recorder() patcher, _ = run_with(recorder) + payload = valid_input(**{field: 'whatever'}) try: with raises(ImportInputError, match='does not accept'): - await import_workflow.run(valid_input(**{field: 'whatever'})) + await import_workflow.run(payload) finally: patcher.stop() @@ -445,9 +450,10 @@ async def test_a_missing_or_malformed_digest_fails_before_anything_is_downloaded ): recorder = Recorder() patcher, _ = run_with(recorder) + payload = valid_input(expected_digest=digest) try: with raises(ImportInputError): - await import_workflow.run(valid_input(expected_digest=digest)) + await import_workflow.run(payload) finally: patcher.stop() @@ -490,9 +496,10 @@ async def test_a_record_naming_another_object_is_a_received_failure( } ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(ImportInputError, match='different object'): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -513,9 +520,10 @@ async def test_a_record_naming_another_digest_is_a_received_failure( } ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(ImportInputError, match='different digest'): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -528,9 +536,10 @@ async def test_an_unusable_record_id_fails_before_any_activity(import_workflow: """There is no record to claim and none to write: the run ends FAILED and nothing is touched.""" recorder = Recorder() patcher, _ = run_with(recorder) + payload = valid_input(import_run_id=value) try: with raises(ImportInputError, match='positive integer'): - await import_workflow.run(valid_input(import_run_id=value)) + await import_workflow.run(payload) finally: patcher.stop() @@ -547,9 +556,10 @@ async def test_a_claim_that_matches_nothing_writes_nothing_at_all(import_workflo } ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(ImportLogRowNotClaimableError): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -651,9 +661,10 @@ async def test_a_terminal_write_failure_does_not_mask_a_real_failure( } ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(RuntimeError, match='registry refused'): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -709,9 +720,10 @@ async def test_the_failure_that_ends_an_import_is_reported_through_the_channel( } ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(RuntimeError, match='Activity task failed'): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -732,9 +744,10 @@ async def test_the_report_comes_before_the_cleanup_and_the_terminal_write( failures={Activities.upload_import_artifacts: activity_failure('bucket gone', 'S3Error')} ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(RuntimeError): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -761,9 +774,10 @@ async def test_the_gate_that_rejected_the_bundle_travels_into_the_report( } ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(ImportGateRejectedError): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -783,9 +797,10 @@ async def test_a_claim_that_matches_nothing_is_not_reported_twice(import_workflo } ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(ImportLogRowNotClaimableError): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -803,9 +818,10 @@ async def test_a_report_that_cannot_be_made_does_not_mask_the_failure( } ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(RuntimeError, match='Activity task failed'): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() @@ -830,9 +846,10 @@ async def test_the_report_gives_up_inside_the_short_bound(import_workflow: Impor } ) patcher, _ = run_with(recorder) + payload = valid_input() try: with raises(RuntimeError): - await import_workflow.run(valid_input()) + await import_workflow.run(payload) finally: patcher.stop() From 72a55c09002b28caf400f464cda25de923209673 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 4 Sep 2026 11:55:18 -0300 Subject: [PATCH 39/64] QTZPOC-20 QTZPOC-20: Raise non-retryable Temporal ApplicationError when no data is returned from query in minimal retrain workflow. --- laborious/workflows/minimal_retrain.py | 7 ++++++- tests/laborious/workflows/test_minimal_retrain.py | 10 ++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/laborious/workflows/minimal_retrain.py b/laborious/workflows/minimal_retrain.py index f8f5151..339a6af 100644 --- a/laborious/workflows/minimal_retrain.py +++ b/laborious/workflows/minimal_retrain.py @@ -5,6 +5,7 @@ with workflow.unsafe.imports_passed_through(): from typing import Any from sientia_do.temporal.policies import retry_policy + from temporalio.exceptions import ApplicationError from laborious.activities.activities import Activities from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload @@ -87,7 +88,11 @@ class MinimalRetrain: storage_payload = MinioDataFramePayload.from_dict(storage_result) if not storage_payload.has_data(): - raise ValueError('No data returned from query') + raise ApplicationError( + 'No data returned from query', + type='NoDataReturnedError', + non_retryable=True, + ) experiment_response = await workflow.execute_activity_method( Activities.retrain_model, diff --git a/tests/laborious/workflows/test_minimal_retrain.py b/tests/laborious/workflows/test_minimal_retrain.py index 94d86da..856e840 100644 --- a/tests/laborious/workflows/test_minimal_retrain.py +++ b/tests/laborious/workflows/test_minimal_retrain.py @@ -1,6 +1,7 @@ from unittest.mock import ANY, AsyncMock, call, patch -from pytest import fixture, mark +from pytest import fixture, mark, raises +from temporalio.exceptions import ApplicationError from laborious.activities.activities import Activities from laborious.workflows.minimal_retrain import MinimalRetrain @@ -194,11 +195,12 @@ async def test_run_storage_fail(workflow_mock: AsyncMock, minimal_retrain: Minim ] ) - from pytest import raises - - with raises(ValueError, match='No data returned from query'): + with raises(ApplicationError, match='No data returned from query') as exc_info: await minimal_retrain.run(input_data) + assert exc_info.value.non_retryable is True + assert exc_info.value.type == 'NoDataReturnedError' + workflow_mock.execute_activity_method.assert_called_once_with( Activities.load_query_with_minio_offload, { From 61db575cdc08f20afa612f2c033e347ee6fde499 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 4 Sep 2026 15:23:55 -0300 Subject: [PATCH 40/64] QTZPOC-20 QTZPOC-20: Add NOSONAR comment to summarize_failure function to suppress static analysis warning. --- laborious/utils/models/import_errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/laborious/utils/models/import_errors.py b/laborious/utils/models/import_errors.py index 888ff5e..8312bd4 100644 --- a/laborious/utils/models/import_errors.py +++ b/laborious/utils/models/import_errors.py @@ -298,7 +298,7 @@ def summarize_failure( """ if exception is not None: cause = _root_cause(exception) - return _type_name(cause), _one_line(str(cause)) + return _type_name(cause), _one_line(str(cause)) # NOSONAR return error_type or _UNKNOWN_ERROR_TYPE, _one_line(detail_text or '') From e4fc12e5292529e9528c12b227a5089fe3fd78d9 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 8 Sep 2026 09:42:23 -0300 Subject: [PATCH 41/64] QTZPOC-20 QTZPOC-20: Enhance system robustness and test coverage across bundle processing, activity error handling, and worker operations. * Strengthen bundle reader validation with new tests for archive integrity, security vulnerabilities (e.g., path traversal, duplicate entries, special file types), and I/O error resilience. * Improve error handling and status propagation in PI Web API, MLflow, and OPC activities, including partial write scenarios and OPC session management. * Add comprehensive tests for worker startup, configuration, graceful shutdown, and Prometheus metrics server initialization. * Introduce a new utility for intelligent DataFrame debugging, summarizing large dataframes for concise logging, and refine exception chain summarization. * Update code coverage configuration to exclude @overload decorator lines. --- pyproject.toml | 1 + tests/helpers/bundle_factory.py | 23 ++ tests/laborious/activities/test_api.py | 38 +++ tests/laborious/activities/test_mlflow.py | 33 +++ .../test_model_import_provisioning.py | 40 +++ .../activities/test_model_import_status.py | 22 ++ tests/laborious/activities/test_opc.py | 33 +++ tests/laborious/utils/bundle/test_reader.py | 123 +++++++++ .../utils/models/test_import_errors.py | 34 +++ .../models/test_minio_dataframe_payload.py | 21 ++ .../utils/repository/test_opc_repository.py | 190 ++++++++++++++ tests/laborious/utils/test_dataframe_debug.py | 40 +++ tests/laborious/utils/test_import_password.py | 23 ++ tests/laborious/worker/test_worker_main.py | 245 ++++++++++++++++++ .../subworkflows/test_prediction_process.py | 26 +- 15 files changed, 891 insertions(+), 1 deletion(-) create mode 100644 tests/laborious/utils/test_dataframe_debug.py create mode 100644 tests/laborious/worker/test_worker_main.py diff --git a/pyproject.toml b/pyproject.toml index 2f2ef0d..7ffcad0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -179,6 +179,7 @@ exclude_lines = [ "if TYPE_CHECKING:", "class .*\\bProtocol\\):", "@(abc\\.)?abstractmethod", + "@overload", ] [tool.coverage.html] diff --git a/tests/helpers/bundle_factory.py b/tests/helpers/bundle_factory.py index db54be5..d7ce3e3 100644 --- a/tests/helpers/bundle_factory.py +++ b/tests/helpers/bundle_factory.py @@ -23,6 +23,7 @@ import hashlib import io import json import os +import stat import struct import unicodedata import zipfile @@ -118,6 +119,15 @@ class ZipSpec: traversal_entry: str | None = None bomb_bytes: int = 0 extra_files: dict[str, bytes] = field(default_factory=dict) + # name -> a raw `stat.S_IF*` mode, for an entry that is neither a symlink, a regular file nor a + # directory (e.g. `stat.S_IFIFO`) — gate 4 must reject these on their file-type bits alone. + special_mode_entries: dict[str, int] = field(default_factory=dict) + # Directory entries the producer never writes (`build_zip` stores none), but a gate 5 defense + # this repository still carries has to be exercised by something. + directory_entries: list[str] = field(default_factory=list) + # A file name already in `files`/`extra_files`, written a second time under the same name — two + # archive entries claiming the same path, which only gate 5's extraction can catch. + repeat_file: str | None = None def build_zip(spec: ZipSpec | None = None) -> bytes: @@ -145,6 +155,19 @@ def build_zip(spec: ZipSpec | None = None) -> bytes: archive.writestr(_regular_file_info(spec.traversal_entry), b'traversal\n') if spec.bomb_bytes: archive.writestr(_regular_file_info('artifacts/bomb.csv'), b'\0' * spec.bomb_bytes) + for name, mode in spec.special_mode_entries.items(): + info = zipfile.ZipInfo(name) + info.create_system = 3 + info.external_attr = mode << 16 + archive.writestr(info, b'') + for name in spec.directory_entries: + info = zipfile.ZipInfo(name if name.endswith('/') else f'{name}/') + info.create_system = 3 + info.external_attr = (stat.S_IFDIR | 0o755) << 16 + archive.writestr(info, b'') + if spec.repeat_file: + content = {**spec.files, **spec.extra_files}[spec.repeat_file] + archive.writestr(_regular_file_info(spec.repeat_file), content) return buffer.getvalue() diff --git a/tests/laborious/activities/test_api.py b/tests/laborious/activities/test_api.py index bf5b9f1..45d5e92 100644 --- a/tests/laborious/activities/test_api.py +++ b/tests/laborious/activities/test_api.py @@ -220,6 +220,44 @@ async def test_write_pi_web_api_data_success(mock_dataframe, api, base_input_dat } +@mark.asyncio +@patch('laborious.activities.api.DataFrame') +async def test_write_pi_web_api_data_partial_prediction_mismatch_downgrades_confidence( + mock_dataframe, api, base_input_data +): + """A partial write (some tags report Errors) doesn't raise — it's caught by the tag-count + mismatch check, which still must downgrade confidence and attach the mismatch message.""" + input_data = { + **base_input_data, + 'pi_web_api_output_config': { + 'endpoint': 'https://test-pi-server.com/piwebapi', + 'prediction_tags': {'tag1': 'web_id_1', 'tag2': 'web_id_2'}, + 'confidence_tags': {'tag3': 'web_id_3'}, + }, + } + mock_df = _create_mock_dataframe() + mock_dataframe.return_value = mock_df + + api.pi_web_api_client.write_value.side_effect = [ + [{'WebId': 'web_id_1', 'Errors': []}, {'WebId': 'web_id_2', 'Errors': ['bad tag']}], + [{'WebId': 'web_id_3', 'Errors': []}], + ] + + await api.write_pi_web_api_data(input_data) + + mock_df.__setitem__.assert_any_call( + 'prediction_confidence', PI_WEB_API_PREDICTION_ERROR_CONFIDENCE + ) + mock_df.__setitem__.assert_any_call('comments', ANY) + api.send_notification_async.assert_called_once_with( + metadata=metadata['metadata'], + notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR', + message=ANY, + block='write_pi_web_api_data', + level=NotificationLevel.ERROR, + ) + + @mark.asyncio @patch('laborious.activities.api.DataFrame') async def test_write_pi_web_api_data_prediction_error(mock_dataframe, api, base_input_data): diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index 5d242e5..da17c94 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -222,6 +222,39 @@ async def test_request_predict(mock_to_datetime, mock_from_dataframe, mlflow): assert response_data == mock_from_dataframe.return_value +@mark.asyncio +@patch( + 'laborious.activities.mlflow.MinioDataFramePayload.from_dataframe', + new_callable=AsyncMock, +) +@patch('laborious.activities.mlflow.to_datetime') +async def test_request_predict_skips_timestamp_reinsertion_when_configured( + mock_to_datetime, mock_from_dataframe, mlflow +): + """skip_predict_timestamp=True leaves the caller's index alone — no reinsertion round-trip.""" + data_mock = MagicMock() + payload = AsyncMock() + payload.retrieve = AsyncMock(return_value=data_mock) + + input_data = { + **metadata, + 'data': payload, + 'model_name': 'test_model', + 'model_config': {'skip_predict_timestamp': True}, + } + + predict_response = {'success': True, 'content': MagicMock()} + mlflow.model_monitoring_repository.predict.return_value = predict_response + + await mlflow.request_predict(input_data) + + data_mock.replace.assert_called_once_with(np.nan, None, inplace=True) + mock_to_datetime.assert_not_called() + mlflow.model_monitoring_repository.predict.assert_called_once_with( + 'test_model', data_mock, {'skip_predict_timestamp': True}, metadata['metadata'] + ) + + @mark.asyncio @patch( 'laborious.activities.mlflow.MinioDataFramePayload.from_dataframe', diff --git a/tests/laborious/activities/test_model_import_provisioning.py b/tests/laborious/activities/test_model_import_provisioning.py index 7915a04..fe5d11f 100644 --- a/tests/laborious/activities/test_model_import_provisioning.py +++ b/tests/laborious/activities/test_model_import_provisioning.py @@ -166,6 +166,17 @@ def upload_input(extracted_dir: Path, **overrides) -> dict: ) +@mark.asyncio +async def test_upload_artifacts_requires_the_mlflow_repository( + import_activity: ModelImport, tmp_path: Path +): + import_activity.import_mlflow_repository = None + payload = upload_input(tmp_path / 'extract') + + with raises(ValueError, match='MLflow repository not initialized'): + await import_activity.upload_import_artifacts(payload) + + @mark.asyncio async def test_every_import_creates_a_new_run_with_the_recorded_name( import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path @@ -235,6 +246,15 @@ async def test_the_run_is_closed_when_its_tree_is_in( # ------------------------------------------------------------------------ registration +@mark.asyncio +async def test_register_version_requires_the_mlflow_repository(import_activity: ModelImport): + import_activity.import_mlflow_repository = None + payload = base(model_name='factory_model', run_id=RUN_ID) + + with raises(ValueError, match='MLflow repository not initialized'): + await import_activity.register_import_model_version(payload) + + @mark.asyncio async def test_the_version_source_is_the_shape_the_legacy_loader_parses( import_activity: ModelImport, mlflow_client: MagicMock @@ -511,6 +531,17 @@ async def test_a_missing_mongo_repository_is_a_configuration_error( await import_activity.write_import_model_document(payload) +@mark.asyncio +async def test_next_model_document_id_also_guards_a_missing_mongo_repository( + import_activity: ModelImport, +): + """The helper repeats the guard so it stays safe if ever called from somewhere new.""" + import_activity.mongo_repository = None + + with raises(ValueError, match='MongoDB repository not initialized'): + await import_activity._next_model_document_id('models', {}) + + def test_no_import_module_writes_an_active_document(): """Asserted from the source too, so `active: false` cannot be flipped by a later edit.""" source = Path('laborious/activities/model_import.py').read_text() @@ -577,6 +608,15 @@ async def test_cleanup_never_touches_the_uploaded_object( client.remove_objects.assert_not_called() +@mark.asyncio +async def test_ensure_retention_requires_the_minio_repository(import_activity: ModelImport): + import_activity.import_minio_repository = None + payload = base() + + with raises(ValueError, match='Minio repository not initialized'): + await import_activity.ensure_import_bundle_retention(payload) + + @mark.asyncio async def test_the_expiry_rule_is_applied_to_the_import_prefix(import_activity: ModelImport): client = import_activity.import_minio_repository.minio_client diff --git a/tests/laborious/activities/test_model_import_status.py b/tests/laborious/activities/test_model_import_status.py index 998d855..2f24080 100644 --- a/tests/laborious/activities/test_model_import_status.py +++ b/tests/laborious/activities/test_model_import_status.py @@ -221,6 +221,18 @@ def test_the_detail_records_a_cleanup_failure_without_a_code(): assert 'code' not in detail +def test_the_detail_carries_the_bundle_metadata_as_source(): + detail = json.loads( + build_import_detail(workflow_id='wf-1', source={'exported_by': 'user@example.com'}) + )['import'] + assert detail['source'] == {'exported_by': 'user@example.com'} + + +def test_the_detail_omits_source_when_empty(): + detail = json.loads(build_import_detail(workflow_id='wf-1', source={}))['import'] + assert 'source' not in detail + + # ------------------------------------------------------------------------------ the claim @@ -834,3 +846,13 @@ async def test_the_engine_is_disposed_on_shutdown(import_activity: ModelImport, import_activity.close() assert engine.disposed is True + + +@mark.asyncio +async def test_close_is_a_no_op_without_an_engine(import_activity: ModelImport, engine): + """A partially-constructed instance (or a second close) must not blow up on a missing engine.""" + del import_activity.status_engine + + import_activity.close() + + assert engine.disposed is False diff --git a/tests/laborious/activities/test_opc.py b/tests/laborious/activities/test_opc.py index 6a36f8a..2e248a6 100644 --- a/tests/laborious/activities/test_opc.py +++ b/tests/laborious/activities/test_opc.py @@ -594,6 +594,39 @@ async def test_write_opc_data_success(mock_dataframe, opc): ) +@mark.asyncio +@patch('laborious.activities.opc.DataFrame') +async def test_write_opc_data_propagates_session_bad_and_reconnect_in_progress(mock_dataframe, opc): + """A single server reporting session_bad/reconnect_in_progress must flip the aggregated + flags passed to process_confidence, even though nothing raised.""" + input_data = { + **metadata, + 'data': {'prediction': [0.75], 'prediction_confidence': [0.95]}, + 'opc_output_config': { + 'server1': { + 'prediction_tags': {'tag1': {'data_type': 'float'}}, + 'confidence_tags': {'tag2': {'data_type': 'float'}}, + } + }, + } + + opc.manage_output_tags = AsyncMock( + return_value=(False, {'tag1': 0.1, 'tag2': 0.2}, True, 'BadSessionClosed', True) + ) + opc.process_confidence = MagicMock(return_value={'data': 'data'}) + + await opc.write_opc_data(input_data) + + opc.process_confidence.assert_called_once_with( + mock_dataframe.return_value, + False, + metadata['metadata'], + session_bad=True, + opc_status='BadSessionClosed', + reconnect_in_progress=True, + ) + + @mark.asyncio async def test_write_opc_data_empty_config(opc): # Arrange diff --git a/tests/laborious/utils/bundle/test_reader.py b/tests/laborious/utils/bundle/test_reader.py index 81dd2bd..ebff1bf 100644 --- a/tests/laborious/utils/bundle/test_reader.py +++ b/tests/laborious/utils/bundle/test_reader.py @@ -1,7 +1,11 @@ """The seven gates: what each one rejects, in what order, and what it leaves on disk.""" +import io import json +import os import random +import stat +import zipfile from pathlib import Path import nacl.bindings as sodium @@ -173,6 +177,16 @@ def test_gate_2_rejects_a_truncated_transfer(work_dir, reader): assert error.value.gate == 2 +def test_gate_2_wraps_an_os_error_reading_the_file(work_dir, reader): + """A download that never landed on disk is an OS error, not a format one.""" + missing = work_dir / 'never-downloaded.sientia' + + with pytest.raises(BundleGateError, match='could not be read') as error: + reader.open(missing, factory.DEFAULT_PASSWORD, work_dir / 'extract', work_dir) + + assert error.value.gate == 2 + + def _never_called(message: str): def _raise(*_args, **_kwargs): raise AssertionError(message) @@ -220,6 +234,22 @@ def test_gate_3_stream_without_a_final_tag_is_rejected(work_dir, reader): assert AMBIGUOUS_REASON in str(error.value) +def test_gate_3_wraps_an_os_error_while_writing_the_decrypted_zip(work_dir, reader, monkeypatch): + """A write failure while streaming the plaintext to disk is reported as disk trouble, not as + a bad password — the AEAD never even got to disagree.""" + payload = factory.make_bundle() + + def _raise_os_error(*_args, **_kwargs): + raise OSError('disk full') + + monkeypatch.setattr(reader, '_pull_stream', _raise_os_error) + + with pytest.raises(BundleGateError, match='could not be written') as error: + _open(reader, work_dir, payload) + + assert error.value.gate == 3 + + def test_gate_3_leaves_no_decrypted_zip_behind(work_dir, reader): payload = factory.make_bundle() @@ -316,6 +346,27 @@ def test_gate_4_rejects_content_that_is_not_an_archive(work_dir, reader): assert error.value.gate == 4 +def test_gate_4_rejects_an_empty_archive(work_dir, reader): + payload = factory.make_bundle(factory.ZipSpec(files={}, include_metadata=False)) + + with pytest.raises(BundleGateError, match='archive is empty') as error: + _open(reader, work_dir, payload) + + assert error.value.gate == 4 + + +def test_gate_4_rejects_an_entry_that_is_neither_a_file_nor_a_directory(work_dir, reader): + """A FIFO's mode bits are neither a symlink's, a regular file's, nor a directory's.""" + payload = factory.make_bundle( + factory.ZipSpec(special_mode_entries={'artifacts/pipe': stat.S_IFIFO | 0o644}) + ) + + with pytest.raises(BundleGateError, match='not a regular file') as error: + _open(reader, work_dir, payload) + + assert error.value.gate == 4 + + # ------------------------------------------------------------------- gate 5 @@ -338,6 +389,61 @@ def test_gate_5_extraction_writes_plain_files(work_dir, reader): assert path.stat().st_mode & 0o111 == 0, path +def test_gate_5_extraction_creates_declared_directory_entries(work_dir, reader): + """The producer never writes directory entries, but a bundle from anywhere else might.""" + payload = factory.make_bundle(factory.ZipSpec(directory_entries=['artifacts/empty_dir'])) + + opened = _open(reader, work_dir, payload) + + assert (opened.extracted_dir / 'artifacts' / 'empty_dir').is_dir() + + +def test_gate_5_rejects_a_duplicate_entry_name(work_dir, reader): + """Two archive entries claiming the same path: gate 4 checks each name on its own and cannot + see the collision, so extraction is where an overwrite is caught — and rejected.""" + payload = factory.make_bundle(factory.ZipSpec(repeat_file='artifacts/prediction_model/MLmodel')) + + with pytest.raises(BundleGateError, match='overwrite') as error: + _open(reader, work_dir, payload) + + assert error.value.gate == 5 + assert not (work_dir / 'extract').exists() + + +def test_gate_5_wraps_an_os_error_during_unpacking(work_dir, reader, monkeypatch): + """A write failure partway through extraction (e.g. disk full) is reported as unpacking + trouble, not surfaced as the raw OSError.""" + payload = factory.make_bundle() + + def _raise_os_error(*_args, **_kwargs): + raise OSError('disk full') + + monkeypatch.setattr('laborious.utils.bundle.reader.shutil.copyfileobj', _raise_os_error) + + with pytest.raises(BundleGateError, match='could not be unpacked') as error: + _open(reader, work_dir, payload) + + assert error.value.gate == 5 + assert not (work_dir / 'extract').exists() + + +def test_extract_member_rechecks_containment_on_the_resolved_path(work_dir, reader): + """Gate 4 already rejects a traversal by name; extraction re-checks on the *resolved* path as + a second, independent line of defense.""" + root = work_dir / 'root' + root.mkdir() + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, 'w') as archive: + archive.writestr('safe.txt', b'hello') + + with zipfile.ZipFile(io.BytesIO(buffer.getvalue())) as archive: + entry = archive.infolist()[0] + with pytest.raises(BundleGateError, match='resolves outside') as error: + reader._extract_member(archive, entry, root, work_dir / 'somewhere-else') + + assert error.value.gate == 5 + + # ------------------------------------------------------------------- gate 6 @@ -375,6 +481,10 @@ def test_gate_5_extraction_writes_plain_files(work_dir, reader): ), 'experiment name', ), + ( + factory.ZipSpec(document={'metadata': 'not-a-dict', 'parameters': {}}), + 'not objects', + ), ], ) def test_gate_6_rejects_a_malformed_metadata_document(work_dir, reader, spec, match): @@ -457,6 +567,19 @@ def test_gate_7_rejects_content_a_model_export_should_not_carry(work_dir, reader assert not (work_dir / 'extract').exists() +def test_gate_7_rejects_a_non_regular_file_left_in_the_tree(work_dir, reader): + """Extraction itself only ever writes plain files and directories; this is the check that + would catch anything else showing up in the tree by some other means.""" + root = work_dir / 'weird_root' + root.mkdir() + os.mkfifo(root / 'pipe') + + with pytest.raises(BundleGateError, match='not a file') as error: + reader._gate_7_content_policy(root) + + assert error.value.gate == 7 + + # ------------------------------------------------------------------ ordering diff --git a/tests/laborious/utils/models/test_import_errors.py b/tests/laborious/utils/models/test_import_errors.py index ee949b7..53472fa 100644 --- a/tests/laborious/utils/models/test_import_errors.py +++ b/tests/laborious/utils/models/test_import_errors.py @@ -536,6 +536,40 @@ def test_a_failure_that_arrives_as_nothing_says_so_rather_than_guessing(): assert summary == '' +def test_summarize_failure_bounds_a_cyclic_cause_chain(): + """A cycle never hits a `None` cause, so the walk must stop by exhausting its own bound.""" + first = RuntimeError('first') + second = RuntimeError('second') + first.__cause__ = second + second.__cause__ = first + + kind, _summary = summarize_failure(first) + + assert kind == 'RuntimeError' + + +def test_a_leading_blank_line_is_skipped_for_the_first_real_one(): + class BlankFirstLineError(Exception): + def __str__(self) -> str: + return '\n \nreal message' + + _kind, summary = summarize_failure(BlankFirstLineError()) + + assert summary == 'real message' + + +def test_describe_failure_chain_reports_a_message_less_link_by_its_type_alone(): + """`Exception()` has no message at all — the link still has to be readable on its own.""" + inner = Exception() + outer = RuntimeError('outer failure') + outer.__cause__ = inner + + kind, detail = describe_failure_chain(outer) + + assert kind == 'Exception' + assert detail.splitlines() == ['Exception', 'RuntimeError: outer failure'] + + # ------------------------------------------------- the channel is the only reporter diff --git a/tests/laborious/utils/models/test_minio_dataframe_payload.py b/tests/laborious/utils/models/test_minio_dataframe_payload.py index c27765d..88d7885 100644 --- a/tests/laborious/utils/models/test_minio_dataframe_payload.py +++ b/tests/laborious/utils/models/test_minio_dataframe_payload.py @@ -214,6 +214,27 @@ async def test_from_dataframe_offloaded(mock_now): minio.upload_file.assert_awaited_once() +@pytest.mark.asyncio +@patch('laborious.utils.models.minio_dataframe_payload.OFFLOAD_THRESHOLD_BYTES', 10**9) +async def test_from_dataframe_inline_keeps_a_provided_last_timestamp(): + """An explicit last_timestamp must survive as-is, skipping the max() derivation.""" + minio = AsyncMock() + logger = MagicMock() + df = _mock_dataframe({'timestamp': ['2024-01-01'], 'value': [42]}) + + result = await MinioDataFramePayload.from_dataframe( + dataframe=df, + minio_repo=minio, + model_name='m', + operation='initial', + last_timestamp='2030-01-01T00:00:00+00:00', + logger=logger, + ) + + assert result.last_timestamp == '2030-01-01T00:00:00+00:00' + logger.custom_debug.assert_called() + + def test_from_dict_inline(): raw = { 'last_timestamp': '2024-01-01T00:00:00+00:00', diff --git a/tests/laborious/utils/repository/test_opc_repository.py b/tests/laborious/utils/repository/test_opc_repository.py index 365a400..f9d489b 100644 --- a/tests/laborious/utils/repository/test_opc_repository.py +++ b/tests/laborious/utils/repository/test_opc_repository.py @@ -1,5 +1,6 @@ import asyncio import json +import time from datetime import datetime from unittest.mock import ANY, AsyncMock, MagicMock, Mock, patch @@ -8,11 +9,13 @@ from asyncua.crypto.security_policies import SecurityPolicyBasic256 from asyncua.ua.uaerrors import BadNodeIdUnknown, BadSessionIdInvalid from sientia_do.notifications.models import NotificationLevel +from laborious import metrics from laborious.utils.repository.opc_repository import ( OpcClientAlreadyExistsError, OpcClientNotInitializedError, OpcRepository, OpcSessionAlreadyConnectedError, + _opc_authentication_token_str, is_reconnectable_opcua_bad, ) @@ -80,6 +83,40 @@ def test_init(opc_repository): assert opc_repository.last_reconnection_time is None +def test_opc_authentication_token_str_no_protocol(): + client = MagicMock() + client.uaclient.protocol = None + + assert _opc_authentication_token_str(client) == 'unknown' + + +def test_opc_authentication_token_str_no_token(): + client = MagicMock() + client.uaclient.protocol = MagicMock(authentication_token=None) + + assert _opc_authentication_token_str(client) == 'unknown' + + +def test_opc_authentication_token_str_swallows_errors(): + class _ExplodingClient: + @property + def uaclient(self): + raise RuntimeError('boom') + + assert _opc_authentication_token_str(_ExplodingClient()) == 'unknown' + + +def test_is_session_open_swallows_errors(opc_repository): + class _ExplodingClient: + @property + def uaclient(self): + raise RuntimeError('boom') + + opc_repository.client = _ExplodingClient() + + assert opc_repository._is_session_open() is False + + @pytest.mark.asyncio async def test_set_security(opc_repository, mock_client): opc_repository.client = mock_client @@ -160,6 +197,47 @@ async def test_create_client_raises_when_client_exists(opc_repository, mock_clie await opc_repository._create_client() +@pytest.mark.asyncio +async def test_create_client_builds_a_secured_client(opc_repository, mock_client): + opc_repository.pod_id = 'pod-1' + opc_repository.set_security = AsyncMock() + + await opc_repository._create_client() + + assert opc_repository.client is mock_client + assert mock_client.name == 'pod-1' + assert mock_client.application_name == 'pod-1' + assert mock_client.application_uri == 'pod:1' + assert mock_client.product_uri == 'pod:1' + opc_repository.set_security.assert_called_once() + + +@pytest.mark.asyncio +async def test_create_client_skips_security_without_a_cert_path(opc_repository, mock_client): + opc_repository.cert_path = None + opc_repository.set_security = AsyncMock() + + await opc_repository._create_client() + + assert opc_repository.client is mock_client + opc_repository.set_security.assert_not_called() + + +@pytest.mark.asyncio +async def test_connect_locked_reuses_an_existing_client(opc_repository, mock_client): + """A client already set (e.g. surviving a prior failed open) is not recreated.""" + opc_repository.client = mock_client + opc_repository._is_session_open = MagicMock(return_value=False) + opc_repository._create_client = AsyncMock() + opc_repository._open_session = AsyncMock(return_value=(True, {})) + + result = await opc_repository._connect_locked() + + opc_repository._create_client.assert_not_called() + opc_repository._open_session.assert_called_once() + assert result == (True, {}) + + @pytest.mark.asyncio async def test_open_session_success(opc_repository): closed_proto = MagicMock() @@ -435,6 +513,82 @@ async def test_write_data_write_value_failed(opc_repository, mock_client): assert error_data['attachment_content'] is not None +@pytest.mark.asyncio +async def test_write_data_with_empty_metadata_uses_unknown_labels(opc_repository, mock_client): + """No model_id/model_name in the write metadata: the metric labels fall back to 'unknown'.""" + opc_repository.validate_connection = AsyncMock(return_value=(True, {})) + opc_repository.client = mock_client + mock_node = AsyncMock() + mock_client.get_node = MagicMock(return_value=mock_node) + + result = await opc_repository.write_data('ns=2;s=TestNode', 42.0, 'float', {}) + + assert result == (True, {'response_time': ANY}) + last_call = opc_repository.emit_metric.call_args_list[-1] + assert last_call.args[0] is metrics.OPC_WRITE_ATTEMPTS_TOTAL + assert last_call.args[1]['model_id'] == 'unknown' + assert last_call.args[1]['model_name'] == 'unknown' + + +@pytest.mark.asyncio +async def test_write_data_get_node_bad_session_id_schedules_reconnect(opc_repository, mock_client): + opc_repository.validate_connection = AsyncMock(return_value=(True, {})) + opc_repository.client = mock_client + opc_repository._start_reconnect = AsyncMock() + mock_client.get_node = MagicMock(side_effect=BadSessionIdInvalid()) + + is_success, error_data = await opc_repository.write_data( + 'ns=2;s=TestNode', 42.0, 'float', metadata['metadata'] + ) + + opc_repository._start_reconnect.assert_called_once() + assert is_success is False + assert error_data['opc_error_kind'] == 'session_bad' + assert error_data['opc_status'] == 'BadSessionIdInvalid' + + +@pytest.mark.asyncio +async def test_write_data_logs_inter_arrival_and_flags_a_stale_session(opc_repository, mock_client): + """The second write on the same session logs the gap since the previous one, and flags it + when the gap outlives the session's revised timeout.""" + opc_repository.validate_connection = AsyncMock(return_value=(True, {})) + opc_repository.client = mock_client + mock_client.session_timeout = 0.001 # 1 microsecond: any real gap will exceed it + mock_node = AsyncMock() + mock_client.get_node = MagicMock(return_value=mock_node) + + await opc_repository.write_data('ns=2;s=TestNode', 1.0, 'float', metadata['metadata']) + await opc_repository.write_data('ns=2;s=TestNode', 2.0, 'float', metadata['metadata']) + + logged = [c.args[0] for c in opc_repository.info.call_args_list] + assert any('OPC write inter-arrival' in message for message in logged) + emitted_metrics = [c.args[0] for c in opc_repository.emit_metric.call_args_list] + assert metrics.OPC_WRITE_INTER_ARRIVAL_OVER_SESSION_TIMEOUT_TOTAL in emitted_metrics + + +@pytest.mark.asyncio +async def test_log_write_inter_arrival_skips_the_metric_without_a_client(opc_repository): + opc_repository._last_write_mono = time.monotonic() - 1 + opc_repository.client = None + + await opc_repository._log_write_inter_arrival('session-1', 'ns=2;s=Node') + + opc_repository.emit_metric.assert_not_called() + + +@pytest.mark.asyncio +async def test_log_write_inter_arrival_skips_the_metric_within_the_session_timeout( + opc_repository, mock_client +): + opc_repository._last_write_mono = time.monotonic() + opc_repository.client = mock_client + mock_client.session_timeout = 600_000 # 600s: comfortably longer than this test takes + + await opc_repository._log_write_inter_arrival('session-1', 'ns=2;s=Node') + + opc_repository.emit_metric.assert_not_called() + + def test_is_reconnectable_opcua_bad(): assert is_reconnectable_opcua_bad(BadSessionIdInvalid()) is True assert is_reconnectable_opcua_bad(BadNodeIdUnknown()) is False @@ -491,6 +645,22 @@ async def test_start_reconnect_skips_within_interval(opc_repository): assert opc_repository._reconnect_task is None +@pytest.mark.asyncio +async def test_start_reconnect_skips_when_already_in_progress(opc_repository): + opc_repository.last_reconnection_time = None + in_progress_task = asyncio.create_task(asyncio.sleep(60)) + opc_repository._reconnect_task = in_progress_task + + await opc_repository._start_reconnect('BadSessionIdInvalid', 'tok') + + assert opc_repository._reconnect_task is in_progress_task + opc_repository.warning.assert_called_once() + + in_progress_task.cancel() + with pytest.raises(asyncio.CancelledError): + await in_progress_task + + @pytest.mark.asyncio async def test_write_data_protocol_closed_schedules_reconnect(opc_repository): opc_repository.client = MagicMock() @@ -608,3 +778,23 @@ async def test_reconnect_locked_sets_last_reconnection_time(mock_datetime, opc_r opc_repository._connect_locked.assert_called_once() assert result == (True, {}) assert opc_repository.last_reconnection_time == datetime(2025, 1, 1, 12, 0, 0) + + +@pytest.mark.asyncio +async def test_run_reconnect_failed_without_error_detail_logs_once(opc_repository): + """A falsy `error` (no message to add) must not log a second, empty line.""" + opc_repository._reconnect_locked = AsyncMock(return_value=(False, {})) + + await opc_repository._run_reconnect('BadSessionIdInvalid', 'tok') + + assert opc_repository.error.call_count == 1 + + +@pytest.mark.asyncio +async def test_run_reconnect_swallows_an_unexpected_exception(opc_repository): + """A bug inside the reconnect itself must not escape the background task.""" + opc_repository._reconnect_locked = AsyncMock(side_effect=RuntimeError('boom')) + + await opc_repository._run_reconnect('BadSessionIdInvalid', 'tok') + + assert opc_repository.error.call_count >= 2 diff --git a/tests/laborious/utils/test_dataframe_debug.py b/tests/laborious/utils/test_dataframe_debug.py new file mode 100644 index 0000000..0d0d6c5 --- /dev/null +++ b/tests/laborious/utils/test_dataframe_debug.py @@ -0,0 +1,40 @@ +"""Debug message formatting: full dataframe when small, a summary when it would flood the log.""" + +from pandas import DataFrame + +from laborious.utils.dataframe_debug import ( + DEFAULT_MAX_DEBUG_DATAFRAME_ROWS, + build_dataframe_debug_message, +) + + +def test_non_dataframe_payloads_are_stringified(): + assert build_dataframe_debug_message('payload:', {'a': 1}) == "payload: {'a': 1}" + + +def test_a_small_dataframe_is_logged_in_full(): + data = DataFrame({'a': [1, 2]}) + + message = build_dataframe_debug_message('payload:', data) + + assert message.startswith('payload:\n') + assert data.to_csv() in message + + +def test_a_dataframe_over_the_row_limit_is_summarized_instead(): + data = DataFrame({'a': range(DEFAULT_MAX_DEBUG_DATAFRAME_ROWS + 1)}) + + message = build_dataframe_debug_message('payload:', data) + + assert 'skipped because dataframe has' in message + assert str(data.shape) in message + assert data.to_csv() not in message + + +def test_the_row_limit_is_configurable(): + data = DataFrame({'a': [1, 2, 3]}) + + message = build_dataframe_debug_message('payload:', data, max_rows=1) + + assert 'skipped because dataframe has 3 rows' in message + assert '(max: 1)' in message diff --git a/tests/laborious/utils/test_import_password.py b/tests/laborious/utils/test_import_password.py index a852861..eaf7a5a 100644 --- a/tests/laborious/utils/test_import_password.py +++ b/tests/laborious/utils/test_import_password.py @@ -1,8 +1,11 @@ """The password envelope: AES-256-GCM in, plaintext password out, nothing in between.""" import base64 +import os +from unittest.mock import patch import pytest +from cryptography.hazmat.primitives.ciphers.aead import AESGCM from laborious.utils.import_password import ( EnvelopeError, @@ -89,6 +92,26 @@ def test_rejects_an_unusable_key(raw_key, match): load_envelope_key(raw_key) +@patch('laborious.utils.import_password.AESGCM') +def test_a_generic_decrypt_failure_is_reported_as_could_not_be_opened(mock_aesgcm): + """Anything AESGCM raises besides InvalidTag (e.g. a malformed nonce) is not authentication.""" + mock_aesgcm.return_value.decrypt.side_effect = ValueError('bad nonce length') + envelope = factory.build_envelope('Correct1Horse!Battery', KEY) + + with pytest.raises(EnvelopeError, match='could not be opened'): + decrypt_password_envelope(envelope, KEY) + + +def test_a_non_utf8_plaintext_is_rejected(): + """The envelope opened but what came out is not text — refuse rather than mangle it.""" + nonce = os.urandom(12) + sealed = AESGCM(KEY).encrypt(nonce, b'\xff\xfe\xfd', None) + envelope = base64.b64encode(nonce + sealed).decode('ascii') + + with pytest.raises(EnvelopeError, match='not valid UTF-8'): + decrypt_password_envelope(envelope, KEY) + + def test_no_secret_appears_in_the_error_text(): password = 'Correct1Horse!Battery' envelope = factory.build_envelope(password, KEY) diff --git a/tests/laborious/worker/test_worker_main.py b/tests/laborious/worker/test_worker_main.py new file mode 100644 index 0000000..e87aac7 --- /dev/null +++ b/tests/laborious/worker/test_worker_main.py @@ -0,0 +1,245 @@ +"""``main()`` and ``start_prometheus_server()``: the orchestration paths ``test_runtime_task_queues`` +doesn't reach. + +Everything I/O-bound (Temporal client, Prometheus HTTP server, Mongo/Postgres/MinIO configs) is +mocked; only the control flow inside ``worker.py`` is under test. +""" + +from unittest.mock import ANY, AsyncMock, MagicMock, patch + +from pytest import mark, raises + +from laborious.worker.worker import main, start_prometheus_server + + +@patch('laborious.worker.worker.metrics.APP_UP') +@patch('laborious.worker.worker.get_logger') +def test_main_exits_when_runtime_is_unset(mock_get_logger, mock_app_up, monkeypatch): + monkeypatch.delenv('RUNTIME', raising=False) + logger = MagicMock() + mock_get_logger.return_value = logger + + with raises(SystemExit) as exc: + import asyncio + + asyncio.run(main()) + + assert exc.value.code == 1 + logger.custom_critical.assert_called_once() + mock_app_up.labels.return_value.set.assert_called_once_with(0) + + +@patch('laborious.worker.worker.prepare_worker') +@patch('laborious.worker.worker.client.Client.connect', new_callable=AsyncMock) +@patch('laborious.worker.worker.TelemetryConfig') +@patch('laborious.worker.worker.PrometheusConfig') +@patch('laborious.worker.worker.Runtime') +@patch('laborious.worker.worker.Activities') +@patch('laborious.worker.worker.NotificationHandler') +@patch('laborious.worker.worker.build_import_status_config') +@patch('laborious.worker.worker.build_import_config') +@patch('laborious.worker.worker.build_api_config') +@patch('laborious.worker.worker.build_opc_config') +@patch('laborious.worker.worker.build_minio_config') +@patch('laborious.worker.worker.build_mlflow_config') +@patch('laborious.worker.worker.build_postgres_config') +@patch('laborious.worker.worker.build_mongodb_config') +@patch('laborious.worker.worker.start_prometheus_server') +@patch('laborious.worker.worker.get_logger') +class TestMainHappyPath: + """One class so every test shares the same (long) patch stack.""" + + def _run(self, mock_prepare_worker, mock_connect): + worker = MagicMock() + worker.run = AsyncMock(return_value=None) + mock_prepare_worker.return_value = worker + mock_connect.return_value = MagicMock() + + import asyncio + + with raises(SystemExit) as exc: + asyncio.run(main()) + return exc.value.code + + def test_runs_four_workers_without_the_import_flag( # noqa: PLR0913 + self, + mock_get_logger, + _mock_start_prometheus, + _mock_mongo_cfg, + _mock_pg_cfg, + _mock_mlflow_cfg, + _mock_minio_cfg, + _mock_opc_cfg, + _mock_api_cfg, + _mock_import_cfg, + _mock_import_status_cfg, + mock_notification_handler, + mock_activities, + _mock_runtime, + _mock_prometheus_config, + _mock_telemetry_config, + mock_connect, + mock_prepare_worker, + monkeypatch, + ): + monkeypatch.setenv('RUNTIME', 'test-runtime') + monkeypatch.delenv('IMPORT_MODEL_ENABLED', raising=False) + mock_get_logger.return_value = MagicMock() + mock_activities.return_value.init_opc = AsyncMock() + mock_activities.return_value.shutdown = AsyncMock() + + exit_code = self._run(mock_prepare_worker, mock_connect) + + assert exit_code == 0 + assert mock_prepare_worker.call_count == 4 + mock_activities.return_value.init_opc.assert_awaited_once() + mock_activities.return_value.shutdown.assert_awaited_once() + mock_notification_handler.return_value.shutdown.assert_called_once() + + def test_runs_five_workers_with_the_import_flag( # noqa: PLR0913 + self, + mock_get_logger, + _mock_start_prometheus, + _mock_mongo_cfg, + _mock_pg_cfg, + _mock_mlflow_cfg, + _mock_minio_cfg, + _mock_opc_cfg, + _mock_api_cfg, + _mock_import_cfg, + _mock_import_status_cfg, + _mock_notification_handler, + mock_activities, + _mock_runtime, + _mock_prometheus_config, + _mock_telemetry_config, + mock_connect, + mock_prepare_worker, + monkeypatch, + ): + monkeypatch.setenv('RUNTIME', 'test-runtime') + monkeypatch.setenv('IMPORT_MODEL_ENABLED', 'true') + mock_get_logger.return_value = MagicMock() + mock_activities.return_value.init_opc = AsyncMock() + mock_activities.return_value.shutdown = AsyncMock() + + exit_code = self._run(mock_prepare_worker, mock_connect) + + assert exit_code == 0 + assert mock_prepare_worker.call_count == 5 + + def test_a_falsy_notification_handler_or_activities_skips_shutdown( # noqa: PLR0913 + self, + mock_get_logger, + _mock_start_prometheus, + _mock_mongo_cfg, + _mock_pg_cfg, + _mock_mlflow_cfg, + _mock_minio_cfg, + _mock_opc_cfg, + _mock_api_cfg, + _mock_import_cfg, + _mock_import_status_cfg, + mock_notification_handler, + mock_activities, + _mock_runtime, + _mock_prometheus_config, + _mock_telemetry_config, + mock_connect, + mock_prepare_worker, + monkeypatch, + ): + """Defensive `if notification_handler:` / `if activities:` guards around shutdown — real + construction always yields truthy instances, but the guard itself still needs a test.""" + monkeypatch.setenv('RUNTIME', 'test-runtime') + monkeypatch.delenv('IMPORT_MODEL_ENABLED', raising=False) + mock_get_logger.return_value = MagicMock() + mock_activities.return_value.init_opc = AsyncMock() + mock_activities.return_value.shutdown = AsyncMock() + mock_activities.return_value.__bool__.return_value = False + mock_notification_handler.return_value.__bool__.return_value = False + + exit_code = self._run(mock_prepare_worker, mock_connect) + + assert exit_code == 0 + mock_activities.return_value.shutdown.assert_not_called() + mock_notification_handler.return_value.shutdown.assert_not_called() + + def test_a_worker_failure_exits_non_zero_and_still_cleans_up( # noqa: PLR0913 + self, + mock_get_logger, + _mock_start_prometheus, + _mock_mongo_cfg, + _mock_pg_cfg, + _mock_mlflow_cfg, + _mock_minio_cfg, + _mock_opc_cfg, + _mock_api_cfg, + _mock_import_cfg, + _mock_import_status_cfg, + mock_notification_handler, + mock_activities, + _mock_runtime, + _mock_prometheus_config, + _mock_telemetry_config, + mock_connect, + mock_prepare_worker, + monkeypatch, + ): + monkeypatch.setenv('RUNTIME', 'test-runtime') + monkeypatch.delenv('IMPORT_MODEL_ENABLED', raising=False) + logger = MagicMock() + mock_get_logger.return_value = logger + mock_activities.return_value.init_opc = AsyncMock() + mock_activities.return_value.shutdown = AsyncMock() + mock_connect.return_value = MagicMock() + + worker = MagicMock() + worker.run = AsyncMock(side_effect=RuntimeError('boom')) + mock_prepare_worker.return_value = worker + + import asyncio + + with raises(SystemExit) as exc: + asyncio.run(main()) + + assert exc.value.code == 1 + logger.custom_error.assert_called_once_with(ANY, ANY) + mock_activities.return_value.shutdown.assert_awaited_once() + mock_notification_handler.return_value.shutdown.assert_called_once() + + +@patch('laborious.worker.worker.metrics.APP_UP') +@patch('laborious.worker.worker.start_http_server') +def test_start_prometheus_server_marks_the_app_up(mock_start_http_server, mock_app_up): + start_prometheus_server() + + mock_start_http_server.assert_called_once() + mock_app_up.labels.return_value.set.assert_called_once_with(1) + + +@patch('laborious.worker.worker.os._exit') +@patch('laborious.worker.worker.metrics.APP_UP') +@patch('laborious.worker.worker.start_http_server') +def test_start_prometheus_server_exits_the_process_on_failure( + mock_start_http_server, mock_app_up, mock_os_exit +): + mock_start_http_server.side_effect = OSError('port in use') + + start_prometheus_server() + + mock_os_exit.assert_called_once_with(1) + mock_app_up.labels.return_value.set.assert_not_called() + + +@mark.parametrize('port_env', ['9999']) +@patch('laborious.worker.worker.metrics.APP_UP') +@patch('laborious.worker.worker.start_http_server') +def test_start_prometheus_server_reads_the_configured_port( + mock_start_http_server, _mock_app_up, port_env, monkeypatch +): + monkeypatch.setenv('HTTP_METRICS_PORT', port_env) + + start_prometheus_server() + + mock_start_http_server.assert_called_once_with(int(port_env)) diff --git a/tests/laborious/workflows/subworkflows/test_prediction_process.py b/tests/laborious/workflows/subworkflows/test_prediction_process.py index a018504..cdef6cd 100644 --- a/tests/laborious/workflows/subworkflows/test_prediction_process.py +++ b/tests/laborious/workflows/subworkflows/test_prediction_process.py @@ -1,6 +1,6 @@ from unittest.mock import ANY, AsyncMock, MagicMock, call, patch -from pytest import fixture, mark +from pytest import fixture, mark, raises from laborious.activities.activities import Activities from laborious.workflows.sub_workflows.prediction_process import PredictionProcess @@ -791,6 +791,30 @@ async def test_path_flag_handler_unknown(workflow_mock, prediction_process): workflow_mock.execute_child_workflow.assert_not_called() +@mark.asyncio +@patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock) +async def test_run_cleans_up_and_reraises_on_pipeline_failure(workflow_mock, prediction_process): + """A failure anywhere in the pipeline must still release the MinIO offload before the + workflow fails — cleanup is not conditional on success.""" + prediction_process._run_prediction_pipeline = AsyncMock(side_effect=RuntimeError('boom')) + input_data = { + 'metadata': metadata, + 'data': {'some': 'data'}, + 'model_id': 1, + 'model_name': 'test_model_name', + } + + with raises(RuntimeError, match='boom'): + await prediction_process.run(input_data) + + workflow_mock.execute_activity_method.assert_called_once_with( + Activities.cleanup_minio_objects_expired, + {**metadata, 'data': input_data['data']}, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + + @mark.asyncio @patch('laborious.workflows.sub_workflows.prediction_process.workflow', new_callable=AsyncMock) async def test_run_with_cleanup_prefixes(workflow_mock, prediction_process): From 192834bde3a54a174348949e4f8f57089244c27e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 8 Sep 2026 17:06:25 -0300 Subject: [PATCH 42/64] QTZPOC-20 QTZPOC-20: Refactor model import workflow, introduce atomic model ID generation, and clarify MinIO expiry. Fuses bundle download, decryption, MLflow experiment creation, and artifact upload into a single activity to prevent cross-worker issues. Implements an atomic counter for MongoDB model IDs, replacing the max(id) + 1 scan to avoid collisions during concurrent imports. Adds comprehensive documentation for MinIO object expiry mechanisms and refines lifecycle rule application. Improves error handling with clearer responsibility classification and centralizes MLflow repository injection. --- docs/minio-object-expiry.md | 40 ++ e2e/import_harness.py | 21 +- e2e/test_import_model_digest.py | 2 +- e2e/test_import_model_gates.py | 2 +- e2e/test_import_model_happy_path.py | 1 + ...test_import_model_provisioning_failures.py | 12 +- e2e/test_import_model_second_import.py | 4 +- e2e/test_import_model_status_record.py | 28 +- laborious/activities/activities.py | 22 +- laborious/activities/mlflow.py | 37 +- laborious/activities/model_import.py | 394 ++++++++++++------ laborious/activities/storage.py | 15 +- laborious/utils/bundle/reader.py | 18 +- laborious/utils/connectors_config.py | 8 + laborious/utils/messages.py | 1 + laborious/utils/models/import_errors.py | 106 +++-- .../utils/repository/model_repository.py | 19 +- laborious/utils/repository/opc_repository.py | 9 +- laborious/utils/time.py | 6 + laborious/worker/worker.py | 8 +- laborious/workflows/import_model.py | 166 ++++---- tests/laborious/activities/test_activities.py | 70 +++- tests/laborious/activities/test_mlflow.py | 35 +- .../activities/test_model_import_bundle.py | 183 ++++++-- .../test_model_import_provisioning.py | 314 +++++++++++--- .../activities/test_model_import_status.py | 113 +++++ tests/laborious/test_import_absence.py | 28 +- tests/laborious/utils/bundle/test_reader.py | 34 ++ .../utils/models/test_import_errors.py | 112 ++++- .../utils/repository/test_model_repository.py | 22 + .../laborious/utils/test_connectors_config.py | 3 + tests/laborious/utils/test_time.py | 19 + .../worker/test_runtime_task_queues.py | 2 +- .../laborious/workflows/test_import_model.py | 162 +++++-- 34 files changed, 1485 insertions(+), 531 deletions(-) create mode 100644 docs/minio-object-expiry.md create mode 100644 laborious/utils/messages.py create mode 100644 laborious/utils/time.py create mode 100644 tests/laborious/utils/test_time.py diff --git a/docs/minio-object-expiry.md b/docs/minio-object-expiry.md new file mode 100644 index 0000000..ebac303 --- /dev/null +++ b/docs/minio-object-expiry.md @@ -0,0 +1,40 @@ +# Two independent MinIO expiry mechanisms + +This platform expires objects in MinIO through two mechanisms that were built independently, for +different features, and remain independent today. This note exists so a reader who finds one does +not assume it is the whole story. + +## `ModelImport.ensure_import_bundle_retention` + +`laborious/activities/model_import.py`. Applies a **bucket lifecycle rule** (a MinIO/S3-native +`Expiration` rule, `ENABLED`, keyed by a fixed `rule_id`) to the import bundle prefix +(`import_config['prefix']`), expiring objects after `import_config['retention_days']` days. The +rule is declarative: once set, MinIO itself removes the object when it ages out — the workflow does +not scan for or delete anything. The activity's job is only to ensure the rule exists and matches +the configured `days`/`prefix`, which is what makes "lifecycle active" a criterion the code can +verify rather than an assumption about a Helm value. + +## `Storage.cleanup_minio_objects_expired` + +`laborious/activities/storage.py`. An older, unrelated activity that **scans and deletes**: given a +list of key prefixes and a retention window (`retention_hours`), it lists objects under each prefix, +compares their age against `now() - retention_hours`, and deletes the ones that have aged out. There +is no bucket-side rule — the deletion happens in this code, on whatever schedule calls the activity. + +## Why they are not unified today + +They expire different prefix families, on different units (days vs. hours), through different +mechanisms (declarative rule vs. active scan), and were written for different features at different +times. Neither mechanism depends on or interferes with the other; a bucket can carry an +`ensure_import_bundle_retention` lifecycle rule for one prefix and still have +`cleanup_minio_objects_expired` scanning a different prefix on its own cadence. + +## What unifying them would require + +Either retiring the scan-and-delete path in favor of a lifecycle rule for every prefix it currently +handles (only possible if every one of its callers can tolerate a fixed number of retention *days*, +since S3-style lifecycle rules do not support hour-granularity windows), or teaching the +lifecycle-rule path to manage a per-prefix table of rules with arbitrary retention windows instead of +the one fixed rule it manages today. Either direction touches callers and configuration this change +does not, and neither is a refactor — this is tracked as a separate, larger change, not a task of +this one. diff --git a/e2e/import_harness.py b/e2e/import_harness.py index 8d4b99d..48c962f 100644 --- a/e2e/import_harness.py +++ b/e2e/import_harness.py @@ -102,6 +102,9 @@ IMPORT_BUCKET = 'sientia-import' IMPORT_PREFIX = 'imported_models/' MODELS_COLLECTION = 'models' MONGO_DATABASE = 'sientia' +# Must track `IMPORT_MODEL_ID_COUNTERS_COLLECTION`'s default in connectors_config.py — the harness +# never sets that env var, so production's default is what the worker under test actually reads from. +MODEL_ID_COUNTERS_COLLECTION = 'model_id_counters' IMPORT_TASK_QUEUE = 'import-model-e2e-queue' # Deliberately gapped, and deliberately strings: the baseline migration stores `id` as a string and @@ -119,10 +122,7 @@ IMPORT_ACTIVITY_NAMES = ( 'record_import_terminal_status', 'report_import_status_write_failure', 'report_import_pipeline_failure', - 'download_import_bundle', - 'open_import_bundle', - 'create_import_experiment', - 'upload_import_artifacts', + 'prepare_and_upload_import_bundle', 'register_import_model_version', 'write_import_model_document', 'cleanup_import_files', @@ -331,6 +331,12 @@ def import_models_collection(import_mongo_client: MongoClient) -> Any: The index is the assertion's teeth: without it "the id continues the collection" and "a name already present is left alone" would both pass against a collection that quietly took a duplicate. + + New ids no longer come from scanning this collection — `_next_model_document_id` reads an + atomic counter in a separate `model_id_counters` collection instead (a different database, no + less), so it survives this drop untouched. Reset it here too, to the same seeded high-water + mark, or the next id handed out silently carries over from whatever an earlier test in the same + run already claimed. """ collection = import_mongo_client[MONGO_DATABASE][MODELS_COLLECTION] collection.drop() @@ -351,6 +357,13 @@ def import_models_collection(import_mongo_client: MongoClient) -> Any: ] ) collection.create_index('id', unique=True) + + counters = import_mongo_client[MONGO_DATABASE][MODEL_ID_COUNTERS_COLLECTION] + counters.replace_one( + {'_id': MODELS_COLLECTION}, + {'_id': MODELS_COLLECTION, 'seq': max(int(seeded) for seeded in SEEDED_MODEL_IDS)}, + upsert=True, + ) return collection diff --git a/e2e/test_import_model_digest.py b/e2e/test_import_model_digest.py index d3a6ee0..62c5c51 100644 --- a/e2e/test_import_model_digest.py +++ b/e2e/test_import_model_digest.py @@ -39,7 +39,7 @@ def _as_dict(value: Any) -> dict[str, Any]: def _assert_step(row: dict[str, Any], step: ImportStep, workflow_id: str) -> None: - code, reason = IMPORT_FAILURE_CATALOG[step] + code, reason, _responsibility = IMPORT_FAILURE_CATALOG[step] detail = _as_dict(row['orchestrator_response_data'])['import'] assert row['status'] == VALIDATION_ERROR_STATUS, row['status'] assert detail['step'] == step.value, detail diff --git a/e2e/test_import_model_gates.py b/e2e/test_import_model_gates.py index fd3e29d..0739e22 100644 --- a/e2e/test_import_model_gates.py +++ b/e2e/test_import_model_gates.py @@ -92,7 +92,7 @@ async def _assert_rejected( f'terminal status is {row["status"]!r}, expected {VALIDATION_ERROR_STATUS!r}' ) - expected_code, expected_reason = IMPORT_FAILURE_CATALOG[step] + expected_code, expected_reason, _expected_responsibility = IMPORT_FAILURE_CATALOG[step] detail = _as_dict(row['orchestrator_response_data'])['import'] assert detail['step'] == step.value, ( f'recorded step is {detail["step"]!r}, expected {step.value!r}' diff --git a/e2e/test_import_model_happy_path.py b/e2e/test_import_model_happy_path.py index 30bf253..293c586 100644 --- a/e2e/test_import_model_happy_path.py +++ b/e2e/test_import_model_happy_path.py @@ -244,6 +244,7 @@ class TestFactoryBundle: 'predict_flavor': 'joblib', 'retention_minutes': 60, 'target': 'SE', + 'skip_predict_timestamp': True, } row = import_read_status_row(import_run_id) diff --git a/e2e/test_import_model_provisioning_failures.py b/e2e/test_import_model_provisioning_failures.py index 44f1263..5853d97 100644 --- a/e2e/test_import_model_provisioning_failures.py +++ b/e2e/test_import_model_provisioning_failures.py @@ -69,7 +69,7 @@ def _assert_platform_failure( f"status is {row['status']!r}; a provisioning failure is not the user's to fix" ) - code, reason = IMPORT_FAILURE_CATALOG[step] + code, reason, _responsibility = IMPORT_FAILURE_CATALOG[step] detail = _as_dict(row['orchestrator_response_data'])['import'] assert detail['step'] == step.value, detail assert detail['code'] == code.value, detail @@ -152,11 +152,13 @@ class TestExperimentCreation: object_key=object_key, ) - # The bundle *was* opened before MLflow was reached, so the names are on the row — which is - # what task 9.3 asks for, observed from a failure that stopped exactly there. + # Download, decryption, experiment creation and artifact upload are one fused activity + # attempt (`model-import-fix-bugs`, bug 8): the workflow only learns the bundle's names once + # that whole attempt returns, and an experiment-creation failure means it never does. The + # placeholder names inserted with the row are therefore still there — narrowed from the old + # "names before MLflow is touched" invariant, per `model-import-status-record`'s updated spec. assert row is not None - assert row['experiment_name'] == import_names['experiment_name'] - assert row['run_name'] == f'import-{import_names["model_name"]}-v2' + assert row['experiment_name'] == 'import-e2e-placeholder' # Nothing in Mongo: the document is the last step and never ran. documents = list(import_models_collection.find({}, {'_id': False})) diff --git a/e2e/test_import_model_second_import.py b/e2e/test_import_model_second_import.py index 78ffddf..7580315 100644 --- a/e2e/test_import_model_second_import.py +++ b/e2e/test_import_model_second_import.py @@ -34,6 +34,7 @@ CONTRACTED_MODEL_CONFIG_KEYS = { 'predict_flavor', 'retention_minutes', 'target', + 'skip_predict_timestamp', } @@ -203,7 +204,7 @@ class TestModelDocumentContent: assert document['active'] is False assert set(document['model_config']) == CONTRACTED_MODEL_CONFIG_KEYS, ( - f'model_config holds {sorted(document["model_config"])}, not exactly the four ' + f'model_config holds {sorted(document["model_config"])}, not exactly the ' f'contracted keys' ) assert document['model_config'] == { @@ -211,6 +212,7 @@ class TestModelDocumentContent: 'predict_flavor': 'joblib', 'retention_minutes': 60, 'target': 'PRODUCT_QUALITY', + 'skip_predict_timestamp': True, } # The document has exactly these fields and no others — nothing was smuggled in alongside. diff --git a/e2e/test_import_model_status_record.py b/e2e/test_import_model_status_record.py index b46238e..9e93ab8 100644 --- a/e2e/test_import_model_status_record.py +++ b/e2e/test_import_model_status_record.py @@ -155,24 +155,30 @@ class TestOrdering: import_envelope: Any, import_read_status_row: Any, ) -> None: - """Observed at the first call experiment creation makes, so nothing exists in MLflow yet. + """Observed at the first call registration makes, so the experiment and run already exist. + + Download, decryption, experiment creation and artifact upload are fused into one activity + attempt (`model-import-status-record`'s design, decision on the fused step): the names are + not known to the workflow, and so cannot be recorded, until that whole activity returns — + the guarantee is anchored to the two steps that follow it, registration and the model + document, not to the experiment or run, which by construction already exist by then. This is the stronger form of the same claim task 7.1 makes from a failure: here the import - *succeeds*, and the names were still already written — so their presence is not an artefact - of the run having stopped. + *succeeds*, and the names were still already written before registration ran — so their + presence is not an artefact of the run having stopped. """ - repository = import_activities.import_mlflow_repository - real_get_experiment = repository.get_experiment + client = import_activities.import_mlflow_repository.client + real_get_registered_model = client.get_registered_model snapshots: list[dict[str, Any] | None] = [] observed_run_id: list[int] = [0] - def observing_get_experiment(*args: Any, **kwargs: Any) -> Any: + def observing_get_registered_model(*args: Any, **kwargs: Any) -> Any: snapshots.append(import_read_status_row(observed_run_id[0])) - return real_get_experiment(*args, **kwargs) + return real_get_registered_model(*args, **kwargs) object_key, digest = import_upload_bundle(import_bundle_factory()) observed_run_id[0] = import_insert_status_row(object_key=object_key, expected_digest=digest) - repository.get_experiment = observing_get_experiment + client.get_registered_model = observing_get_registered_model try: handle = await import_execute( import_build_input( @@ -184,9 +190,9 @@ class TestOrdering: ) await handle.result() finally: - repository.get_experiment = real_get_experiment + client.get_registered_model = real_get_registered_model - assert snapshots, 'experiment creation never ran' + assert snapshots, 'registration never ran' mid_flight = snapshots[0] assert mid_flight is not None assert mid_flight['status'] == RUNNING_STATUS @@ -436,7 +442,7 @@ class TestInputAgainstTheRow: row = import_read_status_row(import_run_id) assert row is not None - code, reason = IMPORT_FAILURE_CATALOG[ImportStep.RECEIVED] + code, reason, _responsibility = IMPORT_FAILURE_CATALOG[ImportStep.RECEIVED] assert row['status'] == VALIDATION_ERROR_STATUS assert row['error_message'] == reason detail = _as_dict(row['orchestrator_response_data'])['import'] diff --git a/laborious/activities/activities.py b/laborious/activities/activities.py index a06eb16..1e09238 100644 --- a/laborious/activities/activities.py +++ b/laborious/activities/activities.py @@ -20,6 +20,7 @@ with workflow.unsafe.imports_passed_through(): build_import_config, build_import_status_config, ) + from laborious.utils.repository.model_repository import MLFlowRepository class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, ModelImport): @@ -101,6 +102,15 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, ModelImport): secure=minio_config['secure'], ) + mlflow_repository = MLFlowRepository( + f'{mlflow_config["host"]}:{mlflow_config["port"]}', + mlflow_config['username'], + mlflow_config['password'], + logger, + notification_handler, + metrics_controller, + ) + # Initialize parent classes Storage.__init__( self, @@ -120,10 +130,7 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, ModelImport): MLFlow.__init__( self, - mlflow_host=mlflow_config['host'], - mlflow_port=mlflow_config['port'], - mlflow_username=mlflow_config['username'], - mlflow_password=mlflow_config['password'], + mlflow_repository=mlflow_repository, minio_repository=minio_repository, logger=logger, notification_handler=notification_handler, @@ -180,12 +187,7 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, ModelImport): status_config=import_status_config or build_import_status_config(), import_config=import_config or build_import_config(), minio_repository=minio_repository, - # The MLflow repository is shared rather than rebuilt: setting the tracking URI is a - # process-wide side effect, so two instances would be two chances to disagree. Read - # through `getattr` because `MLFlow.__init__` is what creates it — if that ever stops - # running before this line, the import activities refuse with "MLflow repository not - # initialized" at the point of use rather than silently provisioning nothing. - mlflow_repository=getattr(self, 'model_monitoring_repository', None), + mlflow_repository=mlflow_repository, mongo_repository=mongo_repository, logger=logger, notification_handler=notification_handler, diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index d075508..f6901e1 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -20,6 +20,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.utils.formatters import create_sample_dict from laborious.utils.dataframe_debug import build_dataframe_debug_message + from laborious.utils.messages import MISSING_MINIO_REPOSITORY from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload from laborious.utils.repository.minio_manager import MinioManager from laborious.utils.repository.model_repository import MLFlowRepository @@ -37,10 +38,6 @@ class MLFlow(MinioManager): MLFlow operations, ensuring reliable model inference in production environments. Attributes: - mlflow_host (str): MLFlow server hostname - mlflow_port (int): MLFlow server port - mlflow_username (str): MLFlow authentication username - mlflow_password (str): MLFlow authentication password model_monitoring_repository (MLFlowRepository): Repository for MLFlow operations """ @@ -48,45 +45,25 @@ class MLFlow(MinioManager): def __init__( self, - mlflow_host: str, - mlflow_port: int, - mlflow_username: str, - mlflow_password: str, + mlflow_repository: MLFlowRepository, minio_repository: MinioRepository | None = None, logger: Logger | None = None, notification_handler: NotificationHandler | None = None, metrics_controller: MetricsController | None = None, ): """ - Initialize MLFlow activities with server configuration. + Initialize MLFlow activities with an already-built repository. Args: - mlflow_host: MLFlow server hostname or IP address - mlflow_port: MLFlow server port number - mlflow_username: Username for MLFlow authentication - mlflow_password: Password for MLFlow authentication + mlflow_repository: Repository for MLFlow operations, built and owned by the caller + (shared with other activities rather than rebuilt here) logger: Logger instance for observability and debugging notification_handler: Notification handler for alerts and monitoring - - Raises: - Exception: If MLFlowRepository initialization fails """ MinioManager.__init__( self, minio_repository, logger, notification_handler, metrics_controller ) - self.mlflow_host = mlflow_host - self.mlflow_port = mlflow_port - self.mlflow_username = mlflow_username - self.mlflow_password = mlflow_password - - self.model_monitoring_repository = MLFlowRepository( - f'{mlflow_host}:{mlflow_port}', - mlflow_username, - mlflow_password, - logger, - notification_handler, - metrics_controller, - ) + self.model_monitoring_repository = mlflow_repository def close(self) -> None: """ @@ -338,7 +315,7 @@ class MLFlow(MinioManager): """ if self.minio_repository is None: - raise ValueError('Minio repository not initialized') + raise ValueError(MISSING_MINIO_REPOSITORY) metadata = input_data['metadata'] diff --git a/laborious/activities/model_import.py b/laborious/activities/model_import.py index 08015ae..1a6f547 100644 --- a/laborious/activities/model_import.py +++ b/laborious/activities/model_import.py @@ -28,6 +28,10 @@ with workflow.unsafe.imports_passed_through(): from minio.commonconfig import ENABLED, Filter from minio.lifecycleconfig import Expiration, LifecycleConfig, Rule + from mlflow.entities import Param + from mlflow.exceptions import MlflowException + from mlflow.protos.databricks_pb2 import RESOURCE_DOES_NOT_EXIST, ErrorCode + from pymongo import ReturnDocument from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import Logger from sientia_do.observability.metrics_controller import MetricsController @@ -38,7 +42,9 @@ with workflow.unsafe.imports_passed_through(): from sqlalchemy import create_engine, text from sqlalchemy.engine import URL from sqlalchemy.exc import DataError, IntegrityError, ProgrammingError + from sqlalchemy.sql import Executable from temporalio.common import RetryPolicy + from temporalio.exceptions import ApplicationError from laborious import metrics from laborious.utils.bundle import format as fmt @@ -49,15 +55,19 @@ with workflow.unsafe.imports_passed_through(): decrypt_password_envelope, load_envelope_key, ) + from laborious.utils.messages import MISSING_MINIO_REPOSITORY as _MISSING_MINIO from laborious.utils.models.import_errors import ( ImportErrorCode, ImportGateRejectedError, ImportLogRowNotClaimableError, + ImportMisconfiguredError, + ImportResponsibility, ImportStatusRejectedError, classify_import_failure, report_import_failure, ) from laborious.utils.repository.model_repository import MLFlowRepository + from laborious.utils.time import to_naive # The table is a constant of the code, never an input: there is exactly one import log on this # platform, and letting a caller name it would be a SQL-injection surface bought for nothing. The @@ -69,7 +79,6 @@ IMPORT_LOG_TABLE: Final[str] = 'public.experiment_run' RUN_NAME_MAX_LENGTH: Final[int] = 50 RUN_NAME_PREFIX: Final[str] = 'import-' -_MISSING_MINIO = 'Minio repository not initialized' _MISSING_MLFLOW = 'MLflow repository not initialized' _MISSING_MONGO = 'MongoDB repository not initialized' @@ -107,20 +116,6 @@ class ExperimentRunStatus(StrEnum): IMPORT_ERROR = 'IMPORT_ERROR' # new in V12 -# A failure at these steps is the user's to act on — re-export, re-type the password, re-submit — -# which is the one thing the import screen can branch on without reading the error code. -USER_ACTIONABLE_STEPS: Final[frozenset[ImportStep]] = frozenset( - { - ImportStep.RECEIVED, - ImportStep.DOWNLOAD, - ImportStep.DECRYPTION, - ImportStep.ARCHIVE_INSPECTION, - ImportStep.EXTRACTION, - ImportStep.STRUCTURE_VALIDATION, - ImportStep.CONTENT_POLICY, - } -) - # Only the status writes retry, and only because they are idempotent: `UPDATE ... SET # WHERE id` leaves the row the same however many times it runs. A provisioning call has no such # shape, which is why it keeps `maximum_attempts=1`. Defined here rather than in `sientia_do`, which @@ -144,20 +139,6 @@ IMPORT_STATUS_HINT_RETRY_POLICY: Final[RetryPolicy] = RetryPolicy( non_retryable_error_types=['ImportLogRowNotClaimableError', 'ImportStatusRejectedError'], ) -_CLAIM_SQL = text( - """ - UPDATE public.experiment_run - SET status = :status, - updated_at = :updated_at, - orchestrator_response_data = CAST(:detail AS JSONB) - WHERE id = :import_run_id - AND run_type = 'IMPORT' - AND (status = 'PENDING' - OR (status = 'ORCHESTRATOR_WAITING_PROC' - AND orchestrator_response_data -> 'import' ->> 'workflow_id' = :workflow_id)) - """ -) - _READ_ROW_SQL = text( """ SELECT id, @@ -172,39 +153,70 @@ _READ_ROW_SQL = text( """ ) + +def _build_update( + columns: dict[str, str], *, casts: dict[str, str] | None = None, extra_where: str = '' +) -> Executable: + """`UPDATE public.experiment_run SET WHERE id = :import_run_id [AND ]`. + + `columns` maps each SET target to its bind parameter name, e.g. `{'status': 'status'}` renders + `status = :status`. `casts` names which of those columns need `CAST(:name AS )` instead of + a bare `:name`. Order is preserved (Python 3.7+ dict order), so the rendered SQL is deterministic + and diff-friendly. + """ + casts = casts or {} + assignments = ',\n '.join( + f'{column} = CAST(:{param} AS {casts[column]})' + if column in casts + else f'{column} = :{param}' + for column, param in columns.items() + ) + where = 'WHERE id = :import_run_id' + if extra_where: + where = f'{where}\n AND {extra_where}' + return text( + f'UPDATE public.experiment_run\n SET {assignments}\n {where}' # nosec B608 - columns/extra_where come only from literal constants in this module, never user input + ) + + +_CLAIM_SQL = _build_update( + {'status': 'status', 'updated_at': 'updated_at', 'orchestrator_response_data': 'detail'}, + casts={'orchestrator_response_data': 'JSONB'}, + extra_where=( + "run_type = 'IMPORT'\n" + " AND (status = 'PENDING'\n" + " OR (status = 'ORCHESTRATOR_WAITING_PROC'\n" + " AND orchestrator_response_data -> 'import' ->> 'workflow_id' " + '= :workflow_id))' + ), +) + # Class B and C key on the id alone: ownership was decided once, by the claim. Re-checking it on # every later write buys nothing and costs the thing that matters — a predicate miss turning a # pipeline that already imported the model into a reported failure. -_PROGRESS_SQL = text( - """ - UPDATE public.experiment_run - SET status = :status, - updated_at = :updated_at, - orchestrator_response_data = CAST(:detail AS JSONB) - WHERE id = :import_run_id - """ +_PROGRESS_SQL = _build_update( + {'status': 'status', 'updated_at': 'updated_at', 'orchestrator_response_data': 'detail'}, + casts={'orchestrator_response_data': 'JSONB'}, ) -_NAMES_SQL = text( - """ - UPDATE public.experiment_run - SET experiment_name = :experiment_name, - run_name = :run_name, - updated_at = :updated_at, - orchestrator_response_data = CAST(:detail AS JSONB) - WHERE id = :import_run_id - """ +_NAMES_SQL = _build_update( + { + 'experiment_name': 'experiment_name', + 'run_name': 'run_name', + 'updated_at': 'updated_at', + 'orchestrator_response_data': 'detail', + }, + casts={'orchestrator_response_data': 'JSONB'}, ) -_TERMINAL_SQL = text( - """ - UPDATE public.experiment_run - SET status = :status, - error_message = :error_message, - updated_at = :updated_at, - orchestrator_response_data = CAST(:detail AS JSONB) - WHERE id = :import_run_id - """ +_TERMINAL_SQL = _build_update( + { + 'status': 'status', + 'error_message': 'error_message', + 'updated_at': 'updated_at', + 'orchestrator_response_data': 'detail', + }, + casts={'orchestrator_response_data': 'JSONB'}, ) @@ -222,7 +234,8 @@ def map_state_to_status( return ExperimentRunStatus.ORCHESTRATOR_WAITING_PROC if state is ImportState.COMPLETED: return ExperimentRunStatus.IMPORT_SUCCESS - if step is not None and ImportStep(step) in USER_ACTIONABLE_STEPS: + failure = classify_import_failure(ImportStep(step) if step is not None else None) + if failure.responsibility is ImportResponsibility.USER: return ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR return ExperimentRunStatus.IMPORT_ERROR @@ -333,8 +346,7 @@ class ModelImport(SientiaMonitoring): @staticmethod def _timestamp() -> Any: """`updated_at` is `TIMESTAMP WITHOUT TIME ZONE`, so the value handed over is naive.""" - moment = now() - return moment.replace(tzinfo=None) if moment.tzinfo else moment + return to_naive(now()) @staticmethod def _digest_from_request_data(request_data: Any) -> str | None: @@ -415,29 +427,22 @@ class ModelImport(SientiaMonitoring): import_run_id = input_data['import_run_id'] workflow_id = input_data['workflow_id'] - affected = self._execute_status_write( - _CLAIM_SQL, - { - 'status': ExperimentRunStatus.ORCHESTRATOR_WAITING_PROC.value, - 'updated_at': self._timestamp(), - 'detail': build_import_detail( - workflow_id=workflow_id, - step=ImportStep.RECEIVED, - source=input_data.get('source'), - ), - 'import_run_id': import_run_id, - 'workflow_id': workflow_id, - }, - ) - - row = self._read_row(import_run_id) - if affected == 0: - observed = self._describe_unclaimable(row, workflow_id) - error = ImportLogRowNotClaimableError( - f'import log row {import_run_id} could not be claimed: {observed}', - import_run_id=import_run_id, - observed=observed, + try: + affected = self._execute_status_write( + _CLAIM_SQL, + { + 'status': ExperimentRunStatus.ORCHESTRATOR_WAITING_PROC.value, + 'updated_at': self._timestamp(), + 'detail': build_import_detail( + workflow_id=workflow_id, + step=ImportStep.RECEIVED, + source=input_data.get('source'), + ), + 'import_run_id': import_run_id, + 'workflow_id': workflow_id, + }, ) + except ImportStatusRejectedError as error: await report_import_failure( self, None, @@ -445,10 +450,30 @@ class ModelImport(SientiaMonitoring): import_run_id=import_run_id, workflow_id=workflow_id, metadata=metadata, + code=ImportErrorCode.STATUS_NOT_RECORDED, + marker='status_write', + ) + raise + + row = self._read_row(import_run_id) + if affected == 0: + observed = self._describe_unclaimable(row, workflow_id) + not_claimable = ImportLogRowNotClaimableError( + f'import log row {import_run_id} could not be claimed: {observed}', + import_run_id=import_run_id, + observed=observed, + ) + await report_import_failure( + self, + None, + not_claimable, + import_run_id=import_run_id, + workflow_id=workflow_id, + metadata=metadata, code=ImportErrorCode.LOG_ROW_NOT_CLAIMABLE, marker='not_claimable', ) - raise error + raise not_claimable await self.emit_metric( metric_object=metrics.MODEL_IMPORT_STARTED_COUNT, @@ -562,8 +587,12 @@ class ModelImport(SientiaMonitoring): code: ImportErrorCode | None = None reason: str | None = None else: - failure = classify_import_failure(ImportStep(step) if step else None) - status = map_state_to_status(ImportState.ERROR, step) + try: + step_enum = ImportStep(step) if step else None + except ValueError: + step_enum = None + failure = classify_import_failure(step_enum) + status = map_state_to_status(ImportState.ERROR, step_enum) code, reason = failure.code, failure.reason verdict = { @@ -698,8 +727,7 @@ class ModelImport(SientiaMonitoring): # --------------------------------------------------------- download and open - @activity.defn(name='download_import_bundle') - async def download_import_bundle(self, input_data: dict[str, Any]) -> dict[str, Any]: + async def _download_bundle(self, input_data: dict[str, Any]) -> dict[str, Any]: """Gate 1, then the download, then the mandatory digest comparison. The object's size and key shape are checked from `stat_object`, before its body is fetched. @@ -722,7 +750,7 @@ class ModelImport(SientiaMonitoring): ImportGateRejectedError: gate 1 rejected the object, or the digests disagree. """ if self.import_minio_repository is None: - raise ValueError(_MISSING_MINIO) + raise ImportMisconfiguredError(_MISSING_MINIO) metadata = input_data.get('metadata', {}) bucket = self._effective_bucket(input_data.get('bucket')) @@ -765,8 +793,7 @@ class ModelImport(SientiaMonitoring): self.info(f'import bundle downloaded from {bucket} ({stat.size} bytes)', metadata) return {'path': str(target), 'size': stat.size, 'digest': digest} - @activity.defn(name='open_import_bundle') - async def open_import_bundle(self, input_data: dict[str, Any]) -> dict[str, Any]: + async def _open_bundle(self, input_data: dict[str, Any]) -> dict[str, Any]: """Open the envelope, then the bundle: gates 2 to 7, in the reader's own order. The password envelope is decrypted **here**, inside the activity that uses the password, so @@ -828,8 +855,7 @@ class ModelImport(SientiaMonitoring): # ------------------------------------------------------------ MLflow provisioning - @activity.defn(name='create_import_experiment') - async def create_import_experiment(self, input_data: dict[str, Any]) -> dict[str, Any]: + async def _create_experiment(self, input_data: dict[str, Any]) -> dict[str, Any]: """Get or create the experiment named by the bundle, keeping the origin's name. Args (input_data): @@ -840,7 +866,7 @@ class ModelImport(SientiaMonitoring): dict: `experiment_id` and `experiment_name`. """ if self.import_mlflow_repository is None: - raise ValueError(_MISSING_MLFLOW) + raise ImportMisconfiguredError(_MISSING_MLFLOW) experiment_name = input_data['experiment_name'] experiment = self.import_mlflow_repository.get_experiment( @@ -852,21 +878,21 @@ class ModelImport(SientiaMonitoring): ) return {'experiment_id': experiment.experiment_id, 'experiment_name': experiment_name} - @activity.defn(name='upload_import_artifacts') - async def upload_import_artifacts(self, input_data: dict[str, Any]) -> dict[str, Any]: + async def _upload_artifacts(self, input_data: dict[str, Any]) -> dict[str, Any]: """Create a new run with the recorded name and log the bundle's tree and parameters. Always a new run — the importer never appends to an existing one. The run is created with - exactly the `run_name` already written on the record, passed in rather than recomputed, so - the two cannot diverge. The artifact tree is logged verbatim, keeping `prediction_model` and - `data_model` at the run's artifact root, which is what the legacy loader resolves. The - bundle's `parameters` are logged unchanged, and the origin's own identity is logged - alongside them under `source.*` — records of where the model came from, not links. + exactly the `run_name` the fused activity computed, and the workflow records that same value + onto the row right after the fused activity returns, so the two cannot diverge. The artifact + tree is logged verbatim, keeping `prediction_model` and `data_model` at the run's artifact + root, which is what the legacy loader resolves. The bundle's `parameters` are logged + unchanged, and the origin's own identity is logged alongside them under `source.*` — records + of where the model came from, not links. Args (input_data): metadata (dict): workflow metadata. experiment_id (str): the experiment to create the run in. - run_name (str): the name already on the record. + run_name (str): the name to create the run with. extracted_dir (str): the opened bundle's directory. parameters (dict): the bundle's `parameters` block, verbatim. source (dict): the bundle's `metadata` block, logged as `source.*`. @@ -875,7 +901,7 @@ class ModelImport(SientiaMonitoring): dict: `run_id` and `run_name`. """ if self.import_mlflow_repository is None: - raise ValueError(_MISSING_MLFLOW) + raise ImportMisconfiguredError(_MISSING_MLFLOW) client = self.import_mlflow_repository.client run = client.create_run( @@ -886,15 +912,92 @@ class ModelImport(SientiaMonitoring): artifacts_dir = Path(input_data['extracted_dir']) / 'artifacts' client.log_artifacts(run_id, str(artifacts_dir)) - for name, value in (input_data.get('parameters') or {}).items(): - client.log_param(run_id, name, value) - for name, value in (input_data.get('source') or {}).items(): - client.log_param(run_id, f'source.{name}', value) + params = [ + Param(name, str(value)) for name, value in (input_data.get('parameters') or {}).items() + ] + params += [ + Param(f'source.{name}', str(value)) + for name, value in (input_data.get('source') or {}).items() + ] + for i in range(0, len(params), 100): # MLflow's log_batch limit is 100 params per call + client.log_batch(run_id, params=params[i : i + 100]) client.set_terminated(run_id, 'FINISHED') self.info(f'import artifacts uploaded to run {run_id}', input_data.get('metadata', {})) return {'run_id': run_id, 'run_name': input_data['run_name']} + @activity.defn(name='prepare_and_upload_import_bundle') + async def prepare_and_upload_import_bundle(self, input_data: dict[str, Any]) -> dict[str, Any]: + """Download, decrypt, open, create the experiment, and upload artifacts — one disk, one try. + + `_download_bundle`, `_open_bundle`, `_create_experiment` and `_upload_artifacts` used to be + four separately-schedulable Temporal activities sharing one worker-local scratch directory, + with nothing guaranteeing they landed on the same worker process — a real hazard once the + import worker runs more than one replica, since a later phase reading a file an earlier + phase wrote on a different pod fails with `FileNotFoundError`. Fusing them into one activity + means every disk write and every disk read this import makes happens inside one attempt, on + one process. + + The run name is computed here, from the bundle's own `model_name`/`model_version`, because it + is needed to create the run — the workflow cannot compute it beforehand, since it does not + learn the bundle's metadata until this activity returns. The workflow records that same + `run_name` onto the row immediately after this activity returns (before the registered + version and the model document are written), rather than before the run is created, which is + the narrowed guarantee `model-import-status-record`'s names requirement now makes. + + A retry of this activity redoes all four phases from the start. Re-uploading the same + artifacts to the same run is safe — MLflow overwrites identical content rather than + duplicating it — so the cost of a retry is bandwidth and time, not corruption. + + Args (input_data): + metadata (dict): workflow metadata. + workflow_id (str): names this import's scratch directory. + bucket (str, optional): the effective bucket; the configured one when absent. + object_key (str): the uploaded object's key. + expected_digest (str): the uploader's SHA-256, lowercase hex. + password_envelope (str): base64 `nonce || ciphertext || tag`, AES-256-GCM. + + Returns: + dict: `extracted_dir`, `metadata` (the bundle's own, i.e. `source`), `parameters`, + `digest`, `experiment_id`, `experiment_name`, `run_id`, `run_name`. + """ + downloaded = await self._download_bundle(input_data) + opened = await self._open_bundle({**input_data, **downloaded}) + source = opened['metadata'] + run_name = derive_run_name(source['model_name'], source['model_version']) + try: + experiment = await self._create_experiment( + {**input_data, 'experiment_name': source['experiment_name']} + ) + except ImportMisconfiguredError: + raise + except Exception as error: + # Raw MLflow/network failures carry no location of their own (unlike the gates, which + # raise `ImportGateRejectedError`), so without this the workflow's fallback step — + # whatever it was attempting before this fused activity started — misclassifies the + # failure. Retryable: this is a broken dependency, not a rejected bundle. + raise ApplicationError( + str(error), {'step': ImportStep.EXPERIMENT_CREATION.value} + ) from error + + try: + run = await self._upload_artifacts( + { + **input_data, + **opened, + **experiment, + 'run_name': run_name, + 'source': source, + } + ) + except ImportMisconfiguredError: + raise + except Exception as error: + raise ApplicationError( + str(error), {'step': ImportStep.ARTIFACT_UPLOAD.value} + ) from error + return {**opened, **experiment, **run} + @activity.defn(name='register_import_model_version') async def register_import_model_version(self, input_data: dict[str, Any]) -> dict[str, Any]: """Register a version whose source the legacy loader can parse. No stage change. @@ -920,7 +1023,7 @@ class ModelImport(SientiaMonitoring): dict: `model_name`, `version` and `source`. """ if self.import_mlflow_repository is None: - raise ValueError(_MISSING_MLFLOW) + raise ImportMisconfiguredError(_MISSING_MLFLOW) client = self.import_mlflow_repository.client model_name = input_data['model_name'] @@ -928,7 +1031,9 @@ class ModelImport(SientiaMonitoring): try: client.get_registered_model(model_name) - except Exception: + except MlflowException as error: + if error.error_code != ErrorCode.Name(RESOURCE_DOES_NOT_EXIST): + raise # First import of this name: the registered model is created on demand. A second import # lands as a new version of the same registered model, never a differently named one. client.create_registered_model(model_name) @@ -949,10 +1054,11 @@ class ModelImport(SientiaMonitoring): This is what the runtime reads to know how to load a model — `model_config.transform_flavor`, `model_config.predict_flavor` and `model_config.target` — so without it an imported model is - invisible to every workflow. `id` continues the collection's own numbering: the largest - existing id **as an integer** plus one, never a count and never a lexicographic sort, which - would answer `"9" > "30"`. A name already in the collection is left exactly as it is: - `active`, `target` and the flavors of a model already on the platform are operator-owned. + invisible to every workflow. `id` continues the collection's own numbering, assigned by an + atomic counter (`_next_model_document_id`) rather than a scan of existing ids, so two + concurrent imports can never collide on the same id. A name already in the collection is left + exactly as it is: `active`, `target` and the flavors of a model already on the platform are + operator-owned. Args (input_data): metadata (dict): workflow metadata. @@ -960,10 +1066,11 @@ class ModelImport(SientiaMonitoring): target (str): the bundle's `parameters.target_variable`. Returns: - dict: `written`, `id`, `skipped` and how many unusable ids were ignored. + dict: `written`, `id`, `skipped` and `ignored_ids` (always `0`, kept for shape + compatibility — no id is skipped under the counter). """ if self.mongo_repository is None: - raise ValueError(_MISSING_MONGO) + raise ImportMisconfiguredError(_MISSING_MONGO) metadata = input_data.get('metadata', {}) collection = self.import_config['models_collection'] @@ -986,6 +1093,7 @@ class ModelImport(SientiaMonitoring): 'predict_flavor': 'joblib', 'retention_minutes': 60, 'target': input_data['target'], + 'skip_predict_timestamp': True, }, } await self.mongo_repository.insert(collection, document, metadata) @@ -995,22 +1103,32 @@ class ModelImport(SientiaMonitoring): async def _next_model_document_id( self, collection: str, metadata: dict[str, Any] ) -> tuple[str, int]: - """`max(int(id)) + 1` as a decimal string, tolerating ids that are not numbers.""" - if self.mongo_repository is None: - raise ValueError(_MISSING_MONGO) - raw_ids = await self.mongo_repository.distinct(collection, 'id', {}, metadata) - numeric: list[int] = [] - ignored = 0 - for value in raw_ids: - try: - numeric.append(int(str(value))) - except (TypeError, ValueError): - # A document whose id is not a decimal number cannot take part in the numbering. - # Ignored rather than fatal, and counted so the log says how many there were. - ignored += 1 - if ignored: - self.info(f'ignored {ignored} non-numeric model document ids', metadata) - return str(max(numeric, default=0) + 1), ignored + """Atomically increment a counter dedicated to `collection`, and return its new value. + + Reaches MongoDB through the repository's own public `database` handle — the same access + pattern `find_one`/`distinct` already use internally — with pymongo's native + `find_one_and_update`, rather than the max-scan this replaces. That scan raced: two imports + writing at the same time could both read the same maximum and collide on the same id. An + atomic `$inc` cannot. + + The counter's backing collection (`import_config['counters_collection']`) is an external + dependency: it must be created and seeded by a migration before this runs against a given + environment (see `openspec/changes/model-import-fix-bugs/design.md`, Decision 3). This code + does not create or seed it. + + Assumes `self.mongo_repository` is not `None` — the only caller, `write_import_model_document`, + already raises before calling this when it is. + """ + counters = self.mongo_repository.database[ # type: ignore[union-attr] + self.import_config['counters_collection'] + ] + doc = counters.find_one_and_update( + {'_id': collection}, + {'$inc': {'seq': 1}}, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + return str(doc['seq']), 0 # ----------------------------------------------------------- cleanup and retention @@ -1059,6 +1177,10 @@ class ModelImport(SientiaMonitoring): ("lifecycle active") is verifiable from the code that depends on it. The uploaded object is never deleted by the import itself — it is left to this rule, as the record of an attempt. + See also: `Storage.cleanup_minio_objects_expired` (laborious/activities/storage.py) — an + older, unrelated scan-and-delete expiry mechanism for a different prefix family. The two are + independent by design for now; see docs/minio-object-expiry.md. + Args (input_data): metadata (dict): workflow metadata. bucket (str, optional): the effective bucket; the configured one when absent. @@ -1067,7 +1189,7 @@ class ModelImport(SientiaMonitoring): dict: whether the rule was already there and what it expires. """ if self.import_minio_repository is None: - raise ValueError(_MISSING_MINIO) + raise ImportMisconfiguredError(_MISSING_MINIO) metadata = input_data.get('metadata', {}) bucket = self._effective_bucket(input_data.get('bucket')) @@ -1083,6 +1205,20 @@ class ModelImport(SientiaMonitoring): # deployments and with `None` on others; both mean "there is no rule yet". existing = None + existing_rule = next( + (rule for rule in (existing.rules if existing else []) if rule.rule_id == rule_id), None + ) + if ( + existing_rule is not None + and existing_rule.status == ENABLED + and getattr(existing_rule.expiration, 'days', None) == days + and getattr(existing_rule.rule_filter, 'prefix', None) == prefix + ): + # The read still runs every time (it is how drift — someone editing the rule outside + # this code — gets detected); only the write is skipped when nothing has drifted. + self.info(f'import retention rule already correct on {bucket} ({days} days)', metadata) + return {'bucket': bucket, 'prefix': prefix, 'days': days, 'rules': len(existing.rules)} + rules = [rule for rule in (existing.rules if existing else []) if rule.rule_id != rule_id] rules.append( Rule( diff --git a/laborious/activities/storage.py b/laborious/activities/storage.py index 7c67466..7bafbe2 100644 --- a/laborious/activities/storage.py +++ b/laborious/activities/storage.py @@ -17,7 +17,9 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.temporal.activities.postgres import Postgres from sientia_do.temporal.constants import now + from laborious.utils.messages import MISSING_MINIO_REPOSITORY from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload + from laborious.utils.time import to_naive _LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'}) @@ -83,7 +85,7 @@ class Storage(Postgres, MinioManager): dict[str, Any]: Flat ``MinioDataFramePayload`` dict or ``success: False`` on failure. """ if self.minio_repository is None: - raise ValueError('Minio repository not initialized') + raise ValueError(MISSING_MINIO_REPOSITORY) metadata: dict = input_data.get('metadata', {}) model_name = input_data['model_name'] @@ -129,6 +131,11 @@ class Storage(Postgres, MinioManager): """ Delete objects under the given prefixes that are older than the retention window. + See also: `ModelImport.ensure_import_bundle_retention` + (laborious/activities/model_import.py) — a newer, unrelated lifecycle-policy expiry + mechanism for the import bundle prefix. The two are independent by design for now; see + docs/minio-object-expiry.md. + Args (input_data): metadata (dict): Workflow metadata for logging and metrics. prefixes (list[str]): Key prefixes to scan (one level or subtree per prefix). @@ -137,15 +144,13 @@ class Storage(Postgres, MinioManager): dict[str, Any]: ``success``, ``deleted_count``, and optional ``message``. """ if self.minio_repository is None: - raise ValueError('Minio repository not initialized') + raise ValueError(MISSING_MINIO_REPOSITORY) metadata = input_data.get('metadata', {}) payload = MinioDataFramePayload.from_dict(input_data['data']) prefix = payload.cleanup_prefix() base = now() - cutoff = (base.replace(tzinfo=None) if base.tzinfo else base) - timedelta( - hours=self.retention_hours - ) + cutoff = to_naive(base) - timedelta(hours=self.retention_hours) report: dict[str, Any] = { 'failed': {}, diff --git a/laborious/utils/bundle/reader.py b/laborious/utils/bundle/reader.py index 61fcd90..aea9646 100644 --- a/laborious/utils/bundle/reader.py +++ b/laborious/utils/bundle/reader.py @@ -180,8 +180,8 @@ class BundleReader: header = self._gate_2_header(encrypted) zip_path, kdf_seconds = self._gate_3_decrypt(encrypted, password, header, staging) try: - self._gate_4_inspect(zip_path) - self._gate_5_extract(zip_path, root) + entries = self._gate_4_inspect(zip_path) + self._gate_5_extract(zip_path, root, entries) metadata, parameters = self._gate_6_structure(root) self._gate_7_content_policy(root) except BaseException: @@ -282,7 +282,7 @@ class BundleReader: # ------------------------------------------------------------------ gate 4 - def _gate_4_inspect(self, zip_path: Path) -> None: + def _gate_4_inspect(self, zip_path: Path) -> list[zipfile.ZipInfo]: """Inspect the central directory only. Nothing is written to disk by this gate.""" try: with zipfile.ZipFile(zip_path) as archive: @@ -319,6 +319,8 @@ class BundleReader: self._check_entry_name(entry.filename) self._check_entry_mode(entry) + return entries + @staticmethod def _check_entry_name(name: str) -> None: """Reject absolute names and traversal, from the declared name alone.""" @@ -346,11 +348,13 @@ class BundleReader: # ------------------------------------------------------------------ gate 5 - def _gate_5_extract(self, zip_path: Path, root: Path) -> None: + def _gate_5_extract(self, zip_path: Path, root: Path, entries: list[zipfile.ZipInfo]) -> None: """Extract into a fresh directory, containing every member by resolved path. A pre-existing extraction root is a failure rather than something to clear: it means - another import, or a previous attempt, owns that path. + another import, or a previous attempt, owns that path. `entries` is the central-directory + listing gate 4 already read; the archive is still opened here to read member bytes, but its + `infolist()` is not read a second time. """ if root.exists(): raise BundleGateError('extraction directory already exists', gate=5) @@ -358,7 +362,7 @@ class BundleReader: resolved_root = root.resolve() try: with zipfile.ZipFile(zip_path) as archive: - for entry in archive.infolist(): + for entry in entries: self._extract_member(archive, entry, root, resolved_root) except BundleGateError: shutil.rmtree(root, ignore_errors=True) @@ -461,7 +465,7 @@ class BundleReader: def _gate_7_content_policy(self, root: Path) -> None: """Only the file kinds an MLflow artifact tree calls for may be present.""" - for path in sorted(root.rglob('*')): + for path in root.rglob('*'): if path.is_dir(): continue if not path.is_file(): diff --git a/laborious/utils/connectors_config.py b/laborious/utils/connectors_config.py index 4b31496..01e29e1 100644 --- a/laborious/utils/connectors_config.py +++ b/laborious/utils/connectors_config.py @@ -112,6 +112,13 @@ def build_import_config() -> dict[str, Any]: IMPORT_MAX_UNCOMPRESSED_BYTES: gate 4 uncompressed size ceiling (default: 4 GiB) IMPORT_MAX_COMPRESSION_RATIO: gate 4 compression ratio ceiling (default: 200) IMPORT_MODELS_COLLECTION: MongoDB collection holding the model listing (default: models) + IMPORT_MODEL_ID_COUNTERS_COLLECTION: MongoDB collection holding the atomic id counters that + assign `models` document ids (default: model_id_counters). **External dependency**: this + collection and its seed value (`seq = max(int(id) for id in models)`) are not created by + this repository — a migration must add them to + `sientia-dataops-database-migrations/mongodb/migrations/`, following the `models` + baseline's `ensureCollection`/`ensureIndexes` convention, before any environment's counter + is first incremented. See `openspec/changes/model-import-fix-bugs/design.md`, Decision 3. IMPORT_BUNDLE_RETENTION_DAYS: lifecycle expiry for the uploaded bundle (default: 7) Returns: @@ -130,6 +137,7 @@ def build_import_config() -> dict[str, Any]: ), 'max_compression_ratio': float(getenv('IMPORT_MAX_COMPRESSION_RATIO', '200')), 'models_collection': getenv('IMPORT_MODELS_COLLECTION', 'models'), + 'counters_collection': getenv('IMPORT_MODEL_ID_COUNTERS_COLLECTION', 'model_id_counters'), 'retention_days': int(getenv('IMPORT_BUNDLE_RETENTION_DAYS', '7')), } diff --git a/laborious/utils/messages.py b/laborious/utils/messages.py new file mode 100644 index 0000000..1b84ea3 --- /dev/null +++ b/laborious/utils/messages.py @@ -0,0 +1 @@ +MISSING_MINIO_REPOSITORY = 'Minio repository not initialized' diff --git a/laborious/utils/models/import_errors.py b/laborious/utils/models/import_errors.py index 8312bd4..8d3c88c 100644 --- a/laborious/utils/models/import_errors.py +++ b/laborious/utils/models/import_errors.py @@ -17,7 +17,8 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): import traceback - from collections.abc import Mapping + from collections import deque + from collections.abc import Iterator, Mapping from dataclasses import dataclass from enum import StrEnum from types import MappingProxyType @@ -58,74 +59,95 @@ class ImportErrorCode(StrEnum): CLEANUP_INCOMPLETE = 'IMPORT_CLEANUP_INCOMPLETE' +class ImportResponsibility(StrEnum): + """Who can act on a step's failure: the user (re-export, re-type, re-submit) or only support.""" + + USER = 'user' + SUPPORT = 'support' + + @dataclass(frozen=True) class ImportFailure: - """One classification of one failure: the step, its code and the sentence it stores.""" + """One classification of one failure: the step, its code, its sentence and whose to fix.""" step: ImportStep | None code: ImportErrorCode reason: str + responsibility: ImportResponsibility = ImportResponsibility.SUPPORT _CONTACT_SUPPORT = "Contact support with this import's identifier." _RE_EXPORT = 'Export the model again from the origin platform.' -IMPORT_FAILURE_CATALOG: Final[Mapping[ImportStep, tuple[ImportErrorCode, str]]] = MappingProxyType( +IMPORT_FAILURE_CATALOG: Final[ + Mapping[ImportStep, tuple[ImportErrorCode, str, ImportResponsibility]] +] = MappingProxyType( { ImportStep.RECEIVED: ( ImportErrorCode.REQUEST_INVALID, 'The import could not be started because the request was incomplete or ' 'contradictory. Nothing was created. Submit the import again.', + ImportResponsibility.USER, ), ImportStep.DOWNLOAD: ( ImportErrorCode.FILE_UNREADABLE, 'The uploaded file could not be read, or it is not the file that was sent. ' 'Upload it again.', + ImportResponsibility.USER, ), ImportStep.DECRYPTION: ( ImportErrorCode.BUNDLE_CANNOT_BE_OPENED, 'This file could not be opened: the password is wrong, or the file is damaged or ' 'was altered. Confirm the password with whoever exported the model, then try ' 'again.', + ImportResponsibility.USER, ), ImportStep.ARCHIVE_INSPECTION: ( ImportErrorCode.BUNDLE_REJECTED, "The file's contents did not pass the platform's safety checks, so it was not " f'opened. {_RE_EXPORT}', + ImportResponsibility.USER, ), ImportStep.EXTRACTION: ( ImportErrorCode.BUNDLE_UNPACK_FAILED, f'The file could not be unpacked safely and nothing from it was kept. {_RE_EXPORT}', + ImportResponsibility.USER, ), ImportStep.STRUCTURE_VALIDATION: ( ImportErrorCode.BUNDLE_INCOMPLETE, 'This file is not a complete model export — part of what the platform needs is ' f'missing from it. {_RE_EXPORT}', + ImportResponsibility.USER, ), ImportStep.CONTENT_POLICY: ( ImportErrorCode.BUNDLE_UNEXPECTED_CONTENT, 'The file contains something a model export should not contain, so it was ' f'rejected. {_RE_EXPORT}', + ImportResponsibility.USER, ), ImportStep.EXPERIMENT_CREATION: ( ImportErrorCode.STORAGE_PREPARATION_FAILED, 'The platform could not prepare a place to keep this model. The import stopped ' f'and no model was created. {_CONTACT_SUPPORT}', + ImportResponsibility.SUPPORT, ), ImportStep.ARTIFACT_UPLOAD: ( ImportErrorCode.MODEL_FILES_NOT_STORED, "The model's files could not be stored on this platform. The import stopped and " f'the model is not available. {_CONTACT_SUPPORT}', + ImportResponsibility.SUPPORT, ), ImportStep.REGISTRATION: ( ImportErrorCode.MODEL_NOT_REGISTERED, "The model's files were stored, but the model itself could not be registered, so " f'it cannot be used. {_CONTACT_SUPPORT}', + ImportResponsibility.SUPPORT, ), ImportStep.MODEL_DOCUMENT: ( ImportErrorCode.MODEL_SETTINGS_NOT_SAVED, 'The model was registered, but the settings that tell the platform how to run it ' f'could not be saved. {_CONTACT_SUPPORT}', + ImportResponsibility.SUPPORT, ), } ) @@ -135,36 +157,44 @@ UNCLASSIFIED_FAILURE: Final[tuple[ImportErrorCode, str]] = ( f'The import stopped for an unexpected reason and no model was created. {_CONTACT_SUPPORT}', ) +# The three explicit-code failures are not pipeline steps, so `classify_import_failure` falls +# through to `UNCLASSIFIED_FAILURE`'s reason for all of them. That sentence says "no model was +# created", which is false for a cleanup-only failure (the model was imported) and misleading for +# the other two (neither ever attempted the pipeline). Each gets its own reason here instead. +_OVERRIDE_REASONS: Final[Mapping[ImportErrorCode, str]] = MappingProxyType( + { + ImportErrorCode.CLEANUP_INCOMPLETE: ( + 'The model was imported successfully; only its temporary files could not be removed ' + 'from the worker. No action needed.' + ), + ImportErrorCode.LOG_ROW_NOT_CLAIMABLE: 'The import request could not be started.', + ImportErrorCode.STATUS_NOT_RECORDED: 'The import outcome could not be recorded.', + } +) + # `cleanup` deletes temporary files on the worker. It is never the reported failure: when every # other step succeeded and only cleanup failed, the import is a success and the leak is a log line. # So it deliberately has no user-facing sentence, and the catalog-completeness test knows it. STEPS_WITHOUT_A_USER_FACING_REASON: Final[frozenset[ImportStep]] = frozenset({ImportStep.CLEANUP}) -def classify_import_failure( - step: ImportStep | None, exception: BaseException | None = None -) -> ImportFailure: +def classify_import_failure(step: ImportStep | None) -> ImportFailure: """Turn a failure into the one code and the one sentence that will be recorded. - The exception's text is **ignored entirely** — it is what carries paths, table names and type - names, and it belongs in the log. What decides the outcome is the step. + The step alone decides the outcome — an exception's text carries paths, table names and type + names, and belongs in the log, never in the record. Args: step: the step that was being attempted, or `None` when none had been entered. - exception: the failure, when there is one to hand. It is accepted so callers read as - "classify this failure" and never absent because the classification needs it: the - catalog is keyed by step, and an exception cannot cross an activity boundary with its - type intact anyway — the terminal write classifies from the step alone. Returns: ImportFailure: step, code and the reason to store. """ - del exception # the catalog is keyed by step; the exception's text never reaches the record if step is None or step not in IMPORT_FAILURE_CATALOG: code, reason = UNCLASSIFIED_FAILURE return ImportFailure(step=step, code=code, reason=reason) - code, reason = IMPORT_FAILURE_CATALOG[step] - return ImportFailure(step=step, code=code, reason=reason) + code, reason, responsibility = IMPORT_FAILURE_CATALOG[step] + return ImportFailure(step=step, code=code, reason=reason, responsibility=responsibility) class ImportLogRowNotClaimableError(ApplicationError): @@ -198,6 +228,13 @@ class ImportStatusRejectedError(ApplicationError): super().__init__(message, type='ImportStatusRejectedError', non_retryable=True) +class ImportMisconfiguredError(ApplicationError): + """A required repository was not injected into this worker. A deploy defect, never transient.""" + + def __init__(self, message: str) -> None: + super().__init__(message, type='ImportMisconfiguredError', non_retryable=True) + + class ImportInputError(ApplicationError): """The request is incomplete or contradictory. Fails at `received`, on the claimed row.""" @@ -241,6 +278,18 @@ _CHAIN_LIMIT = 10 _UNKNOWN_ERROR_TYPE = 'unavailable' +def walk_cause_chain( + exception: BaseException, limit: int = _CHAIN_LIMIT +) -> Iterator[BaseException]: + """Yield `exception`, then each `__cause__`, up to `limit` links. Bounded against a cycle.""" + candidate: BaseException | None = exception + for _ in range(limit): + if candidate is None: + return + yield candidate + candidate = candidate.__cause__ + + def _root_cause(exception: BaseException) -> BaseException: """The deepest `__cause__` in the chain: the failure that actually happened. @@ -249,12 +298,7 @@ def _root_cause(exception: BaseException) -> BaseException: says nothing an operator can act on. The chain is walked to its end, and bounded, because a cycle in `__cause__` would otherwise hang the reporting of a failure. """ - candidate = exception - for _ in range(_CHAIN_LIMIT): - if candidate.__cause__ is None: - break - candidate = candidate.__cause__ - return candidate + return deque(walk_cause_chain(exception), maxlen=1)[-1] def _type_name(exception: BaseException) -> str: @@ -316,10 +360,7 @@ def describe_failure_chain(exception: BaseException) -> tuple[str, str]: `ApplicationError` whose `type` says so. """ links: list[str] = [] - candidate: BaseException | None = exception - for _ in range(_CHAIN_LIMIT): - if candidate is None: - break + for candidate in walk_cause_chain(exception): kind = _type_name(candidate) message = _one_line(str(candidate)) # `ApplicationError.__str__` already prefixes its own type, and `MlflowException: @@ -330,7 +371,6 @@ def describe_failure_chain(exception: BaseException) -> tuple[str, str]: links.append(message) else: links.append(f'{kind}: {message}') - candidate = candidate.__cause__ links.reverse() root = links[0].split(':', 1)[0] if links else _UNKNOWN_ERROR_TYPE return root, '\n'.join(links) @@ -379,16 +419,14 @@ def failure_location(exception: BaseException | None) -> tuple[str | None, int | Returns `(None, None)` when the failure did not say where it happened, and the caller then falls back to the step it was attempting. """ - seen = 0 - candidate: BaseException | None = exception - while candidate is not None and seen < 10: + if exception is None: + return None, None + for candidate in walk_cause_chain(exception): details = getattr(candidate, 'details', None) if details and isinstance(details[0], dict) and 'step' in details[0]: location = details[0] gate = location.get('gate') return location.get('step'), int(gate) if isinstance(gate, int) else None - candidate = candidate.__cause__ - seen += 1 return None, None @@ -466,9 +504,11 @@ async def report_import_failure( Returns: ImportFailure: the classification that was reported. """ - failure = classify_import_failure(step, exception) + failure = classify_import_failure(step) if code is not None: - failure = ImportFailure(step=step, code=code, reason=failure.reason) + failure = ImportFailure( + step=step, code=code, reason=_OVERRIDE_REASONS.get(code, failure.reason) + ) where = marker or (step.value if step else 'unknown') if exception is not None: diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 15b4a10..1c32baf 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -807,16 +807,23 @@ class MLFlowRepository(SientiaMonitoring): ) if load_wrapper: + core_labels = self.get_core_labels(metadata, operation_type='download_model_wrapper') self.info(f'Loading wrapper for {model_type} model {model_name} with flavor {flavor}') - artifact_path = await self.dowload_artifacts(model_name, metadata, target) + try: + artifact_path = await self.dowload_artifacts(model_name, metadata, target) - self.info( - f'Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}' - ) + self.info( + f'Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}' + ) - raw_model = mlflow.pyfunc.load_model(artifact_path) - model = raw_model._model_impl.python_model + raw_model = mlflow.pyfunc.load_model(artifact_path) + model = raw_model._model_impl.python_model + except Exception as e: + await self.emit_metric( + metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels + ) + raise e self.debug( f'Model wrapper loaded: {model.__class__.__name__}:{model.__dict__}', metadata diff --git a/laborious/utils/repository/opc_repository.py b/laborious/utils/repository/opc_repository.py index 2ecc27a..802c318 100644 --- a/laborious/utils/repository/opc_repository.py +++ b/laborious/utils/repository/opc_repository.py @@ -17,6 +17,7 @@ from sientia_do.observability.metrics_controller import MetricsController from sientia_do.observability.sientia_monitoring import SientiaMonitoring from laborious import metrics +from laborious.utils.models.import_errors import walk_cause_chain # Requested session and secure channel lifetime (ms) before server revision; 10 minutes. OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS = 10 * 60 * 1000 @@ -92,11 +93,9 @@ def _opc_status_from_exception(exc: BaseException) -> str: Return: str: Status class name or generic Python exception name. """ - current: BaseException | None = exc - while current is not None: - if isinstance(current, UaStatusCodeError): - return type(current).__name__ - current = current.__cause__ + for candidate in walk_cause_chain(exc): + if isinstance(candidate, UaStatusCodeError): + return type(candidate).__name__ return type(exc).__name__ diff --git a/laborious/utils/time.py b/laborious/utils/time.py new file mode 100644 index 0000000..eebe7c5 --- /dev/null +++ b/laborious/utils/time.py @@ -0,0 +1,6 @@ +from datetime import datetime + + +def to_naive(moment: datetime) -> datetime: + """Strip tzinfo, if any. Postgres `TIMESTAMP WITHOUT TIME ZONE` columns need naive values.""" + return moment.replace(tzinfo=None) if moment.tzinfo else moment diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index f4dbf4d..09cc2c8 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -278,12 +278,10 @@ async def main(): activities.record_import_terminal_status, activities.report_import_status_write_failure, activities.report_import_pipeline_failure, - # Bundle - activities.download_import_bundle, - activities.open_import_bundle, + # Bundle download/open, MLflow experiment creation and artifact upload — one + # fused activity, so they never split across worker processes. + activities.prepare_and_upload_import_bundle, # MLflow provisioning - activities.create_import_experiment, - activities.upload_import_artifacts, activities.register_import_model_version, # The model listing activities.write_import_model_document, diff --git a/laborious/workflows/import_model.py b/laborious/workflows/import_model.py index b8aadd2..3a81df7 100644 --- a/laborious/workflows/import_model.py +++ b/laborious/workflows/import_model.py @@ -16,11 +16,13 @@ Shape of this workflow, and the reasons it has that shape: boundary, and the password reaches this module only as an envelope it cannot open. """ +import asyncio from datetime import timedelta from temporalio import workflow with workflow.unsafe.imports_passed_through(): + from collections.abc import Awaitable, Callable from typing import Any from sientia_do.temporal.policies import retry_policy @@ -29,7 +31,6 @@ with workflow.unsafe.imports_passed_through(): from laborious.activities.model_import import ( IMPORT_STATUS_HINT_RETRY_POLICY, IMPORT_STATUS_RETRY_POLICY, - derive_run_name, ) from laborious.utils.bundle.steps import ImportStep from laborious.utils.models.import_errors import ( @@ -44,10 +45,15 @@ with workflow.unsafe.imports_passed_through(): # which is the one configuration mistake that would silently undo design D20. STATUS_WRITE_TIMEOUT = timedelta(seconds=30) STATUS_WRITE_ENVELOPE = timedelta(seconds=900) -DOWNLOAD_TIMEOUT = timedelta(seconds=1800) -OPEN_TIMEOUT = timedelta(seconds=1800) -EXPERIMENT_TIMEOUT = timedelta(seconds=300) -ARTIFACT_UPLOAD_TIMEOUT = timedelta(seconds=1800) +# Download, decryption/extraction, experiment creation and artifact upload now run inside one +# activity attempt (bug 8): the budget is the sum of what the four separate activities used to get, +# since a retry of the fused activity redoes all four phases. +PREPARE_AND_UPLOAD_TIMEOUT = ( + timedelta(seconds=1800) + + timedelta(seconds=1800) + + timedelta(seconds=300) + + timedelta(seconds=1800) +) REGISTRATION_TIMEOUT = timedelta(seconds=600) DOCUMENT_TIMEOUT = timedelta(seconds=300) CLEANUP_TIMEOUT = timedelta(seconds=600) @@ -124,8 +130,7 @@ class ImportModel: schedule_to_close_timeout=STATUS_WRITE_ENVELOPE, ) - current_step = ImportStep.RECEIVED - failed_step: str | None = None + step: str = ImportStep.RECEIVED.value gate: int | None = None succeeded = False source: dict[str, Any] = {} @@ -134,79 +139,74 @@ class ImportModel: try: self._validate_input(input_data, claim) - current_step = ImportStep.DOWNLOAD - await self._hint(base, current_step) - await self._ensure_retention(base, input_data) - download = await workflow.execute_activity_method( - Activities.download_import_bundle, - { - **base, - 'bucket': input_data.get('bucket'), - 'object_key': input_data['object_key'], - 'expected_digest': input_data['expected_digest'], - }, - retry_policy=retry_policy, - start_to_close_timeout=DOWNLOAD_TIMEOUT, + # Download, decryption/extraction, experiment creation and artifact upload run inside + # one activity attempt (bug 8), so no later phase can depend on a file a different, + # separately-scheduled activity wrote to a different worker's disk. One hint covers the + # whole fused step; an exception it raises with no location of its own falls back to + # this step, since that is what the workflow was attempting. + step = ImportStep.DOWNLOAD.value + await self._record_hint( + base, Activities.record_import_status, {'step': step, 'source': None} ) - - current_step = ImportStep.DECRYPTION - await self._hint(base, current_step) - opened = await workflow.execute_activity_method( - Activities.open_import_bundle, - { - **base, - 'path': download['path'], - 'password_envelope': input_data['password_envelope'], - }, - retry_policy=retry_policy, - start_to_close_timeout=OPEN_TIMEOUT, + # Achado #9: the retention check carries no data this step consumes and already + # swallows its own failure (`_ensure_retention`), so it runs alongside the fused + # download+decrypt+provision activity instead of blocking it. Achado #8 fused + # `download_import_bundle` into `prepare_and_upload_import_bundle` before this change + # landed, so this gathers against that fused activity, not the original isolated one. + _, prepared = await asyncio.gather( + self._ensure_retention(base, input_data), + workflow.execute_activity_method( + Activities.prepare_and_upload_import_bundle, + { + **base, + 'bucket': input_data.get('bucket'), + 'object_key': input_data['object_key'], + 'expected_digest': input_data['expected_digest'], + 'password_envelope': input_data['password_envelope'], + }, + retry_policy=retry_policy, + start_to_close_timeout=PREPARE_AND_UPLOAD_TIMEOUT, + ), ) - source = opened['metadata'] - parameters = opened['parameters'] + source = prepared['metadata'] + parameters = prepared['parameters'] model_name = source['model_name'] metadata['model_name'] = model_name - run_name = derive_run_name(model_name, source['model_version']) + run_name = prepared['run_name'] - # The names go on the record before anything exists in MLflow, so a person watching the - # import list sees *which* model is being imported while it is still running. - await self._record_names(base, source, run_name) - - current_step = ImportStep.EXPERIMENT_CREATION - await self._hint(base, current_step, source) - experiment = await workflow.execute_activity_method( - Activities.create_import_experiment, - {**base, 'experiment_name': source['experiment_name']}, - retry_policy=retry_policy, - start_to_close_timeout=EXPERIMENT_TIMEOUT, - ) - - current_step = ImportStep.ARTIFACT_UPLOAD - await self._hint(base, current_step, source) - run = await workflow.execute_activity_method( - Activities.upload_import_artifacts, + # The names are known only once the fused activity returns; they are still recorded + # before the two provisioning steps that follow (the registered version and the model + # document), which is the guarantee `model-import-status-record` now makes. + await self._record_hint( + base, + Activities.record_import_names, { - **base, - 'experiment_id': experiment['experiment_id'], + 'experiment_name': source['experiment_name'], 'run_name': run_name, - 'extracted_dir': opened['extracted_dir'], - 'parameters': parameters, 'source': source, + 'step': ImportStep.EXPERIMENT_CREATION.value, }, - retry_policy=retry_policy, - start_to_close_timeout=ARTIFACT_UPLOAD_TIMEOUT, ) - current_step = ImportStep.REGISTRATION - await self._hint(base, current_step, source) + step = ImportStep.REGISTRATION.value + await self._record_hint( + base, + Activities.record_import_status, + {'step': step, 'source': source or None}, + ) version = await workflow.execute_activity_method( Activities.register_import_model_version, - {**base, 'model_name': model_name, 'run_id': run['run_id']}, + {**base, 'model_name': model_name, 'run_id': prepared['run_id']}, retry_policy=retry_policy, start_to_close_timeout=REGISTRATION_TIMEOUT, ) - current_step = ImportStep.MODEL_DOCUMENT - await self._hint(base, current_step, source) + step = ImportStep.MODEL_DOCUMENT.value + await self._record_hint( + base, + Activities.record_import_status, + {'step': step, 'source': source or None}, + ) document = await workflow.execute_activity_method( Activities.write_import_model_document, { @@ -221,23 +221,23 @@ class ImportModel: succeeded = True result = { 'import_run_id': import_run_id, - 'run_id': run['run_id'], + 'run_id': prepared['run_id'], 'model_name': model_name, 'version': version['version'], 'document_id': document.get('id'), } except BaseException as error: # NOSONAR - classified once below, then re-raised untouched located_step, located_gate = failure_location(error) - failed_step = located_step or current_step.value + step = located_step or step gate = located_gate - await self._report_failure(base, error, step=failed_step, gate=gate) + await self._report_failure(base, error, step=step, gate=gate) raise finally: cleanup = await self._cleanup(base) await self._record_terminal( base, succeeded=succeeded, - step=failed_step or current_step.value, + step=step, gate=gate, source=source, cleanup_failed=not cleanup.get('cleaned', True), @@ -294,43 +294,23 @@ class ImportModel: # --------------------------------------------------------------- status writes - async def _hint( + async def _record_hint( self, base: dict[str, Any], - step: ImportStep, - source: dict[str, Any] | None = None, + activity: Callable[..., Awaitable[Any]], + extra: dict[str, Any], ) -> None: - """Record the step about to be attempted. A hint: its failure never fails the import.""" + """Record a status hint. A hint: its failure never fails the import.""" try: await workflow.execute_activity_method( - Activities.record_import_status, - {**base, 'step': step.value, 'source': source or None}, + activity, + {**base, **extra}, retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY, start_to_close_timeout=STATUS_WRITE_TIMEOUT, ) except BaseException as error: # NOSONAR - a lost hint must not fail a working import await self._report_status_write_failure(base, error) - async def _record_names( - self, base: dict[str, Any], source: dict[str, Any], run_name: str - ) -> None: - """Write the experiment and run names. Also a hint: the run is created either way.""" - try: - await workflow.execute_activity_method( - Activities.record_import_names, - { - **base, - 'experiment_name': source['experiment_name'], - 'run_name': run_name, - 'source': source, - 'step': ImportStep.EXPERIMENT_CREATION.value, - }, - retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY, - start_to_close_timeout=STATUS_WRITE_TIMEOUT, - ) - except BaseException as error: # NOSONAR - the names are for a human reading the list - await self._report_status_write_failure(base, error) - async def _record_terminal( self, base: dict[str, Any], diff --git a/tests/laborious/activities/test_activities.py b/tests/laborious/activities/test_activities.py index dd07c1f..44dc194 100644 --- a/tests/laborious/activities/test_activities.py +++ b/tests/laborious/activities/test_activities.py @@ -17,11 +17,13 @@ from laborious.activities.storage import Storage @patch('laborious.activities.activities.Gates.__init__') @patch('laborious.activities.activities.ModelMetrics.__init__') @patch('laborious.activities.activities.API.__init__') +@patch('laborious.activities.activities.MLFlowRepository') @patch('laborious.activities.activities.MinioRepository') @patch('laborious.activities.activities.MetricsController') def test___init__( mock_metrics_controller, mock_minio_repository, + mock_mlflow_repository, mock_api_init, mock_model_metrics_init, mock_gates_init, @@ -101,16 +103,22 @@ def test___init__( mock_mlflow_init.assert_called_once_with( ANY, - mlflow_host=mlflow_config['host'], - mlflow_port=mlflow_config['port'], - mlflow_username=mlflow_config['username'], - mlflow_password=mlflow_config['password'], + mlflow_repository=mock_mlflow_repository.return_value, minio_repository=mock_minio_repository.return_value, logger=logger, notification_handler=notification_handler, metrics_controller=mock_metrics_controller.return_value, ) + mock_mlflow_repository.assert_called_once_with( + f'{mlflow_config["host"]}:{mlflow_config["port"]}', + mlflow_config['username'], + mlflow_config['password'], + logger, + notification_handler, + mock_metrics_controller.return_value, + ) + mock_opc_init.assert_called_once_with( ANY, opc_servers=opc_config, @@ -198,11 +206,13 @@ IMPORT_STATUS_CONFIG = { @patch('laborious.activities.activities.ModelMetrics.__init__') @patch('laborious.activities.activities.API.__init__') @patch('laborious.activities.activities.MongoDBRepository') +@patch('laborious.activities.activities.MLFlowRepository') @patch('laborious.activities.activities.MinioRepository') @patch('laborious.activities.activities.MetricsController') -def test_the_import_activities_share_the_worker_s_mlflow_repository( # noqa: PLR0913 +def test_mlflow_and_import_activities_receive_the_same_mlflow_repository_instance( # noqa: PLR0913 _mock_metrics_controller, mock_minio_repository, + mock_mlflow_repository, mock_mongo_repository, _mock_api_init, _mock_model_metrics_init, @@ -214,11 +224,6 @@ def test_the_import_activities_share_the_worker_s_mlflow_repository( # noqa: PL ): """Setting the MLflow tracking URI is a process-wide side effect, so two repository instances would be two chances to disagree about where MLflow is.""" - repository = MagicMock(name='model_monitoring_repository') - mock_mlflow_init.side_effect = lambda instance, **_: setattr( - instance, 'model_monitoring_repository', repository - ) - Activities( **BASE_CONFIGS, logger=MagicMock(), @@ -228,6 +233,9 @@ def test_the_import_activities_share_the_worker_s_mlflow_repository( # noqa: PL import_status_config=IMPORT_STATUS_CONFIG, ) + repository = mock_mlflow_repository.return_value + assert mock_mlflow_init.call_args.kwargs['mlflow_repository'] is repository + passed = mock_model_import_init.call_args.kwargs assert passed['mlflow_repository'] is repository assert passed['minio_repository'] is mock_minio_repository.return_value @@ -244,11 +252,13 @@ def test_the_import_activities_share_the_worker_s_mlflow_repository( # noqa: PL @patch('laborious.activities.activities.ModelMetrics.__init__') @patch('laborious.activities.activities.API.__init__') @patch('laborious.activities.activities.MongoDBRepository') +@patch('laborious.activities.activities.MLFlowRepository') @patch('laborious.activities.activities.MinioRepository') @patch('laborious.activities.activities.MetricsController') def test_without_a_mongo_config_the_import_gets_no_repository( # noqa: PLR0913 _mock_metrics_controller, _mock_minio_repository, + _mock_mlflow_repository, mock_mongo_repository, _mock_api_init, _mock_model_metrics_init, @@ -271,6 +281,44 @@ def test_without_a_mongo_config_the_import_gets_no_repository( # noqa: PLR0913 mock_mongo_repository.assert_not_called() +@patch('laborious.activities.activities.ModelImport.__init__') +@patch('laborious.activities.activities.Storage.__init__') +@patch('laborious.activities.activities.MLFlow.__init__') +@patch('laborious.activities.activities.OPC.__init__') +@patch('laborious.activities.activities.Gates.__init__') +@patch('laborious.activities.activities.ModelMetrics.__init__') +@patch('laborious.activities.activities.API.__init__') +@patch('laborious.activities.activities.MongoDBRepository') +@patch('laborious.activities.activities.MLFlowRepository') +@patch('laborious.activities.activities.MinioRepository') +@patch('laborious.activities.activities.MetricsController') +def test_exactly_one_mlflow_repository_is_built_even_with_import_active( # noqa: PLR0913 + _mock_metrics_controller, + _mock_minio_repository, + mock_mlflow_repository, + _mock_mongo_repository, + _mock_api_init, + _mock_model_metrics_init, + _mock_gates_init, + _mock_opc_init, + _mock_mlflow_init, + _mock_storage_init, + _mock_model_import_init, +): + """No second `MLFlowRepository` is ever created on any path: `MLFlow` and `ModelImport` only + receive the one instance built here, they never build their own.""" + Activities( + **BASE_CONFIGS, + logger=MagicMock(), + notification_handler=MagicMock(), + mongo_config={'connection_string': 'mongodb://mongo', 'database_name': 'sientia'}, + import_config=IMPORT_CONFIG, + import_status_config=IMPORT_STATUS_CONFIG, + ) + + mock_mlflow_repository.assert_called_once() + + @mark.asyncio @patch('laborious.activities.activities.Storage') @patch('laborious.activities.activities.MLFlow') @@ -278,9 +326,11 @@ def test_without_a_mongo_config_the_import_gets_no_repository( # noqa: PLR0913 @patch('laborious.activities.activities.Gates') @patch('laborious.activities.activities.ModelMetrics') @patch('laborious.activities.activities.API') +@patch('laborious.activities.activities.MLFlowRepository') @patch('laborious.activities.activities.MinioRepository') async def test_shutdown( _mock_minio_repository, + _mock_mlflow_repository, mock_api_init, mock_model_metrics_init, mock_gates_init, diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index da17c94..e87c618 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -16,12 +16,12 @@ def _passthrough_from_dict(): yield -@patch('laborious.activities.mlflow.MLFlowRepository') @patch('laborious.activities.mlflow.MinioRepository') -def test___init__(mock_minio_repository, mock_mlflow_repository): +def test___init__(mock_minio_repository): logger = MagicMock() notification_handler = MagicMock() metrics_controller = AsyncMock() + mock_mlflow_repository = MagicMock() minio_repo = mock_minio_repository( endpoint='localhost:9000', access_key='minio', @@ -32,24 +32,14 @@ def test___init__(mock_minio_repository, mock_mlflow_repository): bucket='test', ) mlflow = MLFlow( - mlflow_host='http://localhost', - mlflow_port=5000, - mlflow_username='admin', - mlflow_password='admin', + mlflow_repository=mock_mlflow_repository, minio_repository=minio_repo, logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller, ) - assert mlflow.mlflow_host == 'http://localhost' - assert mlflow.mlflow_port == 5000 - assert mlflow.mlflow_username == 'admin' - assert mlflow.mlflow_password == 'admin' - - mock_mlflow_repository.assert_called_once_with( - 'http://localhost:5000', 'admin', 'admin', ANY, ANY, ANY - ) + assert mlflow.model_monitoring_repository is mock_mlflow_repository mock_minio_repository.assert_called_once_with( endpoint='localhost:9000', @@ -62,10 +52,18 @@ def test___init__(mock_minio_repository, mock_mlflow_repository): ) +def test___init___requires_mlflow_repository(): + """The failure moves to the signature, at construction time, instead of a `None` that only + surfaces silently at first use.""" + minio_repo = MagicMock() + logger = MagicMock() + with raises(TypeError): + MLFlow(minio_repository=minio_repo, logger=logger) + + @fixture -@patch('laborious.activities.mlflow.MLFlowRepository') @patch('laborious.activities.mlflow.MinioRepository') -def mlflow(mock_minio_repository, mock_mlflow_repository): +def mlflow(mock_minio_repository): logger = MagicMock() notification_handler = MagicMock() metrics_controller = AsyncMock() @@ -79,10 +77,7 @@ def mlflow(mock_minio_repository, mock_mlflow_repository): bucket='test', ) mlflow = MLFlow( - mlflow_host='http://localhost:5000', - mlflow_port=5000, - mlflow_username='admin', - mlflow_password='admin', + mlflow_repository=MagicMock(), minio_repository=minio_repo, logger=logger, notification_handler=notification_handler, diff --git a/tests/laborious/activities/test_model_import_bundle.py b/tests/laborious/activities/test_model_import_bundle.py index edf4018..ba9fd4d 100644 --- a/tests/laborious/activities/test_model_import_bundle.py +++ b/tests/laborious/activities/test_model_import_bundle.py @@ -21,11 +21,12 @@ from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch from pytest import fixture, mark, raises +from temporalio.exceptions import ApplicationError from laborious import metrics -from laborious.activities.model_import import ModelImport +from laborious.activities.model_import import ModelImport, derive_run_name from laborious.utils.bundle.steps import ImportStep -from laborious.utils.models.import_errors import ImportGateRejectedError +from laborious.utils.models.import_errors import ImportGateRejectedError, ImportMisconfiguredError from tests.helpers import bundle_factory as factory from tests.helpers.status_engine import FakeStatusEngine @@ -150,7 +151,7 @@ async def test_a_matching_digest_yields_the_downloaded_file( serve_object(minio_client, payload) digest = hashlib.sha256(payload).hexdigest() - result = await import_activity.download_import_bundle(download_input(digest)) + result = await import_activity._download_bundle(download_input(digest)) assert Path(result['path']).read_bytes() == payload assert result['digest'] == digest @@ -165,9 +166,7 @@ async def test_the_configured_bucket_is_used_when_the_request_names_none( payload = factory.make_bundle(password=BUNDLE_PASSWORD) serve_object(minio_client, payload) - await import_activity.download_import_bundle( - download_input(hashlib.sha256(payload).hexdigest()) - ) + await import_activity._download_bundle(download_input(hashlib.sha256(payload).hexdigest())) assert minio_client.stat_object.call_args.args[0] == 'sientia' @@ -179,7 +178,7 @@ async def test_the_request_may_name_another_bucket( payload = factory.make_bundle(password=BUNDLE_PASSWORD) serve_object(minio_client, payload) - await import_activity.download_import_bundle( + await import_activity._download_bundle( download_input(hashlib.sha256(payload).hexdigest(), bucket='other-bucket') ) @@ -197,7 +196,7 @@ async def test_a_digest_from_another_file_fails_before_the_header_is_parsed( request = download_input(hashlib.sha256(other).hexdigest()) with raises(ImportGateRejectedError) as failure: - await import_activity.download_import_bundle(request) + await import_activity._download_bundle(request) assert failure.value.details[0]['step'] == ImportStep.DOWNLOAD.value assert failure.value.details[0]['gate'] == 1 @@ -212,7 +211,7 @@ async def test_a_file_that_does_not_match_is_not_left_on_the_worker( request = download_input('0' * 64) with raises(ImportGateRejectedError): - await import_activity.download_import_bundle(request) + await import_activity._download_bundle(request) assert not (tmp_path / WORKFLOW_ID / 'bundle.sientia').exists() @@ -224,7 +223,7 @@ async def test_the_digest_comparison_is_case_insensitive_on_the_input( payload = factory.make_bundle(password=BUNDLE_PASSWORD) serve_object(minio_client, payload) - result = await import_activity.download_import_bundle( + result = await import_activity._download_bundle( download_input(hashlib.sha256(payload).hexdigest().upper()) ) @@ -239,7 +238,7 @@ async def test_an_object_over_the_ceiling_is_refused_before_its_body_is_fetched( request = download_input('a' * 64) with raises(ImportGateRejectedError) as failure: - await import_activity.download_import_bundle(request) + await import_activity._download_bundle(request) assert failure.value.details[0]['step'] == ImportStep.DOWNLOAD.value minio_client.fget_object.assert_not_called() @@ -256,7 +255,7 @@ async def test_a_key_of_the_wrong_shape_is_refused( request = download_input('a' * 64, object_key=object_key) with raises(ImportGateRejectedError): - await import_activity.download_import_bundle(request) + await import_activity._download_bundle(request) minio_client.fget_object.assert_not_called() @@ -269,7 +268,7 @@ async def test_an_unreadable_object_is_a_download_failure( request = download_input('a' * 64) with raises(ImportGateRejectedError) as failure: - await import_activity.download_import_bundle(request) + await import_activity._download_bundle(request) assert failure.value.details[0]['step'] == ImportStep.DOWNLOAD.value @@ -282,7 +281,7 @@ async def test_a_failed_transfer_is_a_download_failure( request = download_input('a' * 64) with raises(ImportGateRejectedError) as failure: - await import_activity.download_import_bundle(request) + await import_activity._download_bundle(request) assert failure.value.details[0]['step'] == ImportStep.DOWNLOAD.value @@ -307,8 +306,11 @@ async def test_a_missing_minio_repository_is_a_configuration_error(tmp_path: Pat request = download_input('a' * 64) - with raises(ValueError, match='Minio repository not initialized'): - await activity.download_import_bundle(request) + with raises(ImportMisconfiguredError, match='Minio repository not initialized') as failure: + await activity._download_bundle(request) + + assert failure.value.non_retryable is True + assert failure.value.type == 'ImportMisconfiguredError' # ----------------------------------------------------------------------- opening the bundle @@ -321,7 +323,7 @@ async def test_a_well_formed_bundle_opens_and_returns_nothing_secret( path = tmp_path / 'bundle.sientia' path.write_bytes(factory.make_bundle(password=BUNDLE_PASSWORD)) - result = await import_activity.open_import_bundle(open_input(str(path))) + result = await import_activity._open_bundle(open_input(str(path))) assert result['metadata']['model_name'] == 'factory_model' assert result['parameters']['target_variable'] == 'SE' @@ -336,7 +338,7 @@ async def test_opening_a_bundle_measures_the_kdf(import_activity: ModelImport, t path = tmp_path / 'bundle.sientia' path.write_bytes(factory.make_bundle(password=BUNDLE_PASSWORD)) - await import_activity.open_import_bundle(open_input(str(path))) + await import_activity._open_bundle(open_input(str(path))) observed = [ call.kwargs @@ -358,7 +360,7 @@ async def test_a_wrong_envelope_key_is_not_confused_with_a_wrong_bundle_password request = open_input(str(path), envelope) with raises(ImportGateRejectedError) as failure: - await import_activity.open_import_bundle(request) + await import_activity._open_bundle(request) details = failure.value.details[0] assert details['step'] == ImportStep.DECRYPTION.value @@ -377,7 +379,7 @@ async def test_a_wrong_bundle_password_is_a_decryption_failure_of_its_own( request = open_input(str(path)) with raises(ImportGateRejectedError) as failure: - await import_activity.open_import_bundle(request) + await import_activity._open_bundle(request) details = failure.value.details[0] assert details['step'] == ImportStep.DECRYPTION.value @@ -394,7 +396,7 @@ async def test_a_missing_password_key_stops_the_import_at_decryption( request = open_input(str(path)) with raises(ImportGateRejectedError) as failure: - await import_activity.open_import_bundle(request) + await import_activity._open_bundle(request) assert failure.value.details[0]['cause'] == 'envelope' @@ -413,7 +415,7 @@ async def test_a_tampered_or_malformed_envelope_is_refused( request = open_input(str(path), envelope) with raises(ImportGateRejectedError) as failure: - await import_activity.open_import_bundle(request) + await import_activity._open_bundle(request) assert failure.value.details[0]['cause'] == 'envelope' @@ -486,7 +488,7 @@ async def test_a_gate_failure_reaches_the_activity_as_its_own_step( request = open_input(str(path)) with raises(ImportGateRejectedError) as failure: - await import_activity.open_import_bundle(request) + await import_activity._open_bundle(request) assert failure.value.details[0]['step'] == step.value assert failure.value.details[0]['gate'] == gate @@ -501,7 +503,7 @@ async def test_a_rejected_bundle_leaves_no_extracted_tree( request = open_input(str(path)) with raises(ImportGateRejectedError): - await import_activity.open_import_bundle(request) + await import_activity._open_bundle(request) extract_root = tmp_path / WORKFLOW_ID / 'extract' assert not extract_root.exists() or list(extract_root.rglob('*')) == [] @@ -545,7 +547,7 @@ async def test_no_failure_path_says_the_password_out_loud( raised: BaseException | None = None result = None try: - result = await import_activity.open_import_bundle(open_input(str(path), envelope)) + result = await import_activity._open_bundle(open_input(str(path), envelope)) except BaseException as error: # noqa: BLE001 - the point is that *whatever* it is says nothing raised = error @@ -572,7 +574,136 @@ async def test_the_envelope_never_leaves_the_activity_that_opens_it( path.write_bytes(factory.make_bundle(password=BUNDLE_PASSWORD)) envelope = factory.build_envelope(BUNDLE_PASSWORD, ENVELOPE_KEY) - result = await import_activity.open_import_bundle(open_input(str(path), envelope)) + result = await import_activity._open_bundle(open_input(str(path), envelope)) assert envelope not in str(result) assert 'password' not in str(result).lower() + + +# ------------------------------------------------------- the fused activity (bug 8) + + +@mark.asyncio +async def test_prepare_and_upload_import_bundle_runs_all_four_phases_end_to_end( + import_activity: ModelImport, minio_client: MagicMock +): + """Download, open, experiment creation and artifact upload, one activity, one attempt.""" + payload = factory.make_bundle(password=BUNDLE_PASSWORD) + serve_object(minio_client, payload) + digest = hashlib.sha256(payload).hexdigest() + + mlflow_repository = import_activity.import_mlflow_repository + mlflow_repository.get_experiment.return_value = MagicMock(experiment_id='17') + client = mlflow_repository.client + client.create_run.return_value.info.run_id = 'imported-run-id' + + request = { + 'metadata': METADATA, + 'import_run_id': 77, + 'workflow_id': WORKFLOW_ID, + 'object_key': OBJECT_KEY, + 'expected_digest': digest, + 'password_envelope': factory.build_envelope(BUNDLE_PASSWORD, ENVELOPE_KEY), + } + + result = await import_activity.prepare_and_upload_import_bundle(request) + + expected_run_name = derive_run_name('factory_model', result['metadata']['model_version']) + assert result['metadata']['model_name'] == 'factory_model' + assert result['experiment_id'] == '17' + assert result['run_id'] == 'imported-run-id' + assert result['run_name'] == expected_run_name + mlflow_repository.get_experiment.assert_called_once_with( + 'factory_experiment', create_if_not_exists=True + ) + client.create_run.assert_called_once_with(experiment_id='17', run_name=expected_run_name) + client.log_artifacts.assert_called_once() + client.set_terminated.assert_called_once_with('imported-run-id', 'FINISHED') + + +@mark.asyncio +async def test_a_gate_failure_inside_the_fused_activity_still_names_its_step_and_gate( + import_activity: ModelImport, minio_client: MagicMock +): + """A gate rejection during the fused activity's decryption/extraction phase is unaffected.""" + payload = tampered() + serve_object(minio_client, payload) + digest = hashlib.sha256(payload).hexdigest() + + request = { + 'metadata': METADATA, + 'import_run_id': 77, + 'workflow_id': WORKFLOW_ID, + 'object_key': OBJECT_KEY, + 'expected_digest': digest, + 'password_envelope': factory.build_envelope(BUNDLE_PASSWORD, ENVELOPE_KEY), + } + + with raises(ImportGateRejectedError) as failure: + await import_activity.prepare_and_upload_import_bundle(request) + + assert failure.value.details[0]['step'] == ImportStep.DECRYPTION.value + assert failure.value.details[0]['gate'] == 3 + + +@mark.asyncio +async def test_a_retry_of_the_fused_activity_redoes_all_four_phases(import_activity: ModelImport): + """A retry re-runs download, open, experiment creation and upload from the start. + + Re-sending the same artifacts to the same run is safe (MLflow overwrites rather than + duplicates), so the cost of a retry is bandwidth and time, not corruption — but it is a real + cost, since nothing here is skipped on a second attempt. + """ + calls: list[str] = [] + + async def fake_download(_input_data: dict) -> dict: + calls.append('download') + return {'path': 'p', 'size': 1, 'digest': 'd'} + + async def fake_open(_input_data: dict) -> dict: + calls.append('open') + return { + 'extracted_dir': 'e', + 'metadata': { + 'model_name': 'factory_model', + 'model_version': '1', + 'experiment_name': 'factory_experiment', + }, + 'parameters': {}, + 'digest': 'd', + } + + async def fake_create_experiment(_input_data: dict) -> dict: + calls.append('experiment') + return {'experiment_id': '1', 'experiment_name': 'factory_experiment'} + + attempts = {'count': 0} + + async def fake_upload(_input_data: dict) -> dict: + calls.append('upload') + attempts['count'] += 1 + if attempts['count'] == 1: + raise RuntimeError('artifact upload failed on the first attempt') + return {'run_id': 'r', 'run_name': 'import-factory_model-v1'} + + with ( + patch.object(import_activity, '_download_bundle', side_effect=fake_download), + patch.object(import_activity, '_open_bundle', side_effect=fake_open), + patch.object(import_activity, '_create_experiment', side_effect=fake_create_experiment), + patch.object(import_activity, '_upload_artifacts', side_effect=fake_upload), + ): + with raises(ApplicationError, match='artifact upload failed on the first attempt'): + await import_activity.prepare_and_upload_import_bundle({'metadata': METADATA}) + + await import_activity.prepare_and_upload_import_bundle({'metadata': METADATA}) + + assert calls == [ + 'download', + 'open', + 'experiment', + 'upload', + 'download', + 'open', + 'experiment', + 'upload', + ] diff --git a/tests/laborious/activities/test_model_import_provisioning.py b/tests/laborious/activities/test_model_import_provisioning.py index fe5d11f..e025898 100644 --- a/tests/laborious/activities/test_model_import_provisioning.py +++ b/tests/laborious/activities/test_model_import_provisioning.py @@ -16,9 +16,13 @@ from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch import mlflow as mlflow_lib +from minio.commonconfig import ENABLED +from mlflow.exceptions import MlflowException +from mlflow.protos.databricks_pb2 import INTERNAL_ERROR, RESOURCE_DOES_NOT_EXIST from pytest import fixture, mark, raises from laborious.activities.model_import import ModelImport +from laborious.utils.models.import_errors import ImportMisconfiguredError from laborious.utils.repository.model_repository import MLFlowRepository from tests.helpers.status_engine import FakeStatusEngine @@ -74,9 +78,23 @@ def mongo() -> AsyncMock: repository = AsyncMock() repository.find_one.return_value = None repository.distinct.return_value = [] + # `_next_model_document_id` reaches Mongo through the repository's plain pymongo `database` + # handle, not through the repository's own async methods, with a *synchronous* call + # (`find_one_and_update` is never awaited). A sane default so tests that don't care about the + # exact id still get one; tests that do override this return value (or its `side_effect`) per + # collection access. + repository.database.__getitem__.return_value.find_one_and_update.return_value = { + '_id': 'models', + 'seq': 1, + } return repository +def counter(mongo: AsyncMock) -> MagicMock: + """The mock pymongo `Collection` behind `mongo_repository.database[collection]`.""" + return mongo.database.__getitem__.return_value.find_one_and_update + + @fixture def import_activity(tmp_path: Path, mlflow_client: MagicMock, mongo: AsyncMock) -> ModelImport: minio_repository = MagicMock() @@ -102,6 +120,7 @@ def import_activity(tmp_path: Path, mlflow_client: MagicMock, mongo: AsyncMock) 'max_uncompressed_bytes': 4096, 'max_compression_ratio': 200.0, 'models_collection': 'models', + 'counters_collection': 'model_id_counters', 'retention_days': 7, }, minio_repository=minio_repository, @@ -133,9 +152,7 @@ async def test_the_experiment_keeps_the_origin_s_name(import_activity: ModelImpo repository = import_activity.import_mlflow_repository repository.get_experiment.return_value = MagicMock(experiment_id='17') - result = await import_activity.create_import_experiment( - base(experiment_name='factory_experiment') - ) + result = await import_activity._create_experiment(base(experiment_name='factory_experiment')) repository.get_experiment.assert_called_once_with( 'factory_experiment', create_if_not_exists=True @@ -148,8 +165,8 @@ async def test_a_missing_mlflow_repository_is_a_configuration_error(import_activ import_activity.import_mlflow_repository = None payload = base(experiment_name='e') - with raises(ValueError, match='MLflow repository not initialized'): - await import_activity.create_import_experiment(payload) + with raises(ImportMisconfiguredError, match='MLflow repository not initialized'): + await import_activity._create_experiment(payload) # ------------------------------------------------------------------- the run and its tree @@ -173,8 +190,8 @@ async def test_upload_artifacts_requires_the_mlflow_repository( import_activity.import_mlflow_repository = None payload = upload_input(tmp_path / 'extract') - with raises(ValueError, match='MLflow repository not initialized'): - await import_activity.upload_import_artifacts(payload) + with raises(ImportMisconfiguredError, match='MLflow repository not initialized'): + await import_activity._upload_artifacts(payload) @mark.asyncio @@ -182,7 +199,7 @@ async def test_every_import_creates_a_new_run_with_the_recorded_name( import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path ): """The name is passed in, never recomputed, so the record and the run cannot diverge.""" - result = await import_activity.upload_import_artifacts(upload_input(tmp_path / 'extract')) + result = await import_activity._upload_artifacts(upload_input(tmp_path / 'extract')) mlflow_client.create_run.assert_called_once_with( experiment_id='17', run_name='import-factory_model-v2' @@ -196,21 +213,30 @@ async def test_the_artifact_tree_is_logged_from_the_bundle_s_artifacts_directory ): extracted = tmp_path / 'extract' - await import_activity.upload_import_artifacts(upload_input(extracted)) + await import_activity._upload_artifacts(upload_input(extracted)) mlflow_client.log_artifacts.assert_called_once_with(RUN_ID, str(extracted / 'artifacts')) +def _logged_params(mlflow_client: MagicMock) -> dict: + """Flatten every `client.log_batch(run_id, params=[...])` call into one key/value map.""" + return { + param.key: param.value + for call in mlflow_client.log_batch.call_args_list + for param in call.kwargs['params'] + } + + @mark.asyncio async def test_the_bundle_s_parameters_are_logged_verbatim( import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path ): - await import_activity.upload_import_artifacts(upload_input(tmp_path / 'extract')) + await import_activity._upload_artifacts(upload_input(tmp_path / 'extract')) logged = { - call.args[1]: call.args[2] - for call in mlflow_client.log_param.call_args_list - if not call.args[1].startswith('source.') + key: value + for key, value in _logged_params(mlflow_client).items() + if not key.startswith('source.') } assert logged == BUNDLE_PARAMETERS assert logged['target_variable'] == 'SE' @@ -220,12 +246,12 @@ async def test_the_bundle_s_parameters_are_logged_verbatim( async def test_the_origin_is_logged_as_source_parameters( import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path ): - await import_activity.upload_import_artifacts(upload_input(tmp_path / 'extract')) + await import_activity._upload_artifacts(upload_input(tmp_path / 'extract')) logged = { - call.args[1]: call.args[2] - for call in mlflow_client.log_param.call_args_list - if call.args[1].startswith('source.') + key: value + for key, value in _logged_params(mlflow_client).items() + if key.startswith('source.') } assert logged == {f'source.{key}': value for key, value in BUNDLE_METADATA.items()} # A record of where the model came from, not a link: the origin run id is never dereferenced. @@ -234,11 +260,45 @@ async def test_the_origin_is_logged_as_source_parameters( mlflow_client.download_artifacts.assert_not_called() +@mark.asyncio +async def test_parameters_are_logged_in_batches_of_at_most_100( + import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path +): + """More params than `log_batch`'s per-call limit must span multiple calls, each within the + limit, and every key must still be logged exactly once across all of them.""" + many_parameters = {f'param_{i}': str(i) for i in range(150)} + payload = upload_input(tmp_path / 'extract') + payload['parameters'] = many_parameters + + await import_activity._upload_artifacts(payload) + + calls = mlflow_client.log_batch.call_args_list + assert len(calls) > 1 + for call in calls: + assert len(call.kwargs['params']) <= 100 + + logged = _logged_params(mlflow_client) + expected = { + **many_parameters, + **{f'source.{key}': value for key, value in BUNDLE_METADATA.items()}, + } + assert logged == expected + + +@mark.asyncio +async def test_upload_artifacts_never_uses_log_param( + import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path +): + await import_activity._upload_artifacts(upload_input(tmp_path / 'extract')) + + mlflow_client.log_param.assert_not_called() + + @mark.asyncio async def test_the_run_is_closed_when_its_tree_is_in( import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path ): - await import_activity.upload_import_artifacts(upload_input(tmp_path / 'extract')) + await import_activity._upload_artifacts(upload_input(tmp_path / 'extract')) mlflow_client.set_terminated.assert_called_once_with(RUN_ID, 'FINISHED') @@ -251,7 +311,7 @@ async def test_register_version_requires_the_mlflow_repository(import_activity: import_activity.import_mlflow_repository = None payload = base(model_name='factory_model', run_id=RUN_ID) - with raises(ValueError, match='MLflow repository not initialized'): + with raises(ImportMisconfiguredError, match='MLflow repository not initialized'): await import_activity.register_import_model_version(payload) @@ -283,11 +343,19 @@ async def test_a_literal_runs_uri_would_not_be_parseable_by_the_legacy_loader(): async def test_a_registered_model_is_created_only_when_it_is_absent( import_activity: ModelImport, mlflow_client: MagicMock ): - mlflow_client.get_registered_model.side_effect = RuntimeError('not found') + """The only exception `get_registered_model` may treat as "does not exist yet": an + `MlflowException` whose `error_code` is literally `RESOURCE_DOES_NOT_EXIST`.""" + mlflow_client.get_registered_model.side_effect = MlflowException( + 'not found', error_code=RESOURCE_DOES_NOT_EXIST + ) - await import_activity.register_import_model_version(base(model_name='new_model', run_id=RUN_ID)) + result = await import_activity.register_import_model_version( + base(model_name='new_model', run_id=RUN_ID) + ) mlflow_client.create_registered_model.assert_called_once_with('new_model') + mlflow_client.create_model_version.assert_called_once() + assert result['model_name'] == 'new_model' @mark.asyncio @@ -303,13 +371,46 @@ async def test_a_second_import_reuses_the_registered_model( mlflow_client.create_registered_model.assert_not_called() +@mark.asyncio +async def test_a_different_mlflow_error_is_not_treated_as_absent( + import_activity: ModelImport, mlflow_client: MagicMock +): + """Only `RESOURCE_DOES_NOT_EXIST` means "create it". Any other MLflow error code — a transient + failure, an internal error — must not be swallowed and must not trigger a spurious create.""" + error = MlflowException('backend is unavailable', error_code=INTERNAL_ERROR) + mlflow_client.get_registered_model.side_effect = error + payload = base(model_name='factory_model', run_id=RUN_ID) + + with raises(MlflowException, match='backend is unavailable') as excinfo: + await import_activity.register_import_model_version(payload) + + assert excinfo.value is error + mlflow_client.create_registered_model.assert_not_called() + mlflow_client.create_model_version.assert_not_called() + + +@mark.asyncio +async def test_a_non_mlflow_exception_is_never_treated_as_absent( + import_activity: ModelImport, mlflow_client: MagicMock +): + """A bare `ConnectionError` (or any non-`MlflowException`) is not this function's business to + interpret at all: it must propagate unmodified, exactly as any other unexpected error would.""" + mlflow_client.get_registered_model.side_effect = ConnectionError('connection refused') + payload = base(model_name='factory_model', run_id=RUN_ID) + + with raises(ConnectionError, match='connection refused'): + await import_activity.register_import_model_version(payload) + + mlflow_client.create_registered_model.assert_not_called() + + @mark.asyncio async def test_the_import_never_promotes_anything( import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path ): """A `COMPLETED` import is a registered model, not a servable one.""" - await import_activity.create_import_experiment(base(experiment_name='factory_experiment')) - await import_activity.upload_import_artifacts(upload_input(tmp_path / 'extract')) + await import_activity._create_experiment(base(experiment_name='factory_experiment')) + await import_activity._upload_artifacts(upload_input(tmp_path / 'extract')) await import_activity.register_import_model_version( base(model_name='factory_model', run_id=RUN_ID) ) @@ -322,7 +423,7 @@ async def test_the_import_never_promotes_anything( async def test_no_provisioning_call_passes_a_stage( import_activity: ModelImport, mlflow_client: MagicMock, tmp_path: Path ): - await import_activity.upload_import_artifacts(upload_input(tmp_path / 'extract')) + await import_activity._upload_artifacts(upload_input(tmp_path / 'extract')) result = await import_activity.register_import_model_version( base(model_name='factory_model', run_id=RUN_ID) ) @@ -391,10 +492,12 @@ def test_the_legacy_loader_cannot_read_a_logged_model_uri(): @mark.asyncio -async def test_the_document_id_continues_the_collection_across_gaps( +async def test_the_document_id_is_whatever_the_atomic_counter_returns( import_activity: ModelImport, mongo: AsyncMock ): - mongo.distinct.return_value = [str(value) for value in [*range(1, 19), *range(20, 25), 30]] + """`id` is no longer computed here at all: it is `str(seq)` off the counter document + `find_one_and_update` hands back, whatever gaps or history the collection has.""" + counter(mongo).return_value = {'_id': 'models', 'seq': 31} result = await import_activity.write_import_model_document( base(model_name='factory_model', target='SE') @@ -404,20 +507,26 @@ async def test_the_document_id_continues_the_collection_across_gaps( @mark.asyncio -async def test_the_document_id_is_numeric_not_lexicographic( +async def test_the_document_id_is_the_counter_s_seq_verbatim_not_reparsed( import_activity: ModelImport, mongo: AsyncMock ): - mongo.distinct.return_value = ['9', '30'] + """A value large enough that the old max-scan's `"9" > "30"` lexicographic bug would have + mangled it, demonstrating the new id is never routed through any string comparison at all.""" + counter(mongo).return_value = {'_id': 'models', 'seq': 100} result = await import_activity.write_import_model_document( base(model_name='factory_model', target='SE') ) - assert result['id'] == '31' + assert result['id'] == '100' @mark.asyncio async def test_an_empty_collection_starts_at_one(import_activity: ModelImport, mongo: AsyncMock): + """`$inc` on a counter document that does not exist yet starts from zero, so the first import + against a fresh collection gets id `1`.""" + counter(mongo).return_value = {'_id': 'models', 'seq': 1} + result = await import_activity.write_import_model_document( base(model_name='factory_model', target='SE') ) @@ -429,7 +538,7 @@ async def test_an_empty_collection_starts_at_one(import_activity: ModelImport, m async def test_the_document_is_exactly_the_contracted_shape( import_activity: ModelImport, mongo: AsyncMock ): - mongo.distinct.return_value = ['7'] + counter(mongo).return_value = {'_id': 'models', 'seq': 8} await import_activity.write_import_model_document(base(model_name='factory_model', target='SE')) @@ -445,10 +554,28 @@ async def test_the_document_is_exactly_the_contracted_shape( 'predict_flavor': 'joblib', 'retention_minutes': 60, 'target': 'SE', + 'skip_predict_timestamp': True, }, } +@mark.asyncio +@mark.parametrize( + 'model_name, target', + [('factory_model', 'SE'), ('another_model', 'temperature')], +) +async def test_the_document_always_carries_skip_predict_timestamp_true( + import_activity: ModelImport, mongo: AsyncMock, model_name: str, target: str +): + """The opt-out is decided by the importer at write time, never left for an operator to set.""" + counter(mongo).return_value = {'_id': 'models', 'seq': 8} + + await import_activity.write_import_model_document(base(model_name=model_name, target=target)) + + _, document, _ = mongo.insert.await_args.args + assert document['model_config']['skip_predict_timestamp'] is True + + @mark.asyncio async def test_the_document_carries_no_field_the_runtime_does_not_read( import_activity: ModelImport, mongo: AsyncMock @@ -474,23 +601,31 @@ async def test_a_name_already_in_the_collection_is_left_exactly_as_it_is( assert result == {'written': False, 'skipped': True, 'id': '12'} mongo.insert.assert_not_awaited() - mongo.distinct.assert_not_awaited() + # No counter increment is spent on a name that was already there: a duplicate import must + # never move the sequence forward for a document it isn't going to write. + counter(mongo).assert_not_called() @mark.asyncio -async def test_unusable_ids_are_ignored_and_counted(import_activity: ModelImport, mongo: AsyncMock): - mongo.distinct.return_value = ['1', 'legacy-model', None, '3'] +async def test_two_imports_in_a_row_never_reuse_an_id( + import_activity: ModelImport, mongo: AsyncMock +): + """No dependency on `distinct` at all: two sequential calls simply relay whatever an atomic + (fake, here) counter hands back, strictly increasing, with no scan of existing documents.""" + seq = {'value': 40} - result = await import_activity.write_import_model_document( - base(model_name='factory_model', target='SE') - ) + def increment(*args, **kwargs): # noqa: ARG001 - mirrors pymongo's call signature + seq['value'] += 1 + return {'_id': 'models', 'seq': seq['value']} - assert result['id'] == '4' - assert result['ignored_ids'] == 2 - assert any( - 'ignored 2 non-numeric' in str(call) - for call in import_activity.logger.custom_info.call_args_list - ) + counter(mongo).side_effect = increment + + first_id, _ = await import_activity._next_model_document_id('models', METADATA) + second_id, _ = await import_activity._next_model_document_id('models', METADATA) + + assert (first_id, second_id) == ('41', '42') + assert int(second_id) > int(first_id) + mongo.distinct.assert_not_called() @mark.asyncio @@ -498,14 +633,14 @@ async def test_a_duplicate_id_fails_the_step_without_recomputing_it( import_activity: ModelImport, mongo: AsyncMock ): """No retry and no second guess: the unique index says the collection moved under us.""" - mongo.distinct.return_value = ['1'] + counter(mongo).return_value = {'_id': 'models', 'seq': 2} mongo.insert.side_effect = RuntimeError('E11000 duplicate key error: id_1') payload = base(model_name='factory_model', target='SE') with raises(RuntimeError, match='duplicate key'): await import_activity.write_import_model_document(payload) - assert mongo.distinct.await_count == 1 + assert counter(mongo).call_count == 1 assert mongo.insert.await_count == 1 @@ -513,11 +648,18 @@ async def test_a_duplicate_id_fails_the_step_without_recomputing_it( async def test_the_import_only_reads_and_inserts_documents( import_activity: ModelImport, mongo: AsyncMock ): - """No update, no replace, no upsert: an import cannot change a model already on the platform.""" + """No update, no replace, no upsert on the model document itself: an import cannot change a + model already on the platform. The atomic counter's own `$inc` against a dedicated counters + collection is a different concern and is explicitly allowed.""" await import_activity.write_import_model_document(base(model_name='factory_model', target='SE')) used = {name for name, _, _ in mongo.mock_calls} - assert used <= {'find_one', 'distinct', 'insert'} + assert used <= { + 'find_one', + 'insert', + 'database.__getitem__', + 'database.__getitem__().find_one_and_update', + } @mark.asyncio @@ -527,29 +669,27 @@ async def test_a_missing_mongo_repository_is_a_configuration_error( import_activity.mongo_repository = None payload = base(model_name='m', target='SE') - with raises(ValueError, match='MongoDB repository not initialized'): + with raises(ImportMisconfiguredError, match='MongoDB repository not initialized'): await import_activity.write_import_model_document(payload) -@mark.asyncio -async def test_next_model_document_id_also_guards_a_missing_mongo_repository( - import_activity: ModelImport, -): - """The helper repeats the guard so it stays safe if ever called from somewhere new.""" - import_activity.mongo_repository = None - - with raises(ValueError, match='MongoDB repository not initialized'): - await import_activity._next_model_document_id('models', {}) - - def test_no_import_module_writes_an_active_document(): - """Asserted from the source too, so `active: false` cannot be flipped by a later edit.""" + """Asserted from the source too, so `active: false` cannot be flipped by a later edit. + + `upsert` is checked against `write_import_model_document`'s own body only, not the whole file: + `_next_model_document_id`'s atomic counter legitimately upserts a *counter* document in a + dedicated collection — a different concern from ever updating, replacing or upserting the model + document itself, which this test still forbids absolutely. + """ source = Path('laborious/activities/model_import.py').read_text() + start = source.index('async def write_import_model_document') + end = source.index('async def _next_model_document_id') + document_write = source[start:end] assert "'active': False" in source assert "'active': True" not in source for forbidden in ('update_one', 'replace_one', 'update_many', 'upsert'): - assert forbidden not in source + assert forbidden not in document_write # ------------------------------------------------------------------ cleanup and retention @@ -613,7 +753,7 @@ async def test_ensure_retention_requires_the_minio_repository(import_activity: M import_activity.import_minio_repository = None payload = base() - with raises(ValueError, match='Minio repository not initialized'): + with raises(ImportMisconfiguredError, match='Minio repository not initialized'): await import_activity.ensure_import_bundle_retention(payload) @@ -688,6 +828,58 @@ async def test_the_retention_rule_can_be_applied_to_a_requested_bucket( assert client.set_bucket_lifecycle.call_args.args[0] == 'other-bucket' +@mark.asyncio +async def test_a_correct_existing_rule_is_not_rewritten(import_activity: ModelImport): + client = import_activity.import_minio_repository.minio_client + existing = MagicMock() + matching_rule = MagicMock(rule_id='sientia-import-bundle-expiry', status=ENABLED) + matching_rule.expiration.days = 7 + matching_rule.rule_filter.prefix = 'imported_models/' + existing.rules = [matching_rule] + client.get_bucket_lifecycle.return_value = existing + + result = await import_activity.ensure_import_bundle_retention(base()) + + client.set_bucket_lifecycle.assert_not_called() + assert result == {'bucket': 'sientia', 'prefix': 'imported_models/', 'days': 7, 'rules': 1} + + +@mark.asyncio +async def test_a_drifted_existing_rule_is_rewritten(import_activity: ModelImport): + client = import_activity.import_minio_repository.minio_client + existing = MagicMock() + drifted_rule = MagicMock(rule_id='sientia-import-bundle-expiry', status=ENABLED) + drifted_rule.expiration.days = 3 + drifted_rule.rule_filter.prefix = 'imported_models/' + existing.rules = [drifted_rule] + client.get_bucket_lifecycle.return_value = existing + + await import_activity.ensure_import_bundle_retention(base()) + + config = client.set_bucket_lifecycle.call_args.args[1] + corrected = next( + rule for rule in config.rules if rule.rule_id == 'sientia-import-bundle-expiry' + ) + assert corrected.expiration.days == 7 + + +@mark.asyncio +async def test_an_unrelated_rule_survives_a_skipped_rewrite(import_activity: ModelImport): + client = import_activity.import_minio_repository.minio_client + existing = MagicMock() + matching_rule = MagicMock(rule_id='sientia-import-bundle-expiry', status=ENABLED) + matching_rule.expiration.days = 7 + matching_rule.rule_filter.prefix = 'imported_models/' + unrelated_rule = MagicMock(rule_id='predictions-offload-expiry') + existing.rules = [unrelated_rule, matching_rule] + client.get_bucket_lifecycle.return_value = existing + + result = await import_activity.ensure_import_bundle_retention(base()) + + client.set_bucket_lifecycle.assert_not_called() + assert result['rules'] == 2 + + def test_the_model_document_shape_is_documented_in_one_place(): """A guard against the shape drifting between the code and what the runtime reads.""" source = Path('laborious/activities/model_import.py').read_text() diff --git a/tests/laborious/activities/test_model_import_status.py b/tests/laborious/activities/test_model_import_status.py index 2f24080..0233406 100644 --- a/tests/laborious/activities/test_model_import_status.py +++ b/tests/laborious/activities/test_model_import_status.py @@ -21,6 +21,10 @@ from sqlalchemy.exc import DataError, IntegrityError, OperationalError, Programm from laborious import metrics from laborious.activities.model_import import ( + _CLAIM_SQL, # noqa: PLC2701 - the SQL-shape test below asserts against these directly + _NAMES_SQL, + _PROGRESS_SQL, + _TERMINAL_SQL, IMPORT_LOG_TABLE, IMPORT_STATUS_HINT_RETRY_POLICY, IMPORT_STATUS_RETRY_POLICY, @@ -28,14 +32,18 @@ from laborious.activities.model_import import ( ExperimentRunStatus, ImportState, ModelImport, + _build_update, build_import_detail, derive_run_name, map_state_to_status, ) from laborious.utils.bundle.steps import ImportStep from laborious.utils.models.import_errors import ( + IMPORT_FAILURE_CATALOG, + UNCLASSIFIED_FAILURE, ImportErrorCode, ImportLogRowNotClaimableError, + ImportResponsibility, ImportStatusRejectedError, ) from tests.helpers.status_engine import FakeStatusEngine @@ -93,6 +101,67 @@ EXPECTED_ERROR_STATUS = { } +# The four statements as they were written out literally before `_build_update` existed. The test +# below proves the helper reproduces each one byte-for-byte (modulo whitespace). +_ORIGINAL_CLAIM_SQL = """ + UPDATE public.experiment_run + SET status = :status, + updated_at = :updated_at, + orchestrator_response_data = CAST(:detail AS JSONB) + WHERE id = :import_run_id + AND run_type = 'IMPORT' + AND (status = 'PENDING' + OR (status = 'ORCHESTRATOR_WAITING_PROC' + AND orchestrator_response_data -> 'import' ->> 'workflow_id' = :workflow_id)) + """ + +_ORIGINAL_PROGRESS_SQL = """ + UPDATE public.experiment_run + SET status = :status, + updated_at = :updated_at, + orchestrator_response_data = CAST(:detail AS JSONB) + WHERE id = :import_run_id + """ + +_ORIGINAL_NAMES_SQL = """ + UPDATE public.experiment_run + SET experiment_name = :experiment_name, + run_name = :run_name, + updated_at = :updated_at, + orchestrator_response_data = CAST(:detail AS JSONB) + WHERE id = :import_run_id + """ + +_ORIGINAL_TERMINAL_SQL = """ + UPDATE public.experiment_run + SET status = :status, + error_message = :error_message, + updated_at = :updated_at, + orchestrator_response_data = CAST(:detail AS JSONB) + WHERE id = :import_run_id + """ + + +def _normalized(sql: object) -> str: + return ' '.join(str(sql).split()) + + +def test_build_update_reproduces_each_original_statement_byte_for_byte(): + assert _normalized(_CLAIM_SQL) == _normalized(_ORIGINAL_CLAIM_SQL) + assert _normalized(_PROGRESS_SQL) == _normalized(_ORIGINAL_PROGRESS_SQL) + assert _normalized(_NAMES_SQL) == _normalized(_ORIGINAL_NAMES_SQL) + assert _normalized(_TERMINAL_SQL) == _normalized(_ORIGINAL_TERMINAL_SQL) + + +def test_build_update_with_no_casts_or_extra_where_renders_a_bare_update(): + statement = _build_update({'status': 'status'}) + + assert ( + _normalized(statement) + == 'UPDATE public.experiment_run SET status = :status WHERE id = :import_run_id' + ) + + def build_activity(engine: FakeStatusEngine) -> ModelImport: """A `ModelImport` whose only connection is the recording engine double.""" with patch('laborious.activities.model_import.create_engine', return_value=engine): @@ -172,6 +241,27 @@ def test_an_unclassified_error_is_a_platform_error(): assert map_state_to_status(ImportState.ERROR) == ExperimentRunStatus.IMPORT_ERROR +@mark.parametrize('step', list(IMPORT_FAILURE_CATALOG)) +def test_responsibility_comes_from_the_catalog_alone(step: ImportStep): + """The regression test for `specs/model-import-status-record/spec.md`'s single-source rule.""" + assert map_state_to_status(ImportState.ERROR, step) == EXPECTED_ERROR_STATUS[step] + + +def test_adding_a_step_to_the_catalog_is_enough_to_resolve_its_status(): + """A fake catalog entry needs nothing but its own `responsibility` to resolve correctly.""" + fake_step = ImportStep.RECEIVED + for responsibility, expected in ( + (ImportResponsibility.USER, ExperimentRunStatus.ORCHESTRATOR_VALIDATION_ERROR), + (ImportResponsibility.SUPPORT, ExperimentRunStatus.IMPORT_ERROR), + ): + fake_catalog = { + **IMPORT_FAILURE_CATALOG, + fake_step: (ImportErrorCode.UNEXPECTED_ERROR, 'fake reason', responsibility), + } + with patch('laborious.utils.models.import_errors.IMPORT_FAILURE_CATALOG', fake_catalog): + assert map_state_to_status(ImportState.ERROR, fake_step) == expected + + # ------------------------------------------------------------------------- the run name @@ -583,6 +673,29 @@ async def test_a_failure_is_recorded_with_its_step_code_and_reason( assert detail['gate'] == 4 +@mark.asyncio +async def test_a_failure_with_an_unrecognized_step_is_recorded_unclassified( + import_activity: ModelImport, engine: FakeStatusEngine +): + """A step string that is not an `ImportStep` member must not crash the one authoritative write. + + It runs in the workflow's `finally`, after cleanup, so raising here would leave the row without + a verdict. The unrecognized step falls back to the unclassified failure instead. + """ + verdict = await import_activity.record_import_terminal_status( + {**claim_input(), 'succeeded': False, 'step': 'no_longer_a_step'} + ) + + write = engine.writes[0] + assert write.parameters['status'] == ExperimentRunStatus.IMPORT_ERROR.value + assert write.parameters['error_message'] == UNCLASSIFIED_FAILURE[1] + detail = detail_of(write) + assert detail['code'] == ImportErrorCode.UNEXPECTED_ERROR.value + assert verdict['status'] == ExperimentRunStatus.IMPORT_ERROR.value + assert verdict['code'] == ImportErrorCode.UNEXPECTED_ERROR.value + assert verdict['reason'] == UNCLASSIFIED_FAILURE[1] + + @mark.asyncio async def test_a_cleanup_failure_does_not_turn_a_success_into_an_error( import_activity: ModelImport, engine: FakeStatusEngine diff --git a/tests/laborious/test_import_absence.py b/tests/laborious/test_import_absence.py index 686d9c5..b0eab28 100644 --- a/tests/laborious/test_import_absence.py +++ b/tests/laborious/test_import_absence.py @@ -9,6 +9,7 @@ behaviour is the point of not existing. """ import ast +import inspect from pathlib import Path from pytest import mark @@ -182,15 +183,24 @@ def test_the_plaintext_password_field_is_refused_by_name(): def _error_message_values(tree: ast.Module) -> list[ast.expr]: - """Every expression written into the record's `error_message`.""" + """Every expression written into the record's `error_message`, at an actual status write. + + Scoped to `_execute_status_write`'s own parameter dict — not every dict in the module — because + `_build_update`'s `columns` argument also carries an `'error_message'` key (naming the column and + its bind parameter, not a value being written), and that is not a write this test is about. + """ values = [] for node in ast.walk(tree): - if isinstance(node, ast.Dict): - for key, value in zip(node.keys, node.values, strict=True): + if not ( + isinstance(node, ast.Call) and ast.unparse(node.func).endswith('_execute_status_write') + ): + continue + for arg in node.args: + if not isinstance(arg, ast.Dict): + continue + for key, value in zip(arg.keys, arg.values, strict=True): if isinstance(key, ast.Constant) and key.value == 'error_message': values.append(value) - if isinstance(node, ast.keyword) and node.arg == 'error_message': - values.append(node.value) return values @@ -211,10 +221,12 @@ def test_the_reason_written_is_the_classified_one(): def test_the_classification_ignores_the_exception_by_construction(): - """`classify_import_failure` deletes its exception argument; nothing can leak from it.""" - source = ERRORS_MODULE.read_text() + """`classify_import_failure` has no exception parameter at all; nothing can leak from it.""" + from laborious.utils.models.import_errors import classify_import_failure - assert 'del exception' in source + parameters = inspect.signature(classify_import_failure).parameters + assert 'exception' not in parameters + assert list(parameters) == ['step'] @mark.parametrize('module_path', (WORKFLOW_MODULE, ACTIVITIES_MODULE), ids=lambda p: p.name) diff --git a/tests/laborious/utils/bundle/test_reader.py b/tests/laborious/utils/bundle/test_reader.py index ebff1bf..e912ad5 100644 --- a/tests/laborious/utils/bundle/test_reader.py +++ b/tests/laborious/utils/bundle/test_reader.py @@ -367,6 +367,25 @@ def test_gate_4_rejects_an_entry_that_is_neither_a_file_nor_a_directory(work_dir assert error.value.gate == 4 +def test_gate_4_and_gate_5_read_the_central_directory_once( + golden_bundle, golden_password, work_dir, reader, monkeypatch +): + """Achado #11: gate 5 extracts using the entries gate 4 already read, instead of reopening the + archive to call `infolist()` a second time.""" + original_infolist = zipfile.ZipFile.infolist + calls: list[None] = [] + + def _counting_infolist(self): + calls.append(None) + return original_infolist(self) + + monkeypatch.setattr(zipfile.ZipFile, 'infolist', _counting_infolist) + + reader.open(golden_bundle, golden_password, work_dir / 'extract', work_dir) + + assert len(calls) == 1 + + # ------------------------------------------------------------------- gate 5 @@ -580,6 +599,21 @@ def test_gate_7_rejects_a_non_regular_file_left_in_the_tree(work_dir, reader): assert error.value.gate == 7 +def test_gate_7_rejects_disallowed_content_regardless_of_traversal_order(work_dir, reader): + """Achado #12: gate 7 no longer sorts the tree before walking it, so a disallowed file that + would not be first or last alphabetically must still be caught.""" + root = work_dir / 'unsorted_root' + root.mkdir() + for name in ('a_file.txt', 'b_file.txt', 'y_file.txt', 'z_file.txt'): + (root / name).write_bytes(b'ok') + (root / 'n_disallowed.sh').write_bytes(b'#!/bin/sh\n') + + with pytest.raises(BundleGateError, match='should not carry') as error: + reader._gate_7_content_policy(root) + + assert error.value.gate == 7 + + # ------------------------------------------------------------------ ordering diff --git a/tests/laborious/utils/models/test_import_errors.py b/tests/laborious/utils/models/test_import_errors.py index 53472fa..173547f 100644 --- a/tests/laborious/utils/models/test_import_errors.py +++ b/tests/laborious/utils/models/test_import_errors.py @@ -34,6 +34,7 @@ from laborious.utils.models.import_errors import ( failure_location, report_import_failure, summarize_failure, + walk_cause_chain, ) REPOSITORY_ROOT = Path(__file__).resolve().parents[4] @@ -137,12 +138,12 @@ def test_every_gate_maps_to_a_step_that_has_a_sentence(): def test_each_step_has_its_own_code(): - codes = [code for code, _ in IMPORT_FAILURE_CATALOG.values()] + codes = [code for code, _, _ in IMPORT_FAILURE_CATALOG.values()] assert len(codes) == len(set(codes)) def test_the_codes_that_never_reach_a_record_are_absent_from_the_catalog(): - catalogued = {code for code, _ in IMPORT_FAILURE_CATALOG.values()} + catalogued = {code for code, _, _ in IMPORT_FAILURE_CATALOG.values()} for code in ( ImportErrorCode.LOG_ROW_NOT_CLAIMABLE, ImportErrorCode.STATUS_NOT_RECORDED, @@ -153,7 +154,7 @@ def test_the_codes_that_never_reach_a_record_are_absent_from_the_catalog(): @mark.parametrize('step', list(IMPORT_FAILURE_CATALOG)) def test_a_reason_never_carries_the_exception(step: ImportStep): - failure = classify_import_failure(step, RuntimeError(LEAKY_MESSAGE)) + failure = classify_import_failure(step) for leak in LEAKS_THAT_MUST_NOT_SURFACE: assert leak not in failure.reason @@ -162,12 +163,11 @@ def test_a_reason_never_carries_the_exception(step: ImportStep): @mark.parametrize('step', list(IMPORT_FAILURE_CATALOG)) def test_the_classification_is_a_function_of_the_step_alone(step: ImportStep): - """Which is what makes a reworded sentence keep its code: neither is read from the exception.""" - first = classify_import_failure(step, RuntimeError(LEAKY_MESSAGE)) - second = classify_import_failure(step, KeyError('target_variable')) - third = classify_import_failure(step, None) + """Classification does not read the exception at all — it was already dropped as a parameter.""" + first = classify_import_failure(step) + second = classify_import_failure(step) - assert first == second == third + assert first == second assert first.code is IMPORT_FAILURE_CATALOG[step][0] @@ -187,7 +187,7 @@ def test_a_reason_the_user_cannot_act_on_names_the_import_identifier(step: Impor def test_a_failure_with_no_step_is_unclassified(): - failure = classify_import_failure(None, RuntimeError('boom')) + failure = classify_import_failure(None) assert failure.step is None assert failure.code is ImportErrorCode.UNEXPECTED_ERROR @@ -196,7 +196,7 @@ def test_a_failure_with_no_step_is_unclassified(): def test_every_reason_is_long_enough_for_the_column(): """`error_message` carries a `LENGTH(...) >= 3` CHECK; every sentence here clears it.""" - for _, reason in IMPORT_FAILURE_CATALOG.values(): + for _, reason, _ in IMPORT_FAILURE_CATALOG.values(): assert len(reason) >= 3 @@ -343,8 +343,13 @@ async def test_a_report_without_an_exception_object_says_so(): @mark.asyncio -async def test_an_explicit_code_keeps_the_step_s_sentence(): - """The cleanup case: its own code, and a sentence a person can still read.""" +async def test_an_explicit_code_gets_its_own_sentence_not_the_unclassified_one(): + """The cleanup case: its own code, and a sentence a person can still read. + + A cleanup-only failure means the model *was* imported — only its temporary files were left + behind. `UNCLASSIFIED_FAILURE`'s sentence ("no model was created") would be false here, so the + override in `_OVERRIDE_REASONS` must be what is reported, not the catalog's fallback. + """ sinks = FakeSinks() failure = await report_import_failure( @@ -358,10 +363,64 @@ async def test_an_explicit_code_keeps_the_step_s_sentence(): ) assert failure.code is ImportErrorCode.CLEANUP_INCOMPLETE - assert failure.reason == UNCLASSIFIED_FAILURE[1] + assert failure.reason != UNCLASSIFIED_FAILURE[1] + assert 'no model was created' not in failure.reason + assert 'unexpected reason' not in failure.reason + assert 'imported successfully' in failure.reason + assert sinks.notifications[0]['message'] == failure.reason assert sinks.notifications[0]['level'] == 'ERROR' +@mark.asyncio +async def test_an_unclaimable_row_gets_its_own_reason_not_the_generic_one(): + """`LOG_ROW_NOT_CLAIMABLE` never attempted the pipeline, so it must not say "no model was + created" either — it gets its own sentence about the request never starting.""" + sinks = FakeSinks() + + failure = await report_import_failure( + sinks, + None, + ImportLogRowNotClaimableError('row 29 is in status IMPORT_ERROR', import_run_id=29), + import_run_id=29, + workflow_id='manual-import-29', + code=ImportErrorCode.LOG_ROW_NOT_CLAIMABLE, + marker='not_claimable', + ) + + assert failure.code is ImportErrorCode.LOG_ROW_NOT_CLAIMABLE + assert failure.reason != UNCLASSIFIED_FAILURE[1] + assert 'no model was created' not in failure.reason + assert 'unexpected reason' not in failure.reason + assert failure.reason == 'The import request could not be started.' + assert sinks.notifications[0]['message'] == failure.reason + + +@mark.asyncio +async def test_a_status_write_failure_gets_its_own_reason_not_the_generic_one(): + """`STATUS_NOT_RECORDED` is a terminal-write failure, not a stalled pipeline — its sentence + says the outcome could not be recorded, not that nothing was created.""" + sinks = FakeSinks() + + failure = await report_import_failure( + sinks, + None, + None, + import_run_id=77, + workflow_id='wf', + code=ImportErrorCode.STATUS_NOT_RECORDED, + marker='status_write', + verdict='succeeded=True step=model_document gate=None', + detail_text='OperationalError: connection refused', + ) + + assert failure.code is ImportErrorCode.STATUS_NOT_RECORDED + assert failure.reason != UNCLASSIFIED_FAILURE[1] + assert 'no model was created' not in failure.reason + assert 'unexpected reason' not in failure.reason + assert failure.reason == 'The import outcome could not be recorded.' + assert sinks.notifications[0]['message'] == failure.reason + + # --------------------------------------------- the fields the two log lines carry # # The message text is for a person reading a log; these fields are for the query that finds it. Both @@ -443,7 +502,7 @@ async def test_the_user_facing_line_is_greppable_by_the_same_fields(): ) notified = sinks.notifications[0] - assert notified['message'] == UNCLASSIFIED_FAILURE[1] + assert notified['message'] == 'The import request could not be started.' assert notified['metadata']['import_code'] == ImportErrorCode.LOG_ROW_NOT_CLAIMABLE.value assert notified['metadata']['import_step'] == 'not_claimable' assert notified['metadata']['import_run_id'] == 29 @@ -498,6 +557,31 @@ async def test_no_field_can_shadow_the_formatter_s_own_keys(): assert not {'at', 'msg', 'timestamp'} & set(fields) +def test_walk_cause_chain_yields_just_the_exception_when_there_is_no_cause(): + error = RuntimeError('alone') + + assert list(walk_cause_chain(error)) == [error] + + +def test_walk_cause_chain_yields_the_whole_chain_when_shorter_than_the_limit(): + root = RuntimeError('root') + middle = RuntimeError('middle') + middle.__cause__ = root + outer = RuntimeError('outer') + outer.__cause__ = middle + + assert list(walk_cause_chain(outer, limit=10)) == [outer, middle, root] + + +def test_walk_cause_chain_stops_at_the_limit_for_a_chain_longer_than_it(): + first = RuntimeError('first') + second = RuntimeError('second') + first.__cause__ = second + second.__cause__ = first + + assert list(walk_cause_chain(first, limit=3)) == [first, second, first] + + def test_the_chain_is_flattened_root_cause_first(): """Because the first line is what becomes `error_summary`, and it has to be the cause.""" kind, detail = describe_failure_chain(_wrapped_activity_failure()) diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index d576b5f..a6ea506 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -1978,6 +1978,28 @@ async def test_download_model_wrapper_failure_propagates(joblib_mock, mlflow, ml mlflow_repository.client.download_artifacts.assert_not_called() +@patch(JOBLIB_PATH) +@pytest.mark.asyncio +async def test_download_model_wrapper_failure_emits_error_metric( + joblib_mock, mlflow, mlflow_repository +): + mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) + mlflow_repository.dowload_artifacts = AsyncMock() + loader_error = mlflow_lib.exceptions.MlflowException('wrapper boom') + mlflow.pyfunc.load_model.side_effect = loader_error + + with pytest.raises(mlflow_lib.exceptions.MlflowException) as exc_info: + await mlflow_repository.download_model('test_model', {}, 'predict', 'pyfunc', True) + + assert exc_info.value is loader_error + joblib_mock.load.assert_not_called() + mlflow_repository.client.download_artifacts.assert_not_called() + + error_tags = emitted_tags(mlflow_repository.emit_metric, metrics.MODEL_READ_ERROR_COUNT) + assert len(error_tags) == 1 + assert error_tags[0]['operation_type'] == 'download_model_wrapper' + + @patch(JOBLIB_PATH) @pytest.mark.asyncio async def test_load_predict_model_joblib_success_metrics( diff --git a/tests/laborious/utils/test_connectors_config.py b/tests/laborious/utils/test_connectors_config.py index 08639c1..015b99b 100644 --- a/tests/laborious/utils/test_connectors_config.py +++ b/tests/laborious/utils/test_connectors_config.py @@ -173,6 +173,7 @@ def test_build_import_config_with_defaults(clean_import_env): 'max_uncompressed_bytes': 4 * 1024 * 1024 * 1024, 'max_compression_ratio': 200.0, 'models_collection': 'models', + 'counters_collection': 'model_id_counters', 'retention_days': 7, } @@ -187,6 +188,7 @@ def test_build_import_config_reads_the_environment(clean_import_env): clean_import_env.setenv('IMPORT_MAX_UNCOMPRESSED_BYTES', '4096') clean_import_env.setenv('IMPORT_MAX_COMPRESSION_RATIO', '12.5') clean_import_env.setenv('IMPORT_MODELS_COLLECTION', 'models_v2') + clean_import_env.setenv('IMPORT_MODEL_ID_COUNTERS_COLLECTION', 'model_id_counters_v2') clean_import_env.setenv('IMPORT_BUNDLE_RETENTION_DAYS', '14') config = build_import_config() @@ -202,6 +204,7 @@ def test_build_import_config_reads_the_environment(clean_import_env): assert config['max_uncompressed_bytes'] == 4096 assert config['max_compression_ratio'] == 12.5 assert config['models_collection'] == 'models_v2' + assert config['counters_collection'] == 'model_id_counters_v2' assert config['retention_days'] == 14 diff --git a/tests/laborious/utils/test_time.py b/tests/laborious/utils/test_time.py new file mode 100644 index 0000000..1feb957 --- /dev/null +++ b/tests/laborious/utils/test_time.py @@ -0,0 +1,19 @@ +from datetime import UTC, datetime + +from laborious.utils.time import to_naive + + +def test_to_naive_returns_naive_input_unchanged() -> None: + moment = datetime(2024, 1, 1, 12, 30, 45) + + assert to_naive(moment) == moment + assert to_naive(moment).tzinfo is None + + +def test_to_naive_strips_tzinfo_from_aware_input() -> None: + moment = datetime(2024, 1, 1, 12, 30, 45, tzinfo=UTC) + + result = to_naive(moment) + + assert result.tzinfo is None + assert result == datetime(2024, 1, 1, 12, 30, 45) diff --git a/tests/laborious/worker/test_runtime_task_queues.py b/tests/laborious/worker/test_runtime_task_queues.py index 3dff897..6dcea16 100644 --- a/tests/laborious/worker/test_runtime_task_queues.py +++ b/tests/laborious/worker/test_runtime_task_queues.py @@ -109,6 +109,6 @@ def test_the_import_worker_registers_only_the_import_activities(): if isinstance(inner, ast.keyword) and inner.arg == 'other_workflows': assert ast.unparse(inner.value) == '[]' - assert len(activities) == 14 + assert len(activities) == 11 assert all(name.startswith('activities.') for name in activities) assert all('import' in name for name in activities) diff --git a/tests/laborious/workflows/test_import_model.py b/tests/laborious/workflows/test_import_model.py index 8d05c5c..1e68237 100644 --- a/tests/laborious/workflows/test_import_model.py +++ b/tests/laborious/workflows/test_import_model.py @@ -9,6 +9,7 @@ Everything is asserted through the recorded activity calls, because that is exac workflow is: a sequence of activity invocations with policies attached. """ +import asyncio from datetime import timedelta from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -27,6 +28,7 @@ from laborious.utils.models.import_errors import ( ImportGateRejectedError, ImportInputError, ImportLogRowNotClaimableError, + ImportStatusRejectedError, ) from laborious.workflows.import_model import ( REJECTED_INPUT_FIELDS, @@ -60,20 +62,16 @@ STATUS_ACTIVITIES = ( CLAIM_AND_TERMINAL = (Activities.claim_import_status, Activities.record_import_terminal_status) HINTS = (Activities.record_import_status, Activities.record_import_names) -# What a correct import does, in order. +# What a correct import does, in order. Download, decryption/extraction, experiment creation and +# artifact upload run inside one fused activity (bug 8), so only one hint precedes it, and there is +# no longer a separate activity call for each of those four phases. EXPECTED_ORDER = [ Activities.claim_import_status, Activities.record_import_status, Activities.ensure_import_bundle_retention, - Activities.download_import_bundle, - Activities.record_import_status, - Activities.open_import_bundle, + Activities.prepare_and_upload_import_bundle, Activities.record_import_names, Activities.record_import_status, - Activities.create_import_experiment, - Activities.record_import_status, - Activities.upload_import_artifacts, - Activities.record_import_status, Activities.register_import_model_version, Activities.record_import_status, Activities.write_import_model_document, @@ -81,9 +79,9 @@ EXPECTED_ORDER = [ Activities.record_import_terminal_status, ] +# The two provisioning activities left as their own Temporal activities after the fusion (bug 8 +# folded experiment creation and artifact upload into `prepare_and_upload_import_bundle`). PROVISIONING = ( - Activities.create_import_experiment, - Activities.upload_import_artifacts, Activities.register_import_model_version, Activities.write_import_model_document, ) @@ -95,19 +93,16 @@ HAPPY_RESULTS = { 'file_name': None, 'request_digest': None, }, - Activities.download_import_bundle: { - 'path': '/work/bundle.sientia', - 'size': 4096, - 'digest': DIGEST, - }, - Activities.open_import_bundle: { + Activities.prepare_and_upload_import_bundle: { 'extracted_dir': '/work/extract', 'metadata': BUNDLE_METADATA, 'parameters': BUNDLE_PARAMETERS, 'digest': DIGEST, + 'experiment_id': '17', + 'experiment_name': 'factory_experiment', + 'run_id': RUN_ID, + 'run_name': 'import-factory_model-v2', }, - Activities.create_import_experiment: {'experiment_id': '17'}, - Activities.upload_import_artifacts: {'run_id': RUN_ID, 'run_name': 'import-factory_model-v2'}, Activities.register_import_model_version: {'model_name': 'factory_model', 'version': '4'}, Activities.write_import_model_document: {'written': True, 'id': '8'}, Activities.cleanup_import_files: {'cleaned': True, 'existed': True}, @@ -210,21 +205,60 @@ async def test_the_claim_is_the_first_thing_that_happens( @mark.asyncio -async def test_the_names_are_recorded_before_mlflow_is_touched( +async def test_the_names_are_recorded_after_the_bundle_is_prepared_and_before_registration( import_workflow: ImportModel, workflow_env ): - """So a person watching the import list sees which model is being imported, while it runs.""" + """The names are only known once the fused activity returns (bug 8); they still land before + the two provisioning steps that follow it — the registered version and the model document.""" await import_workflow.run(valid_input()) order = workflow_env.recorder.order - assert order.index(Activities.record_import_names) < order.index( - Activities.create_import_experiment + assert ( + order.index(Activities.prepare_and_upload_import_bundle) + < order.index(Activities.record_import_names) + < order.index(Activities.register_import_model_version) ) names = workflow_env.recorder.payload_of(Activities.record_import_names) assert names['experiment_name'] == 'factory_experiment' assert names['run_name'] == 'import-factory_model-v2' +@mark.asyncio +async def test_retention_and_the_bundle_activity_are_both_scheduled_before_either_finishes( + import_workflow: ImportModel, +): + """Achado #9: `_ensure_retention` and the fused prepare-and-upload activity run under + `asyncio.gather`, so neither has to complete — or even be scheduled — before the other. Call + order between them isn't asserted (gather makes no such guarantee); only that the second + activity gets recorded while the first is still pending confirms neither blocks the other.""" + retention_started = asyncio.Event() + release_retention = asyncio.Event() + + class BlockingRecorder(Recorder): + async def __call__(self, method, payload, **kwargs): + self.calls.append(SimpleNamespace(method=method, payload=payload, kwargs=kwargs)) + if method is Activities.ensure_import_bundle_retention: + retention_started.set() + await release_retention.wait() + if method in self.failures: + raise self.failures[method] + return self.results.get(method, {}) + + recorder = BlockingRecorder() + patcher, _ = run_with(recorder) + task = asyncio.ensure_future(import_workflow.run(valid_input())) + try: + await asyncio.wait_for(retention_started.wait(), timeout=1) + # Retention is still awaiting `release_retention`; the bundle activity must already have + # been scheduled and recorded rather than waiting for retention to finish first. + await asyncio.sleep(0) + assert Activities.prepare_and_upload_import_bundle in recorder.order + finally: + release_retention.set() + await task + patcher.stop() + + @mark.asyncio async def test_the_model_is_named_from_the_bundles_model_name( import_workflow: ImportModel, workflow_env @@ -247,11 +281,13 @@ async def test_the_model_is_named_from_the_bundles_model_name( async def test_the_run_is_created_with_exactly_the_recorded_name( import_workflow: ImportModel, workflow_env ): + """The fused activity computes the run name and creates the run with it; the workflow then + records that same value, so the two cannot diverge.""" await import_workflow.run(valid_input()) recorded = workflow_env.recorder.payload_of(Activities.record_import_names)['run_name'] - used = workflow_env.recorder.payload_of(Activities.upload_import_artifacts)['run_name'] - assert used == recorded + created = workflow_env.recorder.results[Activities.prepare_and_upload_import_bundle]['run_name'] + assert recorded == created @mark.asyncio @@ -296,7 +332,7 @@ async def test_the_download_is_handed_the_digest_the_request_carried( ): await import_workflow.run(valid_input()) - payload = workflow_env.recorder.payload_of(Activities.download_import_bundle) + payload = workflow_env.recorder.payload_of(Activities.prepare_and_upload_import_bundle) assert payload['expected_digest'] == DIGEST assert payload['object_key'] == OBJECT_KEY @@ -308,10 +344,9 @@ async def test_the_download_is_handed_the_digest_the_request_carried( @mark.parametrize( ('failing', 'step'), [ - (Activities.download_import_bundle, ImportStep.DOWNLOAD), - (Activities.open_import_bundle, ImportStep.DECRYPTION), - (Activities.create_import_experiment, ImportStep.EXPERIMENT_CREATION), - (Activities.upload_import_artifacts, ImportStep.ARTIFACT_UPLOAD), + # An unclassified failure from the fused activity (download/decryption/experiment/upload) + # falls back to its first phase, since that is the step the workflow was attempting. + (Activities.prepare_and_upload_import_bundle, ImportStep.DOWNLOAD), (Activities.register_import_model_version, ImportStep.REGISTRATION), (Activities.write_import_model_document, ImportStep.MODEL_DOCUMENT), ], @@ -348,7 +383,7 @@ async def test_a_failing_step_is_recorded_as_that_step_and_stops_the_import( async def test_nothing_is_provisioned_before_the_gates_pass(import_workflow: ImportModel): recorder = Recorder( failures={ - Activities.open_import_bundle: ImportGateRejectedError( + Activities.prepare_and_upload_import_bundle: ImportGateRejectedError( 'rejected', step=ImportStep.ARCHIVE_INSPECTION, gate=4 ) } @@ -373,7 +408,7 @@ async def test_a_gate_rejection_is_recorded_with_the_gate_that_rejected_it( step the workflow was attempting.""" recorder = Recorder( failures={ - Activities.open_import_bundle: ImportGateRejectedError( + Activities.prepare_and_upload_import_bundle: ImportGateRejectedError( 'rejected', step=ImportStep.CONTENT_POLICY, gate=7 ) } @@ -395,7 +430,7 @@ async def test_a_gate_rejection_is_recorded_with_the_gate_that_rejected_it( async def test_an_ambiguous_decryption_failure_keeps_its_step(import_workflow: ImportModel): recorder = Recorder( failures={ - Activities.open_import_bundle: ImportGateRejectedError( + Activities.prepare_and_upload_import_bundle: ImportGateRejectedError( 'the bundle could not be opened', step=ImportStep.DECRYPTION, gate=3 ) } @@ -433,7 +468,7 @@ async def test_an_unknown_field_is_rejected_rather_than_ignored( # After the claim, so the refusal lands on the record instead of only in the logs. assert recorder.order[0] is Activities.claim_import_status - assert Activities.download_import_bundle not in recorder.order + assert Activities.prepare_and_upload_import_bundle not in recorder.order terminal = recorder.payload_of(Activities.record_import_terminal_status) assert terminal['step'] == ImportStep.RECEIVED.value assert terminal['succeeded'] is False @@ -457,7 +492,7 @@ async def test_a_missing_or_malformed_digest_fails_before_anything_is_downloaded finally: patcher.stop() - assert Activities.download_import_bundle not in recorder.order + assert Activities.prepare_and_upload_import_bundle not in recorder.order assert ( recorder.payload_of(Activities.record_import_terminal_status)['step'] == ImportStep.RECEIVED.value @@ -479,7 +514,7 @@ async def test_a_request_missing_a_required_field_fails_at_received( finally: patcher.stop() - assert Activities.download_import_bundle not in recorder.order + assert Activities.prepare_and_upload_import_bundle not in recorder.order @mark.asyncio @@ -503,7 +538,7 @@ async def test_a_record_naming_another_object_is_a_received_failure( finally: patcher.stop() - assert Activities.download_import_bundle not in recorder.order + assert Activities.prepare_and_upload_import_bundle not in recorder.order @mark.asyncio @@ -527,7 +562,7 @@ async def test_a_record_naming_another_digest_is_a_received_failure( finally: patcher.stop() - assert Activities.download_import_bundle not in recorder.order + assert Activities.prepare_and_upload_import_bundle not in recorder.order @mark.asyncio @@ -595,7 +630,7 @@ async def test_a_lost_names_write_does_not_fail_a_working_import(import_workflow patcher.stop() assert result['version'] == '4' - assert Activities.upload_import_artifacts in recorder.order + assert Activities.prepare_and_upload_import_bundle in recorder.order @mark.asyncio @@ -712,7 +747,7 @@ async def test_the_failure_that_ends_an_import_is_reported_through_the_channel( nothing at all — only the SDK's own traceback for the attempt, with no id and no step.""" recorder = Recorder( failures={ - Activities.create_import_experiment: activity_failure( + Activities.prepare_and_upload_import_bundle: activity_failure( 'API request to http://localhost:5080/api/2.0/mlflow/experiments/get-by-name ' 'failed with exception Connection refused', 'MlflowException', @@ -728,7 +763,10 @@ async def test_the_failure_that_ends_an_import_is_reported_through_the_channel( patcher.stop() reported = recorder.payload_of(Activities.report_import_pipeline_failure) - assert reported['step'] == ImportStep.EXPERIMENT_CREATION.value + # An unclassified failure from the fused activity falls back to its first phase: the workflow + # has no more precise location for it, since `current_step` only advances past `DOWNLOAD` once + # the fused activity returns. + assert reported['step'] == ImportStep.DOWNLOAD.value assert reported['error_type'] == 'MlflowException' assert reported['detail'].splitlines()[0].startswith('MlflowException: API request to') assert reported['import_run_id'] == 77 @@ -741,7 +779,9 @@ async def test_the_report_comes_before_the_cleanup_and_the_terminal_write( ): """So the log reads in the order things happened, and the row's step matches the line's.""" recorder = Recorder( - failures={Activities.upload_import_artifacts: activity_failure('bucket gone', 'S3Error')} + failures={ + Activities.prepare_and_upload_import_bundle: activity_failure('bucket gone', 'S3Error') + } ) patcher, _ = run_with(recorder) payload = valid_input() @@ -759,7 +799,7 @@ async def test_the_report_comes_before_the_cleanup_and_the_terminal_write( ) reported = recorder.payload_of(Activities.report_import_pipeline_failure) terminal = recorder.payload_of(Activities.record_import_terminal_status) - assert reported['step'] == terminal['step'] == ImportStep.ARTIFACT_UPLOAD.value + assert reported['step'] == terminal['step'] == ImportStep.DOWNLOAD.value @mark.asyncio @@ -768,7 +808,7 @@ async def test_the_gate_that_rejected_the_bundle_travels_into_the_report( ): recorder = Recorder( failures={ - Activities.open_import_bundle: ImportGateRejectedError( + Activities.prepare_and_upload_import_bundle: ImportGateRejectedError( 'gate 3 rejected the bundle', step=ImportStep.DECRYPTION, gate=3 ) } @@ -807,6 +847,33 @@ async def test_a_claim_that_matches_nothing_is_not_reported_twice(import_workflo assert Activities.report_import_pipeline_failure not in recorder.order +@mark.asyncio +async def test_a_claim_write_that_is_refused_is_not_reported_twice_by_the_workflow( + import_workflow: ImportModel, +): + """Bug 7: the claim's own write failure is now reported *inside* `claim_import_status` itself, + before it re-raises (`code=STATUS_NOT_RECORDED`). The claim still runs outside the workflow's + try/finally — there is no row this workflow owns yet — so the workflow's own pipeline-failure + channel must not report it a second time.""" + recorder = Recorder( + failures={ + Activities.claim_import_status: ImportStatusRejectedError( + 'the import log database refused the status statement' + ) + } + ) + patcher, _ = run_with(recorder) + payload = valid_input() + try: + with raises(ImportStatusRejectedError): + await import_workflow.run(payload) + finally: + patcher.stop() + + assert recorder.order == [Activities.claim_import_status] + assert Activities.report_import_pipeline_failure not in recorder.order + + @mark.asyncio async def test_a_report_that_cannot_be_made_does_not_mask_the_failure( import_workflow: ImportModel, @@ -842,7 +909,9 @@ async def test_the_report_gives_up_inside_the_short_bound(import_workflow: Impor """A failure already happened; spending 151 seconds telling the log about it would be worse.""" recorder = Recorder( failures={ - Activities.create_import_experiment: activity_failure('refused', 'MlflowException') + Activities.prepare_and_upload_import_bundle: activity_failure( + 'refused', 'MlflowException' + ) } ) patcher, _ = run_with(recorder) @@ -904,7 +973,10 @@ async def test_no_provisioning_call_references_a_retrying_policy( await import_workflow.run(valid_input()) for call in workflow_env.recorder.calls: - if call.method in PROVISIONING or call.method is Activities.open_import_bundle: + if ( + call.method in PROVISIONING + or call.method is Activities.prepare_and_upload_import_bundle + ): assert call.kwargs['retry_policy'] not in ( IMPORT_STATUS_RETRY_POLICY, IMPORT_STATUS_HINT_RETRY_POLICY, From eb89917c5a86ae4f51f08a0f0f1fcf72d95d3d44 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 8 Sep 2026 17:19:45 -0300 Subject: [PATCH 43/64] QTZPOC-20 QTZPOC-20: Clarify broad exception handling in OPC repository and refactor asyncio imports in worker tests. --- laborious/utils/repository/opc_repository.py | 2 +- tests/laborious/worker/test_worker_main.py | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/laborious/utils/repository/opc_repository.py b/laborious/utils/repository/opc_repository.py index 802c318..9e866b3 100644 --- a/laborious/utils/repository/opc_repository.py +++ b/laborious/utils/repository/opc_repository.py @@ -79,7 +79,7 @@ def _opc_authentication_token_str(client: Client | None) -> str: if tok is None: return 'unknown' return str(tok) - except Exception: + except Exception: # NOSONAR - best-effort label for logs/metrics, must not fail loud return 'unknown' diff --git a/tests/laborious/worker/test_worker_main.py b/tests/laborious/worker/test_worker_main.py index e87aac7..44ce6d4 100644 --- a/tests/laborious/worker/test_worker_main.py +++ b/tests/laborious/worker/test_worker_main.py @@ -5,6 +5,7 @@ Everything I/O-bound (Temporal client, Prometheus HTTP server, Mongo/Postgres/Mi mocked; only the control flow inside ``worker.py`` is under test. """ +import asyncio from unittest.mock import ANY, AsyncMock, MagicMock, patch from pytest import mark, raises @@ -20,8 +21,6 @@ def test_main_exits_when_runtime_is_unset(mock_get_logger, mock_app_up, monkeypa mock_get_logger.return_value = logger with raises(SystemExit) as exc: - import asyncio - asyncio.run(main()) assert exc.value.code == 1 @@ -55,8 +54,6 @@ class TestMainHappyPath: mock_prepare_worker.return_value = worker mock_connect.return_value = MagicMock() - import asyncio - with raises(SystemExit) as exc: asyncio.run(main()) return exc.value.code @@ -198,8 +195,6 @@ class TestMainHappyPath: worker.run = AsyncMock(side_effect=RuntimeError('boom')) mock_prepare_worker.return_value = worker - import asyncio - with raises(SystemExit) as exc: asyncio.run(main()) From abfd474e4ddf58472a9ee07c62b779cd8f83aa2a Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 9 Sep 2026 08:26:59 -0300 Subject: [PATCH 44/64] QTZPOC-20 QTZPOC-20: Enhance OPC repository client configuration and secure connection validation; refine MLflow test error handling and async test execution; add SonarQube suppressions. --- laborious/utils/connectors_config.py | 2 +- laborious/utils/models/import_errors.py | 6 +++--- laborious/utils/repository/opc_repository.py | 9 +++++---- tests/laborious/activities/test_mlflow.py | 6 +++--- tests/laborious/utils/repository/test_opc_repository.py | 1 - tests/laborious/worker/test_worker_main.py | 9 ++++++--- 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/laborious/utils/connectors_config.py b/laborious/utils/connectors_config.py index 01e29e1..90f11c8 100644 --- a/laborious/utils/connectors_config.py +++ b/laborious/utils/connectors_config.py @@ -25,7 +25,7 @@ def build_mlflow_config() -> dict[str, Any]: 'host': getenv('MLFLOW_HOST', 'http://localhost'), 'port': int(getenv('MLFLOW_PORT', '5080')), 'username': getenv('MLFLOW_USERNAME', 'aignosi'), - 'password': getenv('MLFLOW_PASSWORD', 'aignosi'), + 'password': getenv('MLFLOW_PASSWORD', 'aignosi'), # NOSONAR } diff --git a/laborious/utils/models/import_errors.py b/laborious/utils/models/import_errors.py index 8d3c88c..96ca875 100644 --- a/laborious/utils/models/import_errors.py +++ b/laborious/utils/models/import_errors.py @@ -361,8 +361,8 @@ def describe_failure_chain(exception: BaseException) -> tuple[str, str]: """ links: list[str] = [] for candidate in walk_cause_chain(exception): - kind = _type_name(candidate) - message = _one_line(str(candidate)) + kind = _type_name(candidate) # NOSONAR + message = _one_line(str(candidate)) # NOSONAR # `ApplicationError.__str__` already prefixes its own type, and `MlflowException: # MlflowException: …` reads like two failures rather than one. if not message: @@ -422,7 +422,7 @@ def failure_location(exception: BaseException | None) -> tuple[str | None, int | if exception is None: return None, None for candidate in walk_cause_chain(exception): - details = getattr(candidate, 'details', None) + details = getattr(candidate, 'details', None) # NOSONAR if details and isinstance(details[0], dict) and 'step' in details[0]: location = details[0] gate = location.get('gate') diff --git a/laborious/utils/repository/opc_repository.py b/laborious/utils/repository/opc_repository.py index 9e866b3..8ea4007 100644 --- a/laborious/utils/repository/opc_repository.py +++ b/laborious/utils/repository/opc_repository.py @@ -94,8 +94,8 @@ def _opc_status_from_exception(exc: BaseException) -> str: str: Status class name or generic Python exception name. """ for candidate in walk_cause_chain(exc): - if isinstance(candidate, UaStatusCodeError): - return type(candidate).__name__ + if isinstance(candidate, UaStatusCodeError): # NOSONAR + return type(candidate).__name__ # NOSONAR return type(exc).__name__ @@ -261,12 +261,14 @@ class OpcRepository(SientiaMonitoring): Configure certificates and timeouts on the asyncua client. Raises: - ValueError: If cert paths or client are missing. + ValueError: If cert paths, server URI or client are missing. """ if self.cert_path is None or self.private_key_path is None: raise ValueError( 'Certificate and private key paths must be provided for secure connection.' ) + if self.server_uri is None: + raise ValueError('server_uri must be provided for secure connection.') cert = Path(self.cert_path) private_key = Path(self.private_key_path) @@ -303,7 +305,6 @@ class OpcRepository(SientiaMonitoring): self.client = Client(self.url, timeout=10, watchdog_intervall=50) # type: ignore[attr-defined] self.client.name = self.pod_id - self.client.application_name = self.pod_id pod_uri = self.pod_id.replace('-', ':') self.client.application_uri = pod_uri self.client.product_uri = pod_uri diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index e87c618..350d6f9 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -519,7 +519,7 @@ async def test_retrain_model_data_error_no_minio_repository(mlflow): } ) - assert str(e.value) == 'Minio repository not initialized' + assert str(e.value) == 'Minio repository not initialized' @mark.asyncio @@ -641,12 +641,12 @@ async def test_get_reference_data_exception(mlflow): 'model_name': 'test_model', } - mlflow.model_monitoring_repository.load_artifact_dataframe.side_effect = Exception( + mlflow.model_monitoring_repository.load_artifact_dataframe.side_effect = RuntimeError( 'Error loading artifact' ) # Act & Assert - with raises(Exception) as e: + with raises(RuntimeError) as e: await mlflow.get_reference_data(input_data) assert str(e.value) == 'Error loading artifact' diff --git a/tests/laborious/utils/repository/test_opc_repository.py b/tests/laborious/utils/repository/test_opc_repository.py index f9d489b..04ffa8a 100644 --- a/tests/laborious/utils/repository/test_opc_repository.py +++ b/tests/laborious/utils/repository/test_opc_repository.py @@ -206,7 +206,6 @@ async def test_create_client_builds_a_secured_client(opc_repository, mock_client assert opc_repository.client is mock_client assert mock_client.name == 'pod-1' - assert mock_client.application_name == 'pod-1' assert mock_client.application_uri == 'pod:1' assert mock_client.product_uri == 'pod:1' opc_repository.set_security.assert_called_once() diff --git a/tests/laborious/worker/test_worker_main.py b/tests/laborious/worker/test_worker_main.py index 44ce6d4..92e872f 100644 --- a/tests/laborious/worker/test_worker_main.py +++ b/tests/laborious/worker/test_worker_main.py @@ -20,8 +20,9 @@ def test_main_exits_when_runtime_is_unset(mock_get_logger, mock_app_up, monkeypa logger = MagicMock() mock_get_logger.return_value = logger + coro = main() with raises(SystemExit) as exc: - asyncio.run(main()) + asyncio.run(coro) assert exc.value.code == 1 logger.custom_critical.assert_called_once() @@ -54,8 +55,9 @@ class TestMainHappyPath: mock_prepare_worker.return_value = worker mock_connect.return_value = MagicMock() + coro = main() with raises(SystemExit) as exc: - asyncio.run(main()) + asyncio.run(coro) return exc.value.code def test_runs_four_workers_without_the_import_flag( # noqa: PLR0913 @@ -195,8 +197,9 @@ class TestMainHappyPath: worker.run = AsyncMock(side_effect=RuntimeError('boom')) mock_prepare_worker.return_value = worker + coro = main() with raises(SystemExit) as exc: - asyncio.run(main()) + asyncio.run(coro) assert exc.value.code == 1 logger.custom_error.assert_called_once_with(ANY, ANY) From 37544edafecf5de8af007ece893438108e9aa250 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 9 Sep 2026 08:49:23 -0300 Subject: [PATCH 45/64] QTZPOC-20 QTZPOC-20: Improve error handling and classification tests for model import activities and add validation for OPC repository security. --- .../activities/test_model_import_bundle.py | 71 +++++++++++++++++++ .../activities/test_model_import_status.py | 18 +++++ .../utils/repository/test_opc_repository.py | 10 +++ 3 files changed, 99 insertions(+) diff --git a/tests/laborious/activities/test_model_import_bundle.py b/tests/laborious/activities/test_model_import_bundle.py index ba9fd4d..8a41ddb 100644 --- a/tests/laborious/activities/test_model_import_bundle.py +++ b/tests/laborious/activities/test_model_import_bundle.py @@ -646,6 +646,77 @@ async def test_a_gate_failure_inside_the_fused_activity_still_names_its_step_and assert failure.value.details[0]['gate'] == 3 +async def _fake_download(_input_data: dict) -> dict: + return {'path': 'p', 'size': 1, 'digest': 'd'} + + +async def _fake_open(_input_data: dict) -> dict: + return { + 'extracted_dir': 'e', + 'metadata': { + 'model_name': 'factory_model', + 'model_version': '1', + 'experiment_name': 'factory_experiment', + }, + 'parameters': {}, + 'digest': 'd', + } + + +@mark.asyncio +async def test_a_misconfigured_repository_at_experiment_creation_is_not_wrapped( + import_activity: ModelImport, +): + """`ImportMisconfiguredError` is a config problem, not a broken dependency — it must reach the + workflow untouched, not get rewrapped into a generic `ApplicationError`.""" + import_activity.import_mlflow_repository = None + + with ( + patch.object(import_activity, '_download_bundle', side_effect=_fake_download), + patch.object(import_activity, '_open_bundle', side_effect=_fake_open), + raises(ImportMisconfiguredError), + ): + await import_activity.prepare_and_upload_import_bundle({'metadata': METADATA}) + + +@mark.asyncio +async def test_a_broken_dependency_at_experiment_creation_is_tagged_with_its_step( + import_activity: ModelImport, +): + """A raw MLflow/network failure carries no location of its own, so the fused activity must tag + it with the step it actually broke at — otherwise the workflow's fallback misclassifies it.""" + + async def fake_create_experiment(_input_data: dict) -> dict: + raise RuntimeError('mlflow unreachable') + + with ( + patch.object(import_activity, '_download_bundle', side_effect=_fake_download), + patch.object(import_activity, '_open_bundle', side_effect=_fake_open), + patch.object(import_activity, '_create_experiment', side_effect=fake_create_experiment), + raises(ApplicationError, match='mlflow unreachable') as failure, + ): + await import_activity.prepare_and_upload_import_bundle({'metadata': METADATA}) + + assert failure.value.details[0]['step'] == ImportStep.EXPERIMENT_CREATION.value + + +@mark.asyncio +async def test_a_misconfigured_repository_at_artifact_upload_is_not_wrapped( + import_activity: ModelImport, +): + async def fake_create_experiment(_input_data: dict) -> dict: + import_activity.import_mlflow_repository = None + return {'experiment_id': '1', 'experiment_name': 'factory_experiment'} + + with ( + patch.object(import_activity, '_download_bundle', side_effect=_fake_download), + patch.object(import_activity, '_open_bundle', side_effect=_fake_open), + patch.object(import_activity, '_create_experiment', side_effect=fake_create_experiment), + raises(ImportMisconfiguredError), + ): + await import_activity.prepare_and_upload_import_bundle({'metadata': METADATA}) + + @mark.asyncio async def test_a_retry_of_the_fused_activity_redoes_all_four_phases(import_activity: ModelImport): """A retry re-runs download, open, experiment creation and upload from the start. diff --git a/tests/laborious/activities/test_model_import_status.py b/tests/laborious/activities/test_model_import_status.py index 0233406..1960880 100644 --- a/tests/laborious/activities/test_model_import_status.py +++ b/tests/laborious/activities/test_model_import_status.py @@ -442,6 +442,24 @@ async def test_the_claim_writes_before_it_reads( assert engine.statements[1].verb == 'SELECT' +@mark.asyncio +async def test_a_refused_claim_statement_is_reported_and_reraised(): + """The database refusing the claim write is a config problem (bad column, CHECK violation), not + a retryable one — it must be reported, and it must still propagate to fail the activity.""" + engine = FakeStatusEngine( + write_errors=[IntegrityError('UPDATE ...', {}, Exception('refused'))], + ) + activity = build_activity(engine) + + with raises(ImportStatusRejectedError) as failure: + await activity.claim_import_status(claim_input()) + + assert failure.value.non_retryable is True + logged = activity.logger.custom_error.call_args_list[0].args[0] + assert ImportErrorCode.STATUS_NOT_RECORDED.value in logged + assert 'status_write' in logged + + @mark.asyncio @mark.parametrize( ('row', 'expected'), diff --git a/tests/laborious/utils/repository/test_opc_repository.py b/tests/laborious/utils/repository/test_opc_repository.py index 04ffa8a..e5bcbc1 100644 --- a/tests/laborious/utils/repository/test_opc_repository.py +++ b/tests/laborious/utils/repository/test_opc_repository.py @@ -144,6 +144,16 @@ async def test_set_security_missing_certificates(opc_repository): assert str(e) == 'Certificate and private key paths must be provided for secure connection.' +@pytest.mark.asyncio +async def test_set_security_missing_server_uri(opc_repository): + opc_repository.server_uri = None + + try: + await opc_repository.set_security() + except ValueError as e: + assert str(e) == 'server_uri must be provided for secure connection.' + + @pytest.mark.asyncio async def test_set_security_missing_client(opc_repository): opc_repository.client = None From d06ce0daae02ec4eded8b8ee99ebd7ef62bb868a Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 9 Sep 2026 08:58:48 -0300 Subject: [PATCH 46/64] QTZPOC-20 QTZPOC-20: Improve test readability by assigning claim_input() to a payload variable before use. --- tests/laborious/activities/test_model_import_status.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/laborious/activities/test_model_import_status.py b/tests/laborious/activities/test_model_import_status.py index 1960880..893d945 100644 --- a/tests/laborious/activities/test_model_import_status.py +++ b/tests/laborious/activities/test_model_import_status.py @@ -450,9 +450,10 @@ async def test_a_refused_claim_statement_is_reported_and_reraised(): write_errors=[IntegrityError('UPDATE ...', {}, Exception('refused'))], ) activity = build_activity(engine) + payload = claim_input() with raises(ImportStatusRejectedError) as failure: - await activity.claim_import_status(claim_input()) + await activity.claim_import_status(payload) assert failure.value.non_retryable is True logged = activity.logger.custom_error.call_args_list[0].args[0] From 4c7cae502f1b171690a54fc07bc90bcdc9d11ca3 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 10 Sep 2026 08:59:31 -0300 Subject: [PATCH 47/64] QTZPOC-20 QTZPOC-20: Refactor ModelImport status engine for lazy initialization, add byte-counted size validation to bundle extraction, optimize MLflow run ID lookups, and ensure workflow cancellation propagates correctly. --- laborious/activities/model_import.py | 57 ++++++++---- laborious/activities/opc.py | 18 +++- laborious/utils/bundle/reader.py | 20 +++- .../utils/repository/model_repository.py | 60 +++++++++--- laborious/workflows/import_model.py | 37 +++++--- .../activities/test_model_import_bundle.py | 37 ++++---- .../test_model_import_provisioning.py | 61 +++++++------ .../activities/test_model_import_status.py | 80 +++++++++++++--- tests/laborious/utils/bundle/test_reader.py | 48 +++++++++- .../utils/repository/test_model_repository.py | 62 +++++++++++-- .../laborious/workflows/test_import_model.py | 91 ++++++++++++++++++- 11 files changed, 452 insertions(+), 119 deletions(-) diff --git a/laborious/activities/model_import.py b/laborious/activities/model_import.py index 1a6f547..166962c 100644 --- a/laborious/activities/model_import.py +++ b/laborious/activities/model_import.py @@ -40,7 +40,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.repository.mongodb_repository import MongoDBRepository from sientia_do.temporal.constants import now from sqlalchemy import create_engine, text - from sqlalchemy.engine import URL + from sqlalchemy.engine import URL, Engine from sqlalchemy.exc import DataError, IntegrityError, ProgrammingError from sqlalchemy.sql import Executable from temporalio.common import RetryPolicy @@ -321,25 +321,37 @@ class ModelImport(SientiaMonitoring): self.import_minio_repository = minio_repository self.import_mlflow_repository = mlflow_repository self.mongo_repository = mongo_repository - self.status_engine = create_engine( - URL.create( - 'postgresql', - username=status_config['user'], - password=status_config['password'], - host=status_config['host'], - port=status_config['port'], - database=status_config['dbname'], - ), - pool_size=2, - max_overflow=3, - pool_pre_ping=True, - ) + self._status_engine: Engine | None = None + + @property + def status_engine(self) -> Engine: + """The BFF status database's engine, created on first access and cached from then on. + + A worker pod where the import feature is never exercised never opens a connection pool + against that Postgres: nothing in this module reaches `status_engine` unless an import + activity actually runs. + """ + if self._status_engine is None: + self._status_engine = create_engine( + URL.create( + 'postgresql', + username=self.status_config['user'], + password=self.status_config['password'], + host=self.status_config['host'], + port=self.status_config['port'], + database=self.status_config['dbname'], + ), + pool_size=2, + max_overflow=3, + pool_pre_ping=True, + ) + return self._status_engine def close(self) -> None: - """Dispose the status engine. The import holds no other connection.""" - engine = getattr(self, 'status_engine', None) - if engine is not None: - engine.dispose() + """Dispose the status engine, when one was ever created. The import holds no other + connection.""" + if self._status_engine is not None: + self._status_engine.dispose() # ------------------------------------------------------------------ helpers @@ -1128,7 +1140,7 @@ class ModelImport(SientiaMonitoring): upsert=True, return_document=ReturnDocument.AFTER, ) - return str(doc['seq']), 0 + return str(doc['seq']), 0 # type: ignore[index] # ----------------------------------------------------------- cleanup and retention @@ -1217,7 +1229,12 @@ class ModelImport(SientiaMonitoring): # The read still runs every time (it is how drift — someone editing the rule outside # this code — gets detected); only the write is skipped when nothing has drifted. self.info(f'import retention rule already correct on {bucket} ({days} days)', metadata) - return {'bucket': bucket, 'prefix': prefix, 'days': days, 'rules': len(existing.rules)} + return { + 'bucket': bucket, + 'prefix': prefix, + 'days': days, + 'rules': len(existing.rules), # type: ignore[union-attr] + } rules = [rule for rule in (existing.rules if existing else []) if rule.rule_id != rule_id] rules.append( diff --git a/laborious/activities/opc.py b/laborious/activities/opc.py index 04ed18e..2c14cf5 100644 --- a/laborious/activities/opc.py +++ b/laborious/activities/opc.py @@ -106,13 +106,21 @@ class OPC(SientiaMonitoring): 4. Logs connection status for operational visibility Raises: - Exception: If OPC repository initialization fails or connection - establishment encounters critical errors + ValueError: A server whose `cert_path` is set but whose `server_uri` + (or private key) is missing fails `OpcRepository.set_security()` + with no try/except between here and `main()`. This is a + configuration error, not a connectivity one, and is intentionally + fatal to the whole worker process — fix the deployment config + rather than catch it here. Note: - Connection failures are logged and reported but do not prevent - the initialization of other OPC servers. Each server is handled - independently to ensure maximum availability. + Connection failures (server unreachable, timeout, etc.) are logged + and reported but do not prevent the initialization of other OPC + servers — each server's `connect()` call catches those on its own. + This isolation does NOT extend to configuration errors raised before + a connection is attempted (see `Raises` above): a single + misconfigured server intentionally stops every other server, and the + whole worker, from starting. """ self.logger.info('Initializing OPC servers...') for opc_id, server in self.opc_servers.items(): diff --git a/laborious/utils/bundle/reader.py b/laborious/utils/bundle/reader.py index aea9646..5d223f4 100644 --- a/laborious/utils/bundle/reader.py +++ b/laborious/utils/bundle/reader.py @@ -35,6 +35,9 @@ import nacl.exceptions from laborious.utils.bundle import format as fmt from laborious.utils.bundle.steps import GATE_TO_STEP, ImportStep +# Gate 5 — chunk size for the byte-counted extraction copy loop. +_EXTRACT_CHUNK_BYTES = 1024 * 1024 + # Gate 6 — the layout `bundle.build_zip` writes and `model_repository.py` reads back. METADATA_FILE_NAME = 'metadata.json' ARTIFACTS_DIR_NAME = 'artifacts' @@ -389,8 +392,23 @@ class BundleReader: resolved.parent.mkdir(parents=True, exist_ok=True) # Written as a plain file: the archive's stored permissions are never restored, so no # member can arrive executable. + # + # Copied in chunks, counting bytes actually written, instead of `shutil.copyfileobj`: + # `entry.file_size` is central-directory metadata gate 4 already checked in aggregate, but a + # DEFLATE stream can be crafted to decompress to far more bytes than its own header + # declares. Aborting as soon as the running count exceeds `file_size` catches that member + # before the excess ever reaches disk, rather than trusting the declared size alone. + written = 0 with archive.open(entry) as source, resolved.open('wb') as sink: - shutil.copyfileobj(source, sink) + while chunk := source.read(_EXTRACT_CHUNK_BYTES): + written += len(chunk) + if written > entry.file_size: + raise BundleGateError( + f'archive member {entry.filename!r} decompresses to more than its ' + f'declared {entry.file_size} bytes', + gate=5, + ) + sink.write(chunk) # ------------------------------------------------------------------ gate 6 diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 1c32baf..118b69e 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -139,9 +139,9 @@ class MLFlowRepository(SientiaMonitoring): """ run_info = mlflow.get_run(run_id) if prediction: - model_uri = run_info.info.artifact_uri + '/prediction_model' + model_uri = run_info.info.artifact_uri + '/prediction_model' # type: ignore[operator] else: - model_uri = run_info.info.artifact_uri + '/data_model' + model_uri = run_info.info.artifact_uri + '/data_model' # type: ignore[operator] return model_uri def get_model_run_id(self, model_name: str, stage: str = 'Production') -> str: @@ -290,7 +290,11 @@ class MLFlowRepository(SientiaMonitoring): """ async def dowload_artifacts( - self, model_name: str, metadata: dict[str, Any], artifact_path: str = 'data_model' + self, + model_name: str, + metadata: dict[str, Any], + artifact_path: str = 'data_model', + run_id: str | None = None, ) -> str: """ Download artifacts from the latest production run of a model. @@ -299,11 +303,14 @@ class MLFlowRepository(SientiaMonitoring): model_name (str): Registered model name. metadata (dict[str, Any]): Metadata used for structured logging. artifact_path (str): Relative path to artifacts within the run. + run_id (str | None): Run already resolved by the caller. When `None`, + the Production run is resolved here. Returns: str: Local filesystem path where artifacts are saved. """ - run_id = self.get_model_run_id(model_name=model_name, stage='Production') + if run_id is None: + run_id = self.get_model_run_id(model_name=model_name, stage='Production') output_dir = f'{ARTIFACTS_PATH}/{model_name}' full_path = path.join(output_dir, artifact_path) @@ -472,6 +479,7 @@ class MLFlowRepository(SientiaMonitoring): artifact_name: str, flavor: str, metadata: dict[str, Any] | None = None, + run_id: str | None = None, ) -> str | None: """ Download only an artifact's `code` directory and put it on `sys.path`. @@ -489,6 +497,8 @@ class MLFlowRepository(SientiaMonitoring): flavor (str): Flavor doing the load, for the log lines only. metadata (dict[str, Any] | None): Metadata used for structured logging. + run_id (str | None): Run already resolved by the caller. When `None`, + the Production run is resolved here. Returns: str | None: Local directory the `code` directory was downloaded @@ -500,7 +510,8 @@ class MLFlowRepository(SientiaMonitoring): Exception: Any error raised while resolving the Production run, listing the artifact or downloading `code`. """ - run_id = self.get_model_run_id(model_name=model_name, stage='Production') + if run_id is None: + run_id = self.get_model_run_id(model_name=model_name, stage='Production') remote_code_path = f'{artifact_name}/code' if not self.check_artifact_exists(run_id, remote_code_path, metadata): @@ -623,7 +634,11 @@ class MLFlowRepository(SientiaMonitoring): return model, local_dir async def load_predict_model( - self, model_name: str, metadata: dict[str, Any], flavor: str = 'sklearn' + self, + model_name: str, + metadata: dict[str, Any], + flavor: str = 'sklearn', + run_id: str | None = None, ) -> tuple[Any, str | None]: """ Load a predictive model from the MLflow Model Registry. @@ -632,6 +647,10 @@ class MLFlowRepository(SientiaMonitoring): model_name (str): The name of the model to download from the registry. metadata (dict[str, Any]): Metadata used for structured logging. flavor (str): Model flavor ('pyfunc', 'sklearn', 'joblib') + run_id (str | None): Run already resolved by the caller. When `None`, + the Production run is resolved where needed (only the `joblib` + branch downloads by run; the `pyfunc`/`sklearn` branches load + from the symbolic `models://production` URI regardless). Returns: tuple[Any, str | None]: The loaded predictive model and the local @@ -667,6 +686,7 @@ class MLFlowRepository(SientiaMonitoring): model_name=model_name, artifact_name='prediction_model', metadata=metadata, + run_id=run_id, ) elif flavor == 'pyfunc': model, local_dir = mlflow.pyfunc.load_model(model_uri), None @@ -681,7 +701,11 @@ class MLFlowRepository(SientiaMonitoring): return model, local_dir async def load_transform_model( - self, model_name: str, metadata: dict[str, Any], flavor: str = 'sklearn' + self, + model_name: str, + metadata: dict[str, Any], + flavor: str = 'sklearn', + run_id: str | None = None, ) -> tuple[Any, str | None]: """ Load the latest Production version of a transformation model. @@ -693,6 +717,8 @@ class MLFlowRepository(SientiaMonitoring): model_name (str): The name of the model to download. metadata (dict[str, Any]): Metadata used for structured logging. flavor (str): Model flavor ('sklearn', 'pyfunc', 'joblib') + run_id (str | None): Run already resolved by the caller. When `None`, + the Production run is resolved here. Returns: tuple[Any, str | None]: The loaded model object, depending on the @@ -719,7 +745,9 @@ class MLFlowRepository(SientiaMonitoring): if flavor not in ('sklearn', 'pyfunc', 'joblib'): raise ValueError(INVALID_FLAVOR_MESSAGE) - latest_production_id = self.get_model_run_id(model_name=model_name, stage='Production') + latest_production_id = run_id or self.get_model_run_id( + model_name=model_name, stage='Production' + ) model_uri = ( None @@ -802,8 +830,10 @@ class MLFlowRepository(SientiaMonitoring): target = 'prediction_model' if model_type == 'predict' else 'data_model' + run_id = self.get_model_run_id(model_name=model_name, stage='Production') + code_artifact_dir = await self.download_artifact_code_dir( - model_name, target, flavor, metadata + model_name, target, flavor, metadata, run_id=run_id ) if load_wrapper: @@ -811,7 +841,9 @@ class MLFlowRepository(SientiaMonitoring): self.info(f'Loading wrapper for {model_type} model {model_name} with flavor {flavor}') try: - artifact_path = await self.dowload_artifacts(model_name, metadata, target) + artifact_path = await self.dowload_artifacts( + model_name, metadata, target, run_id=run_id + ) self.info( f'Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}' @@ -832,9 +864,13 @@ class MLFlowRepository(SientiaMonitoring): return model, code_artifact_dir or artifact_path if model_type == 'predict': - model, local_dir = await self.load_predict_model(model_name, metadata, flavor) + model, local_dir = await self.load_predict_model( + model_name, metadata, flavor, run_id=run_id + ) else: - model, local_dir = await self.load_transform_model(model_name, metadata, flavor) + model, local_dir = await self.load_transform_model( + model_name, metadata, flavor, run_id=run_id + ) return model, code_artifact_dir or local_dir diff --git a/laborious/workflows/import_model.py b/laborious/workflows/import_model.py index 3a81df7..0d8ea0d 100644 --- a/laborious/workflows/import_model.py +++ b/laborious/workflows/import_model.py @@ -26,6 +26,7 @@ with workflow.unsafe.imports_passed_through(): from typing import Any from sientia_do.temporal.policies import retry_policy + from temporalio.exceptions import is_cancelled_exception from laborious.activities.activities import Activities from laborious.activities.model_import import ( @@ -309,6 +310,8 @@ class ImportModel: start_to_close_timeout=STATUS_WRITE_TIMEOUT, ) except BaseException as error: # NOSONAR - a lost hint must not fail a working import + if is_cancelled_exception(error): + raise await self._report_status_write_failure(base, error) async def _record_terminal( @@ -344,6 +347,8 @@ class ImportModel: except ( BaseException ) as error: # NOSONAR - the outcome stands even when it cannot be written + if is_cancelled_exception(error): + raise await self._report_status_write_failure( base, error, @@ -377,8 +382,9 @@ class ImportModel: retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY, start_to_close_timeout=STATUS_WRITE_TIMEOUT, ) - except BaseException: # noqa: S110 # NOSONAR - the report of a report is where this stops - pass + except BaseException as error: # NOSONAR - the report of a report is where this stops + if is_cancelled_exception(error): + raise async def _report_status_write_failure( self, @@ -394,10 +400,12 @@ class ImportModel: retry_policy=retry_policy, start_to_close_timeout=STATUS_WRITE_TIMEOUT, ) - except BaseException: # noqa: S110 # NOSONAR - the report of a report is where this stops - # Deliberately silent: the channel this was trying to reach is the thing that logs, so - # there is nowhere left to say it, and an import that worked must not fail over it. - pass + except BaseException as error: # NOSONAR - the report of a report is where this stops + if is_cancelled_exception(error): + raise + # Deliberately silent otherwise: the channel this was trying to reach is the thing that + # logs, so there is nowhere left to say it, and an import that worked must not fail + # over it. # ----------------------------------------------------------- side-effect steps @@ -418,12 +426,13 @@ class ImportModel: retry_policy=retry_policy, start_to_close_timeout=RETENTION_TIMEOUT, ) - except BaseException: # noqa: S110 # NOSONAR - retention is not a precondition of an import - # Dropped, not reported: the activity's own failure is already in the event history, - # and a bucket policy the importer cannot set is an operational problem rather than a - # bad import. Failing a valid model import over a lifecycle API would be the wrong - # trade. - pass + except BaseException as error: # NOSONAR - retention is not a precondition of an import + if is_cancelled_exception(error): + raise + # Dropped, not reported otherwise: the activity's own failure is already in the event + # history, and a bucket policy the importer cannot set is an operational problem rather + # than a bad import. Failing a valid model import over a lifecycle API would be the + # wrong trade. @staticmethod async def _cleanup(base: dict[str, Any]) -> dict[str, Any]: @@ -435,5 +444,7 @@ class ImportModel: retry_policy=retry_policy, start_to_close_timeout=CLEANUP_TIMEOUT, ) - except BaseException: # NOSONAR - cleanup never replaces the failure that brought us here + except BaseException as error: # NOSONAR - cleanup never replaces the failure that brought us here + if is_cancelled_exception(error): + raise return {'cleaned': False} diff --git a/tests/laborious/activities/test_model_import_bundle.py b/tests/laborious/activities/test_model_import_bundle.py index 8a41ddb..e4940b1 100644 --- a/tests/laborious/activities/test_model_import_bundle.py +++ b/tests/laborious/activities/test_model_import_bundle.py @@ -72,24 +72,25 @@ def minio_client() -> MagicMock: def import_activity(tmp_path: Path, minio_client: MagicMock) -> ModelImport: repository = MagicMock() repository.minio_client = minio_client - with patch('laborious.activities.model_import.create_engine', return_value=FakeStatusEngine()): - activity = ModelImport( - status_config={ - 'host': 'h', - 'port': 5432, - 'dbname': 'sientia-core-mlops-bff', - 'user': 'u', - 'password': 'p', - 'schema': 'public', - }, - import_config=import_config(tmp_path), - minio_repository=repository, - mlflow_repository=MagicMock(), - mongo_repository=AsyncMock(), - logger=MagicMock(), - notification_handler=MagicMock(), - metrics_controller=AsyncMock(), - ) + activity = ModelImport( + status_config={ + 'host': 'h', + 'port': 5432, + 'dbname': 'sientia-core-mlops-bff', + 'user': 'u', + 'password': 'p', + 'schema': 'public', + }, + import_config=import_config(tmp_path), + minio_repository=repository, + mlflow_repository=MagicMock(), + mongo_repository=AsyncMock(), + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=AsyncMock(), + ) + # `status_engine` is created lazily on first access; pre-seed it with the recording double. + activity._status_engine = FakeStatusEngine() activity.emit_metric = AsyncMock() activity.send_notification_async = AsyncMock() return activity diff --git a/tests/laborious/activities/test_model_import_provisioning.py b/tests/laborious/activities/test_model_import_provisioning.py index e025898..3054ada 100644 --- a/tests/laborious/activities/test_model_import_provisioning.py +++ b/tests/laborious/activities/test_model_import_provisioning.py @@ -100,36 +100,37 @@ def import_activity(tmp_path: Path, mlflow_client: MagicMock, mongo: AsyncMock) minio_repository = MagicMock() mlflow_repository = MagicMock() mlflow_repository.client = mlflow_client - with patch('laborious.activities.model_import.create_engine', return_value=FakeStatusEngine()): - activity = ModelImport( - status_config={ - 'host': 'h', - 'port': 5432, - 'dbname': 'sientia-core-mlops-bff', - 'user': 'u', - 'password': 'p', - 'schema': 'public', - }, - import_config={ - 'password_key': None, - 'bucket': 'sientia', - 'prefix': 'imported_models/', - 'work_dir': str(tmp_path), - 'max_object_bytes': 1024, - 'max_entries': 10, - 'max_uncompressed_bytes': 4096, - 'max_compression_ratio': 200.0, - 'models_collection': 'models', - 'counters_collection': 'model_id_counters', - 'retention_days': 7, - }, - minio_repository=minio_repository, - mlflow_repository=mlflow_repository, - mongo_repository=mongo, - logger=MagicMock(), - notification_handler=MagicMock(), - metrics_controller=AsyncMock(), - ) + activity = ModelImport( + status_config={ + 'host': 'h', + 'port': 5432, + 'dbname': 'sientia-core-mlops-bff', + 'user': 'u', + 'password': 'p', + 'schema': 'public', + }, + import_config={ + 'password_key': None, + 'bucket': 'sientia', + 'prefix': 'imported_models/', + 'work_dir': str(tmp_path), + 'max_object_bytes': 1024, + 'max_entries': 10, + 'max_uncompressed_bytes': 4096, + 'max_compression_ratio': 200.0, + 'models_collection': 'models', + 'counters_collection': 'model_id_counters', + 'retention_days': 7, + }, + minio_repository=minio_repository, + mlflow_repository=mlflow_repository, + mongo_repository=mongo, + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=AsyncMock(), + ) + # `status_engine` is created lazily on first access; pre-seed it with the recording double. + activity._status_engine = FakeStatusEngine() activity.emit_metric = AsyncMock() activity.send_notification_async = AsyncMock() return activity diff --git a/tests/laborious/activities/test_model_import_status.py b/tests/laborious/activities/test_model_import_status.py index 893d945..684c715 100644 --- a/tests/laborious/activities/test_model_import_status.py +++ b/tests/laborious/activities/test_model_import_status.py @@ -164,17 +164,19 @@ def test_build_update_with_no_casts_or_extra_where_renders_a_bare_update(): def build_activity(engine: FakeStatusEngine) -> ModelImport: """A `ModelImport` whose only connection is the recording engine double.""" - with patch('laborious.activities.model_import.create_engine', return_value=engine): - activity = ModelImport( - status_config=STATUS_CONFIG, - import_config=IMPORT_CONFIG, - minio_repository=MagicMock(), - mlflow_repository=MagicMock(), - mongo_repository=AsyncMock(), - logger=MagicMock(), - notification_handler=MagicMock(), - metrics_controller=AsyncMock(), - ) + activity = ModelImport( + status_config=STATUS_CONFIG, + import_config=IMPORT_CONFIG, + minio_repository=MagicMock(), + mlflow_repository=MagicMock(), + mongo_repository=AsyncMock(), + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=AsyncMock(), + ) + # `status_engine` is created lazily on first access; setting the private attribute directly + # pre-seeds it with the recording double instead of exercising the real `create_engine`. + activity._status_engine = engine activity.emit_metric = AsyncMock() activity.send_notification_async = AsyncMock() return activity @@ -983,8 +985,62 @@ async def test_the_engine_is_disposed_on_shutdown(import_activity: ModelImport, @mark.asyncio async def test_close_is_a_no_op_without_an_engine(import_activity: ModelImport, engine): """A partially-constructed instance (or a second close) must not blow up on a missing engine.""" - del import_activity.status_engine + import_activity._status_engine = None import_activity.close() assert engine.disposed is False + + +# ------------------------------------------------------------- the lazy status engine + + +def test_no_engine_is_created_when_status_engine_is_never_accessed(): + """A worker pod that never runs an import activity never opens a pool against the BFF Postgres.""" + with patch('laborious.activities.model_import.create_engine') as create_engine_mock: + activity = ModelImport( + status_config=STATUS_CONFIG, + import_config=IMPORT_CONFIG, + minio_repository=MagicMock(), + mlflow_repository=MagicMock(), + mongo_repository=AsyncMock(), + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=AsyncMock(), + ) + + create_engine_mock.assert_not_called() + assert activity._status_engine is None + + +def test_first_access_creates_the_engine_and_later_accesses_reuse_it(): + with patch( + 'laborious.activities.model_import.create_engine', return_value=FakeStatusEngine() + ) as create_engine_mock: + activity = ModelImport( + status_config=STATUS_CONFIG, + import_config=IMPORT_CONFIG, + minio_repository=MagicMock(), + mlflow_repository=MagicMock(), + mongo_repository=AsyncMock(), + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=AsyncMock(), + ) + + first = activity.status_engine + second = activity.status_engine + + create_engine_mock.assert_called_once() + assert first is second + + +@mark.asyncio +async def test_close_never_accessed_the_engine_does_not_create_one(import_activity: ModelImport): + """`close()` must not force the lazy engine into existence just to dispose it.""" + import_activity._status_engine = None + + with patch('laborious.activities.model_import.create_engine') as create_engine_mock: + import_activity.close() + + create_engine_mock.assert_not_called() diff --git a/tests/laborious/utils/bundle/test_reader.py b/tests/laborious/utils/bundle/test_reader.py index e912ad5..da22105 100644 --- a/tests/laborious/utils/bundle/test_reader.py +++ b/tests/laborious/utils/bundle/test_reader.py @@ -437,7 +437,7 @@ def test_gate_5_wraps_an_os_error_during_unpacking(work_dir, reader, monkeypatch def _raise_os_error(*_args, **_kwargs): raise OSError('disk full') - monkeypatch.setattr('laborious.utils.bundle.reader.shutil.copyfileobj', _raise_os_error) + monkeypatch.setattr(zipfile.ZipExtFile, 'read', _raise_os_error) with pytest.raises(BundleGateError, match='could not be unpacked') as error: _open(reader, work_dir, payload) @@ -463,6 +463,52 @@ def test_extract_member_rechecks_containment_on_the_resolved_path(work_dir, read assert error.value.gate == 5 +def test_extract_member_rejects_a_stream_larger_than_its_declared_size(work_dir, reader, monkeypatch): + """A decompressed stream longer than the entry's declared `file_size` is rejected mid-write, + instead of trusting the central-directory metadata gate 4 already checked in aggregate. + + A single crafted DEFLATE chunk can make `ZipExtFile.read()` return far more bytes than + `file_size` declares before its own EOF/CRC bookkeeping catches up, so the stand-in here + forces exactly that response shape rather than fighting zipfile's own CRC validation.""" + root = work_dir / 'root' + root.mkdir() + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, 'w') as archive: + archive.writestr('payload.bin', b'x' * 10) + + call_count = {'n': 0} + + def _oversized_read(self, *_args, **_kwargs): + call_count['n'] += 1 + return b'x' * 1000 if call_count['n'] == 1 else b'' + + monkeypatch.setattr(zipfile.ZipExtFile, 'read', _oversized_read) + + with zipfile.ZipFile(io.BytesIO(buffer.getvalue())) as archive: + entry = archive.infolist()[0] + with pytest.raises(BundleGateError, match='more than its declared') as error: + reader._extract_member(archive, entry, root, root.resolve()) + + assert error.value.gate == 5 + + +def test_extract_member_writes_a_well_formed_member_in_full(work_dir, reader): + """A member whose decompressed size matches its declared `file_size` is extracted byte for + byte, exactly as before the byte-counting check was added.""" + root = work_dir / 'root' + root.mkdir() + payload = b'y' * 100 + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, 'w') as archive: + archive.writestr('payload.bin', payload) + + with zipfile.ZipFile(io.BytesIO(buffer.getvalue())) as archive: + entry = archive.infolist()[0] + reader._extract_member(archive, entry, root, root.resolve()) + + assert (root / 'payload.bin').read_bytes() == payload + + # ------------------------------------------------------------------- gate 6 diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index a6ea506..883da62 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -557,12 +557,15 @@ async def test_download_model_invalid_model_type(mlflow_repository): 'model_type', [('predict', 'prediction_model'), ('transform', 'data_model')] ) async def test_download_model_load_wrapper(mlflow, mlflow_repository, model_type, tmp_path): + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) mlflow_repository.dowload_artifacts = AsyncMock(return_value=str(tmp_path)) result = await mlflow_repository.download_model('test_model', {}, model_type[0], 'pyfunc', True) - mlflow_repository.dowload_artifacts.assert_called_once_with('test_model', {}, model_type[1]) + mlflow_repository.dowload_artifacts.assert_called_once_with( + 'test_model', {}, model_type[1], run_id='run-123' + ) mlflow.pyfunc.load_model.assert_called_once_with( mlflow_repository.dowload_artifacts.return_value @@ -576,13 +579,16 @@ async def test_download_model_load_wrapper(mlflow, mlflow_repository, model_type @pytest.mark.asyncio async def test_download_model_predict(mlflow_repository): + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) mlflow_repository.load_predict_model = AsyncMock(return_value=('model', 'local_dir')) mlflow_repository.load_transform_model = AsyncMock() result = await mlflow_repository.download_model('test_model', {}, 'predict', 'pyfunc', False) - mlflow_repository.load_predict_model.assert_called_once_with('test_model', {}, 'pyfunc') + mlflow_repository.load_predict_model.assert_called_once_with( + 'test_model', {}, 'pyfunc', run_id='run-123' + ) mlflow_repository.load_transform_model.assert_not_called() assert result == ('model', 'local_dir') @@ -590,6 +596,7 @@ async def test_download_model_predict(mlflow_repository): @pytest.mark.asyncio async def test_download_model_transform(mlflow_repository): + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) mlflow_repository.load_predict_model = AsyncMock() mlflow_repository.load_transform_model = AsyncMock(return_value=('model', 'local_dir')) @@ -597,7 +604,9 @@ async def test_download_model_transform(mlflow_repository): result = await mlflow_repository.download_model('test_model', {}, 'transform', 'pyfunc', False) mlflow_repository.load_predict_model.assert_not_called() - mlflow_repository.load_transform_model.assert_called_once_with('test_model', {}, 'pyfunc') + mlflow_repository.load_transform_model.assert_called_once_with( + 'test_model', {}, 'pyfunc', run_id='run-123' + ) assert result == ('model', 'local_dir') @@ -616,10 +625,11 @@ async def test_download_model_prepares_the_code_dir_first( calls.append('code') return 'code-artifact-dir' - async def record_load(*_a): + async def record_load(*_a, **_k): calls.append('load') return 'model', None + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.download_artifact_code_dir = AsyncMock(side_effect=record_code_dir) loader = AsyncMock(side_effect=record_load) mlflow_repository.load_predict_model = loader @@ -628,7 +638,7 @@ async def test_download_model_prepares_the_code_dir_first( result = await mlflow_repository.download_model('test_model', {}, model_type, 'sklearn', False) mlflow_repository.download_artifact_code_dir.assert_called_once_with( - 'test_model', artifact_name, 'sklearn', {} + 'test_model', artifact_name, 'sklearn', {}, run_id='run-123' ) assert calls == ['code', 'load'] assert result == ('model', 'code-artifact-dir') @@ -637,6 +647,7 @@ async def test_download_model_prepares_the_code_dir_first( @pytest.mark.asyncio async def test_download_model_wrapper_returns_the_code_artifact_dir(mlflow, mlflow_repository): """With a `code` directory downloaded, the wrapper branch returns it over its own download.""" + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.download_artifact_code_dir = AsyncMock(return_value='code-artifact-dir') mlflow_repository.dowload_artifacts = AsyncMock(return_value='wrapper-dir') @@ -647,6 +658,42 @@ async def test_download_model_wrapper_returns_the_code_artifact_dir(mlflow, mlfl assert artifact_dir == 'code-artifact-dir' +@pytest.mark.asyncio +async def test_download_model_resolves_run_id_once_non_wrapper(mlflow_repository): + """The `run_id` `download_model` resolves once is the same one every downstream call uses.""" + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) + mlflow_repository.load_predict_model = AsyncMock(return_value=('model', None)) + + await mlflow_repository.download_model('test_model', {}, 'predict', 'sklearn', False) + + mlflow_repository.get_model_run_id.assert_called_once_with(model_name='test_model', stage='Production') + mlflow_repository.download_artifact_code_dir.assert_called_once_with( + 'test_model', 'prediction_model', 'sklearn', {}, run_id='run-123' + ) + mlflow_repository.load_predict_model.assert_called_once_with( + 'test_model', {}, 'sklearn', run_id='run-123' + ) + + +@pytest.mark.asyncio +async def test_download_model_resolves_run_id_once_wrapper(mlflow, mlflow_repository, tmp_path): + """Same single resolution, for the `load_wrapper=True` branch.""" + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') + mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) + mlflow_repository.dowload_artifacts = AsyncMock(return_value=str(tmp_path)) + + await mlflow_repository.download_model('test_model', {}, 'predict', 'pyfunc', True) + + mlflow_repository.get_model_run_id.assert_called_once_with(model_name='test_model', stage='Production') + mlflow_repository.download_artifact_code_dir.assert_called_once_with( + 'test_model', 'prediction_model', 'pyfunc', {}, run_id='run-123' + ) + mlflow_repository.dowload_artifacts.assert_called_once_with( + 'test_model', {}, 'prediction_model', run_id='run-123' + ) + + def test_detect_and_parse_datetime_index_empty(mlflow_repository): input_data = DataFrame() response = mlflow_repository.detect_and_parse_datetime_index(input_data, metadata['metadata']) @@ -1965,6 +2012,7 @@ async def test_load_transform_model_flavor_failure_propagates( @patch(JOBLIB_PATH) @pytest.mark.asyncio async def test_download_model_wrapper_failure_propagates(joblib_mock, mlflow, mlflow_repository): + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) mlflow_repository.dowload_artifacts = AsyncMock() loader_error = mlflow_lib.exceptions.MlflowException('wrapper boom') @@ -1983,6 +2031,7 @@ async def test_download_model_wrapper_failure_propagates(joblib_mock, mlflow, ml async def test_download_model_wrapper_failure_emits_error_metric( joblib_mock, mlflow, mlflow_repository ): + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) mlflow_repository.dowload_artifacts = AsyncMock() loader_error = mlflow_lib.exceptions.MlflowException('wrapper boom') @@ -2503,13 +2552,14 @@ async def test_download_model_delegates_every_flavor_to_the_loaders( mlflow_repository, model_type, loader ): """`download_model` owns no flavor branch: joblib delegates like the other two.""" + mlflow_repository.get_model_run_id = MagicMock(return_value='run-123') mlflow_repository.download_artifact_code_dir = AsyncMock(return_value=None) delegate = AsyncMock(return_value=('model', 'local-dir')) setattr(mlflow_repository, loader, delegate) result = await mlflow_repository.download_model('test_model', {}, model_type, 'joblib') - delegate.assert_called_once_with('test_model', {}, 'joblib') + delegate.assert_called_once_with('test_model', {}, 'joblib', run_id='run-123') assert result == ('model', 'local-dir') diff --git a/tests/laborious/workflows/test_import_model.py b/tests/laborious/workflows/test_import_model.py index 1e68237..1bc0266 100644 --- a/tests/laborious/workflows/test_import_model.py +++ b/tests/laborious/workflows/test_import_model.py @@ -12,7 +12,7 @@ workflow is: a sequence of activity invocations with policies attached. import asyncio from datetime import timedelta from types import SimpleNamespace -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch from pytest import fixture, mark, raises from sientia_do.temporal.policies import retry_policy @@ -725,6 +725,95 @@ async def test_reporting_the_report_is_where_it_stops(import_workflow: ImportMod assert result['run_id'] == RUN_ID +# --------------------------------------------------- cancellation is never swallowed + + +def cancelling_activities(): + """Patch the workflow module's `workflow` object so every activity call raises cancellation.""" + patcher = patch('laborious.workflows.import_model.workflow') + workflow_mock = patcher.start() + workflow_mock.execute_activity_method = AsyncMock(side_effect=asyncio.CancelledError()) + return patcher + + +BASE = {'metadata': {}, 'import_run_id': 77, 'workflow_id': WORKFLOW_ID} + + +@mark.asyncio +async def test_a_cancelled_hint_propagates_instead_of_being_reported(import_workflow: ImportModel): + patcher = cancelling_activities() + try: + with raises(asyncio.CancelledError): + await import_workflow._record_hint( + BASE, Activities.record_import_status, {'step': ImportStep.DOWNLOAD.value} + ) + finally: + patcher.stop() + + +@mark.asyncio +async def test_a_cancelled_terminal_write_propagates_instead_of_being_reported( + import_workflow: ImportModel, +): + patcher = cancelling_activities() + try: + with raises(asyncio.CancelledError): + await import_workflow._record_terminal( + BASE, + succeeded=True, + step=ImportStep.MODEL_DOCUMENT.value, + gate=None, + source={}, + cleanup_failed=False, + ) + finally: + patcher.stop() + + +@mark.asyncio +async def test_a_cancelled_pipeline_failure_report_propagates(import_workflow: ImportModel): + patcher = cancelling_activities() + try: + with raises(asyncio.CancelledError): + await import_workflow._report_failure( + BASE, RuntimeError('boom'), step=ImportStep.REGISTRATION.value, gate=None + ) + finally: + patcher.stop() + + +@mark.asyncio +async def test_a_cancelled_status_write_failure_report_propagates(import_workflow: ImportModel): + patcher = cancelling_activities() + try: + with raises(asyncio.CancelledError): + await import_workflow._report_status_write_failure(BASE, RuntimeError('db down')) + finally: + patcher.stop() + + +@mark.asyncio +async def test_a_cancelled_retention_check_propagates(import_workflow: ImportModel): + patcher = cancelling_activities() + try: + with raises(asyncio.CancelledError): + await import_workflow._ensure_retention(BASE, valid_input()) + finally: + patcher.stop() + + +@mark.asyncio +async def test_a_cancelled_cleanup_propagates_instead_of_returning_not_cleaned( + import_workflow: ImportModel, +): + patcher = cancelling_activities() + try: + with raises(asyncio.CancelledError): + await import_workflow._cleanup(BASE) + finally: + patcher.stop() + + # --------------------------------------------------------------- the failure report From 7c45dbb407dc81e71fbdbba44d5205d14203a893 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 10 Sep 2026 09:23:31 -0300 Subject: [PATCH 48/64] QTZPOC-20 QTZPOC-20: Apply minor formatting adjustments to improve code readability. --- laborious/workflows/import_model.py | 4 +++- tests/laborious/utils/bundle/test_reader.py | 4 +++- tests/laborious/utils/repository/test_model_repository.py | 8 ++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/laborious/workflows/import_model.py b/laborious/workflows/import_model.py index 0d8ea0d..b85df35 100644 --- a/laborious/workflows/import_model.py +++ b/laborious/workflows/import_model.py @@ -444,7 +444,9 @@ class ImportModel: retry_policy=retry_policy, start_to_close_timeout=CLEANUP_TIMEOUT, ) - except BaseException as error: # NOSONAR - cleanup never replaces the failure that brought us here + except ( + BaseException + ) as error: # NOSONAR - cleanup never replaces the failure that brought us here if is_cancelled_exception(error): raise return {'cleaned': False} diff --git a/tests/laborious/utils/bundle/test_reader.py b/tests/laborious/utils/bundle/test_reader.py index da22105..5db00ec 100644 --- a/tests/laborious/utils/bundle/test_reader.py +++ b/tests/laborious/utils/bundle/test_reader.py @@ -463,7 +463,9 @@ def test_extract_member_rechecks_containment_on_the_resolved_path(work_dir, read assert error.value.gate == 5 -def test_extract_member_rejects_a_stream_larger_than_its_declared_size(work_dir, reader, monkeypatch): +def test_extract_member_rejects_a_stream_larger_than_its_declared_size( + work_dir, reader, monkeypatch +): """A decompressed stream longer than the entry's declared `file_size` is rejected mid-write, instead of trusting the central-directory metadata gate 4 already checked in aggregate. diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 883da62..2014bdd 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -667,7 +667,9 @@ async def test_download_model_resolves_run_id_once_non_wrapper(mlflow_repository await mlflow_repository.download_model('test_model', {}, 'predict', 'sklearn', False) - mlflow_repository.get_model_run_id.assert_called_once_with(model_name='test_model', stage='Production') + mlflow_repository.get_model_run_id.assert_called_once_with( + model_name='test_model', stage='Production' + ) mlflow_repository.download_artifact_code_dir.assert_called_once_with( 'test_model', 'prediction_model', 'sklearn', {}, run_id='run-123' ) @@ -685,7 +687,9 @@ async def test_download_model_resolves_run_id_once_wrapper(mlflow, mlflow_reposi await mlflow_repository.download_model('test_model', {}, 'predict', 'pyfunc', True) - mlflow_repository.get_model_run_id.assert_called_once_with(model_name='test_model', stage='Production') + mlflow_repository.get_model_run_id.assert_called_once_with( + model_name='test_model', stage='Production' + ) mlflow_repository.download_artifact_code_dir.assert_called_once_with( 'test_model', 'prediction_model', 'pyfunc', {}, run_id='run-123' ) From 9e0da03717e83fab553ade4ca65b1c6cc63b2c81 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 10 Sep 2026 09:34:06 -0300 Subject: [PATCH 49/64] QTZPOC-20 QTZPOC-20: Refactor test calls by extracting inline arguments into local variables for improved readability. --- tests/laborious/utils/bundle/test_reader.py | 3 ++- tests/laborious/workflows/test_import_model.py | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/laborious/utils/bundle/test_reader.py b/tests/laborious/utils/bundle/test_reader.py index 5db00ec..73d05b8 100644 --- a/tests/laborious/utils/bundle/test_reader.py +++ b/tests/laborious/utils/bundle/test_reader.py @@ -486,10 +486,11 @@ def test_extract_member_rejects_a_stream_larger_than_its_declared_size( monkeypatch.setattr(zipfile.ZipExtFile, 'read', _oversized_read) + resolved_root = root.resolve() with zipfile.ZipFile(io.BytesIO(buffer.getvalue())) as archive: entry = archive.infolist()[0] with pytest.raises(BundleGateError, match='more than its declared') as error: - reader._extract_member(archive, entry, root, root.resolve()) + reader._extract_member(archive, entry, root, resolved_root) assert error.value.gate == 5 diff --git a/tests/laborious/workflows/test_import_model.py b/tests/laborious/workflows/test_import_model.py index 1bc0266..a93864e 100644 --- a/tests/laborious/workflows/test_import_model.py +++ b/tests/laborious/workflows/test_import_model.py @@ -773,10 +773,11 @@ async def test_a_cancelled_terminal_write_propagates_instead_of_being_reported( @mark.asyncio async def test_a_cancelled_pipeline_failure_report_propagates(import_workflow: ImportModel): patcher = cancelling_activities() + error = RuntimeError('boom') try: with raises(asyncio.CancelledError): await import_workflow._report_failure( - BASE, RuntimeError('boom'), step=ImportStep.REGISTRATION.value, gate=None + BASE, error, step=ImportStep.REGISTRATION.value, gate=None ) finally: patcher.stop() @@ -785,9 +786,10 @@ async def test_a_cancelled_pipeline_failure_report_propagates(import_workflow: I @mark.asyncio async def test_a_cancelled_status_write_failure_report_propagates(import_workflow: ImportModel): patcher = cancelling_activities() + error = RuntimeError('db down') try: with raises(asyncio.CancelledError): - await import_workflow._report_status_write_failure(BASE, RuntimeError('db down')) + await import_workflow._report_status_write_failure(BASE, error) finally: patcher.stop() @@ -795,9 +797,10 @@ async def test_a_cancelled_status_write_failure_report_propagates(import_workflo @mark.asyncio async def test_a_cancelled_retention_check_propagates(import_workflow: ImportModel): patcher = cancelling_activities() + input_data = valid_input() try: with raises(asyncio.CancelledError): - await import_workflow._ensure_retention(BASE, valid_input()) + await import_workflow._ensure_retention(BASE, input_data) finally: patcher.stop() From e600c7922e8635beae463d7ec12679f0d586de7a Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 10 Sep 2026 09:39:21 -0300 Subject: [PATCH 50/64] QTZPOC-20 QTZPOC-20: Remove redundant NOSONAR comments across multiple files. --- laborious/utils/connectors_config.py | 2 +- laborious/utils/models/import_errors.py | 8 ++++---- laborious/utils/repository/model_repository.py | 4 ++-- laborious/utils/repository/opc_repository.py | 6 +++--- laborious/worker/worker.py | 2 +- laborious/workflows/import_model.py | 14 +++++++------- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/laborious/utils/connectors_config.py b/laborious/utils/connectors_config.py index 90f11c8..01e29e1 100644 --- a/laborious/utils/connectors_config.py +++ b/laborious/utils/connectors_config.py @@ -25,7 +25,7 @@ def build_mlflow_config() -> dict[str, Any]: 'host': getenv('MLFLOW_HOST', 'http://localhost'), 'port': int(getenv('MLFLOW_PORT', '5080')), 'username': getenv('MLFLOW_USERNAME', 'aignosi'), - 'password': getenv('MLFLOW_PASSWORD', 'aignosi'), # NOSONAR + 'password': getenv('MLFLOW_PASSWORD', 'aignosi'), } diff --git a/laborious/utils/models/import_errors.py b/laborious/utils/models/import_errors.py index 96ca875..c8c91e5 100644 --- a/laborious/utils/models/import_errors.py +++ b/laborious/utils/models/import_errors.py @@ -342,7 +342,7 @@ def summarize_failure( """ if exception is not None: cause = _root_cause(exception) - return _type_name(cause), _one_line(str(cause)) # NOSONAR + return _type_name(cause), _one_line(str(cause)) return error_type or _UNKNOWN_ERROR_TYPE, _one_line(detail_text or '') @@ -361,8 +361,8 @@ def describe_failure_chain(exception: BaseException) -> tuple[str, str]: """ links: list[str] = [] for candidate in walk_cause_chain(exception): - kind = _type_name(candidate) # NOSONAR - message = _one_line(str(candidate)) # NOSONAR + kind = _type_name(candidate) + message = _one_line(str(candidate)) # `ApplicationError.__str__` already prefixes its own type, and `MlflowException: # MlflowException: …` reads like two failures rather than one. if not message: @@ -422,7 +422,7 @@ def failure_location(exception: BaseException | None) -> tuple[str | None, int | if exception is None: return None, None for candidate in walk_cause_chain(exception): - details = getattr(candidate, 'details', None) # NOSONAR + details = getattr(candidate, 'details', None) if details and isinstance(details[0], dict) and 'step' in details[0]: location = details[0] gate = location.get('gate') diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 118b69e..4d6fcc8 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -1150,7 +1150,7 @@ class MLFlowRepository(SientiaMonitoring): prediction_data.index = input_index # Merge prediction data with retrain_dataset on index - prediction_data = pd.merge( # NOSONAR + prediction_data = pd.merge( retrain_dataset, prediction_data, left_index=True, right_index=True, how='left' ) @@ -1275,7 +1275,7 @@ class MLFlowRepository(SientiaMonitoring): # Aligns data with treated data indexes to get target variable aligned_data = data.loc[treated_data.index] aligned_series = aligned_data[target_name] - retrain_dataset = pd.merge( # NOSONAR + retrain_dataset = pd.merge( treated_data, aligned_series, left_index=True, right_index=True ) else: diff --git a/laborious/utils/repository/opc_repository.py b/laborious/utils/repository/opc_repository.py index 8ea4007..656ec2e 100644 --- a/laborious/utils/repository/opc_repository.py +++ b/laborious/utils/repository/opc_repository.py @@ -79,7 +79,7 @@ def _opc_authentication_token_str(client: Client | None) -> str: if tok is None: return 'unknown' return str(tok) - except Exception: # NOSONAR - best-effort label for logs/metrics, must not fail loud + except Exception: # best-effort label for logs/metrics, must not fail loud return 'unknown' @@ -94,8 +94,8 @@ def _opc_status_from_exception(exc: BaseException) -> str: str: Status class name or generic Python exception name. """ for candidate in walk_cause_chain(exc): - if isinstance(candidate, UaStatusCodeError): # NOSONAR - return type(candidate).__name__ # NOSONAR + if isinstance(candidate, UaStatusCodeError): + return type(candidate).__name__ return type(exc).__name__ diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index 09cc2c8..231cc8f 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -305,7 +305,7 @@ async def main(): # This will run the workers and wait for them to complete. # If an exception occurs in any of the worker handlers, it will be propagated here. await asyncio.gather(*handlers) - except BaseException as e: # NOSONAR + except BaseException as e: logger.custom_error(f'An unhandled exception occurred: {e}', metadata) exit_code = 1 finally: diff --git a/laborious/workflows/import_model.py b/laborious/workflows/import_model.py index b85df35..4c63c41 100644 --- a/laborious/workflows/import_model.py +++ b/laborious/workflows/import_model.py @@ -227,7 +227,7 @@ class ImportModel: 'version': version['version'], 'document_id': document.get('id'), } - except BaseException as error: # NOSONAR - classified once below, then re-raised untouched + except BaseException as error: # classified once below, then re-raised untouched located_step, located_gate = failure_location(error) step = located_step or step gate = located_gate @@ -309,7 +309,7 @@ class ImportModel: retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY, start_to_close_timeout=STATUS_WRITE_TIMEOUT, ) - except BaseException as error: # NOSONAR - a lost hint must not fail a working import + except BaseException as error: # a lost hint must not fail a working import if is_cancelled_exception(error): raise await self._report_status_write_failure(base, error) @@ -346,7 +346,7 @@ class ImportModel: ) except ( BaseException - ) as error: # NOSONAR - the outcome stands even when it cannot be written + ) as error: # the outcome stands even when it cannot be written if is_cancelled_exception(error): raise await self._report_status_write_failure( @@ -382,7 +382,7 @@ class ImportModel: retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY, start_to_close_timeout=STATUS_WRITE_TIMEOUT, ) - except BaseException as error: # NOSONAR - the report of a report is where this stops + except BaseException as error: # the report of a report is where this stops if is_cancelled_exception(error): raise @@ -400,7 +400,7 @@ class ImportModel: retry_policy=retry_policy, start_to_close_timeout=STATUS_WRITE_TIMEOUT, ) - except BaseException as error: # NOSONAR - the report of a report is where this stops + except BaseException as error: # the report of a report is where this stops if is_cancelled_exception(error): raise # Deliberately silent otherwise: the channel this was trying to reach is the thing that @@ -426,7 +426,7 @@ class ImportModel: retry_policy=retry_policy, start_to_close_timeout=RETENTION_TIMEOUT, ) - except BaseException as error: # NOSONAR - retention is not a precondition of an import + except BaseException as error: # retention is not a precondition of an import if is_cancelled_exception(error): raise # Dropped, not reported otherwise: the activity's own failure is already in the event @@ -446,7 +446,7 @@ class ImportModel: ) except ( BaseException - ) as error: # NOSONAR - cleanup never replaces the failure that brought us here + ) as error: # cleanup never replaces the failure that brought us here if is_cancelled_exception(error): raise return {'cleaned': False} From bda83e7943384ad12b461b6de32417b69779da20 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 10 Sep 2026 09:43:04 -0300 Subject: [PATCH 51/64] QTZPOC-20 QTZPOC-20: Remove redundant parentheses from BaseException catch clauses --- laborious/workflows/import_model.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/laborious/workflows/import_model.py b/laborious/workflows/import_model.py index 4c63c41..711aef4 100644 --- a/laborious/workflows/import_model.py +++ b/laborious/workflows/import_model.py @@ -344,9 +344,7 @@ class ImportModel: start_to_close_timeout=STATUS_WRITE_TIMEOUT, schedule_to_close_timeout=STATUS_WRITE_ENVELOPE, ) - except ( - BaseException - ) as error: # the outcome stands even when it cannot be written + except BaseException as error: # the outcome stands even when it cannot be written if is_cancelled_exception(error): raise await self._report_status_write_failure( @@ -444,9 +442,7 @@ class ImportModel: retry_policy=retry_policy, start_to_close_timeout=CLEANUP_TIMEOUT, ) - except ( - BaseException - ) as error: # cleanup never replaces the failure that brought us here + except BaseException as error: # cleanup never replaces the failure that brought us here if is_cancelled_exception(error): raise return {'cleaned': False} From 512a49f78f23b0e534ecd434e517d5159a3c1dee Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 10 Sep 2026 11:46:29 -0300 Subject: [PATCH 52/64] QTZPOC-20 QTZPOC-20: Enable code_paths for sklearn models; improve worker shutdown handling The MLFlowRepository now resolves and forwards code_paths to mlflow.sklearn.log_model, ensuring custom code dependencies for sklearn models are preserved and correctly reloaded. A new integration test validates this end-to-end functionality. Additionally, worker exception handling is refined to distinguish between deliberate shutdowns (SystemExit, KeyboardInterrupt) and unhandled exceptions, preventing unnecessary error logging for expected exits. --- .../utils/repository/model_repository.py | 18 ++- laborious/worker/worker.py | 9 +- .../utils/repository/test_model_repository.py | 26 ++++- .../test_sklearn_flavor_retrain_e2e.py | 104 ++++++++++++++++++ 4 files changed, 147 insertions(+), 10 deletions(-) create mode 100644 tests/laborious/utils/repository/test_sklearn_flavor_retrain_e2e.py diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 4d6fcc8..8d3d9f6 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -1330,7 +1330,7 @@ class MLFlowRepository(SientiaMonitoring): return code_dir if path.isdir(code_dir) else None def resolve_code_paths(self, artifact_path: str | None, metadata: dict) -> list[str]: - """Resolve the code dependencies to forward when re-logging a pyfunc model. + """Resolve the code dependencies to forward when re-logging a model. MLflow flattens whatever was given as `code_paths` into `/code/`, so the layout differs per model: some ship a `utils` package, others a @@ -1390,15 +1390,21 @@ class MLFlowRepository(SientiaMonitoring): core_labels = self.get_core_labels(metadata, operation_type='log_model') start_time = time.time() + code_paths = ( + self.resolve_code_paths(model_data.get('artifact_path'), metadata) + if flavor in ('sklearn', 'pyfunc') + else None + ) + try: if flavor == 'sklearn': - mlflow.sklearn.log_model(model, model_type) + self.debug(f'Code paths: {code_paths}', metadata) + + mlflow.sklearn.log_model(model, model_type, code_paths=code_paths or None) elif flavor == 'pyfunc': - code_path = self.resolve_code_paths(model_data.get('artifact_path'), metadata) + self.debug(f'Code paths: {code_paths}', metadata) - self.debug(f'Code path: {code_path}', metadata) - - model.store_model(artifact_path=model_type, code_path=code_path, to_disk=False) + model.store_model(artifact_path=model_type, code_path=code_paths, to_disk=False) self.debug('Model uploaded successfully', metadata) elif flavor == 'joblib': diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index 231cc8f..e585a2d 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -305,7 +305,14 @@ async def main(): # This will run the workers and wait for them to complete. # If an exception occurs in any of the worker handlers, it will be propagated here. await asyncio.gather(*handlers) - except BaseException as e: + except (SystemExit, KeyboardInterrupt) as e: + # Deliberate shutdown (Ctrl+C, a handler's own sys.exit) — not a failure to log as one. + # `finally` below calls sys.exit(exit_code) unconditionally, which supersedes this raise + # and becomes the exception that actually propagates — the raise is here so this branch + # still honors the reraise-on-SystemExit contract, not because it changes what exits. + exit_code = e.code if isinstance(e, SystemExit) and isinstance(e.code, int) else 0 + raise + except Exception as e: logger.custom_error(f'An unhandled exception occurred: {e}', metadata) exit_code = 1 finally: diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 2014bdd..0311ef1 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -1131,7 +1131,9 @@ async def test_log_model_sklearn(mlflow, mlflow_repository): await mlflow_repository.log_model( model_data, 'sklearn', 'prediction_model', metadata['metadata'] ) - mlflow.sklearn.log_model.assert_called_once_with(model_data['model'], 'prediction_model') + mlflow.sklearn.log_model.assert_called_once_with( + model_data['model'], 'prediction_model', code_paths=None + ) mlflow_repository.observe_lag.assert_called_once_with(ANY, metrics.MODEL_WRITE_LAG, ANY) mlflow_repository.emit_metric.assert_called_once_with( @@ -1139,6 +1141,22 @@ async def test_log_model_sklearn(mlflow, mlflow_repository): ) +@pytest.mark.asyncio +async def test_log_model_sklearn_forwards_code_paths(mlflow, mlflow_repository, tmp_path): + code_dir = build_code_dir(tmp_path) + model_data = {'model': MagicMock(), 'artifact_path': str(tmp_path)} + + await mlflow_repository.log_model( + model_data, 'sklearn', 'prediction_model', metadata['metadata'] + ) + + mlflow.sklearn.log_model.assert_called_once_with( + model_data['model'], + 'prediction_model', + code_paths=[str(Path(code_dir) / 'probe_flavor_model.py')], + ) + + def test_resolve_code_paths_package_layout(tmp_path, mlflow_repository): code_dir = tmp_path / 'code' (code_dir / 'utils').mkdir(parents=True) @@ -2489,7 +2507,9 @@ async def test_log_model_sklearn_does_not_dump_joblib(joblib_mock, mlflow, mlflo model_data, 'sklearn', 'prediction_model', metadata['metadata'] ) - mlflow.sklearn.log_model.assert_called_once_with(model_data['model'], 'prediction_model') + mlflow.sklearn.log_model.assert_called_once_with( + model_data['model'], 'prediction_model', code_paths=None + ) joblib_mock.dump.assert_not_called() mlflow.log_artifact.assert_not_called() @@ -2617,7 +2637,7 @@ async def test_retrain_model_joblib_reaches_joblib_write_branch( assert joblib_mock.dump.call_args.kwargs == {'compress': 3} mlflow.log_artifact.assert_any_call(ANY, artifact_path='prediction_model') mlflow.log_artifacts.assert_called_once_with(code_dir, artifact_path='prediction_model/code') - mlflow.sklearn.log_model.assert_called_once_with(ANY, 'data_model') + mlflow.sklearn.log_model.assert_called_once_with(ANY, 'data_model', code_paths=None) def test_module_defines_no_joblib_fallback_routine(): diff --git a/tests/laborious/utils/repository/test_sklearn_flavor_retrain_e2e.py b/tests/laborious/utils/repository/test_sklearn_flavor_retrain_e2e.py new file mode 100644 index 0000000..f1b77fe --- /dev/null +++ b/tests/laborious/utils/repository/test_sklearn_flavor_retrain_e2e.py @@ -0,0 +1,104 @@ +""" +Integration test: a `sklearn`-flavor model with a custom class survives a retrain round trip. + +Unlike the rest of `test_model_repository.py`, this test does not mock the `mlflow` module — +`resolve_code_paths` forwarding `code_paths=` to `mlflow.sklearn.log_model` is only proven by +letting the real MLflow file-based `code_paths` validation/copy machinery run, and by really +dropping and reloading the custom class module afterwards. A fully mocked `mlflow.sklearn` (as +used elsewhere in this file, and in `e2e/test_retrain_joblib_flavor.py`) would never exercise +that machinery at all. + +The Model Registry itself is not exercised here — MLflow's registry requires a database-backed +store, not a bare `file:` tracking URI — so this drives the write (`log_model`) and read +(`client.download_artifacts` + `mlflow.sklearn.load_model`) by run id directly, the same +building blocks `download_model` composes for the `sklearn` flavor once a Production run is +resolved. +""" + +import importlib +import sys +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import mlflow +import pytest + +from laborious.utils.repository.model_repository import MLFlowRepository + +CODE_MODULE_NAME = 'sklearn_flavor_retrain_probe' + +CODE_MODULE_SOURCE = '''"""Custom estimator that exists only inside the artifact's code/ directory.""" + + +class SklearnFlavorProbePredictor: + def __init__(self, offset): + self.offset = offset + + def fit(self, X, y=None): + return self + + def predict(self, X): + return [row[0] + self.offset for row in X] +''' + + +def _write_code_module(code_dir: Path) -> Path: + code_dir.mkdir(parents=True) + (code_dir / f'{CODE_MODULE_NAME}.py').write_text(CODE_MODULE_SOURCE) + return code_dir + + +def _import_fresh_probe_model(code_dir: Path, offset: int): + """Instantiate the custom class with its module on `sys.path`, then undo both.""" + sys.path.insert(0, str(code_dir)) + try: + module = importlib.import_module(CODE_MODULE_NAME) + return module.SklearnFlavorProbePredictor(offset) + finally: + sys.path.remove(str(code_dir)) + sys.modules.pop(CODE_MODULE_NAME, None) + importlib.invalidate_caches() + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_retrain_sklearn_model_preserves_code_dir_and_reloads(tmp_path): + """A sklearn retrain with a custom `code/` class keeps it and can reload it afterwards.""" + tracking_uri = f'file://{tmp_path / "mlruns"}' + mlflow.set_tracking_uri(tracking_uri) + mlflow.set_experiment('sklearn_flavor_retrain_e2e') + + repository = MLFlowRepository( + host=tracking_uri, + username='test', + password='test', + logger=MagicMock(), + notification_handler=MagicMock(), + metrics_controller=AsyncMock(), + ) + repository.emit_metric = AsyncMock() + repository.observe_lag = AsyncMock() + + source_code_dir = _write_code_module(tmp_path / 'source_artifact' / 'code') + model = _import_fresh_probe_model(source_code_dir, offset=10) + model_data = {'model': model, 'artifact_path': str(tmp_path / 'source_artifact')} + + with mlflow.start_run() as run: + await repository.log_model(model_data, 'sklearn', 'prediction_model', {'metadata': {}}) + run_id = run.info.run_id + + written_code_dir = Path( + repository.client.download_artifacts(run_id, 'prediction_model/code') + ) + assert (written_code_dir / f'{CODE_MODULE_NAME}.py').is_file() + + # The class must resolve from the artifact's written code/, not from anything left in memory. + sys.modules.pop(CODE_MODULE_NAME, None) + importlib.invalidate_caches() + + local_model_dir = repository.client.download_artifacts(run_id, 'prediction_model') + repository.prepend_artifact_code_dir(local_model_dir, 'sklearn', {}) + + reloaded = mlflow.sklearn.load_model(local_model_dir) + + assert reloaded.predict([[1], [2]]) == [11, 12] From 4cccc45c34cf887b346fe4101f272dafbe5b151e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 10 Sep 2026 11:50:51 -0300 Subject: [PATCH 53/64] QTZPOC-20 QTZPOC-20: Consolidate written_code_dir assignment into a single line in sklearn flavor retrain E2E test. --- .../utils/repository/test_sklearn_flavor_retrain_e2e.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/laborious/utils/repository/test_sklearn_flavor_retrain_e2e.py b/tests/laborious/utils/repository/test_sklearn_flavor_retrain_e2e.py index f1b77fe..965b61e 100644 --- a/tests/laborious/utils/repository/test_sklearn_flavor_retrain_e2e.py +++ b/tests/laborious/utils/repository/test_sklearn_flavor_retrain_e2e.py @@ -87,9 +87,7 @@ async def test_retrain_sklearn_model_preserves_code_dir_and_reloads(tmp_path): await repository.log_model(model_data, 'sklearn', 'prediction_model', {'metadata': {}}) run_id = run.info.run_id - written_code_dir = Path( - repository.client.download_artifacts(run_id, 'prediction_model/code') - ) + written_code_dir = Path(repository.client.download_artifacts(run_id, 'prediction_model/code')) assert (written_code_dir / f'{CODE_MODULE_NAME}.py').is_file() # The class must resolve from the artifact's written code/, not from anything left in memory. From 6c0d178f99b1ea3222de7ed09f7c949e5980dab9 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 10 Sep 2026 14:15:42 -0300 Subject: [PATCH 54/64] QTZPOC-20 QTZPOC-20: Implement structured logging with metadata for repository, worker, and OPC activities. --- laborious/activities/opc.py | 6 +- .../utils/repository/model_repository.py | 55 ++++++++++++------- laborious/worker/worker.py | 10 ++-- .../utils/repository/test_model_repository.py | 24 ++++---- tests/laborious/worker/test_worker_main.py | 6 +- 5 files changed, 60 insertions(+), 41 deletions(-) diff --git a/laborious/activities/opc.py b/laborious/activities/opc.py index 2c14cf5..7dac8da 100644 --- a/laborious/activities/opc.py +++ b/laborious/activities/opc.py @@ -122,7 +122,7 @@ class OPC(SientiaMonitoring): misconfigured server intentionally stops every other server, and the whole worker, from starting. """ - self.logger.info('Initializing OPC servers...') + self.info('Initializing OPC servers...') for opc_id, server in self.opc_servers.items(): self.opc_repository[opc_id] = OpcRepository( opc_id=server['id'], @@ -153,9 +153,7 @@ class OPC(SientiaMonitoring): attachment_content=error_data.get('attachment_content', None), ) else: - self.logger.info( - f'OPC server {opc_id}:{server["server_name"]} connected successfully.' - ) + self.info(f'OPC server {opc_id}:{server["server_name"]} connected successfully.') async def write_data( self, diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py index 8d3d9f6..285957a 100644 --- a/laborious/utils/repository/model_repository.py +++ b/laborious/utils/repository/model_repository.py @@ -52,7 +52,7 @@ MODEL_PICKLE_NAME = 'model.pkl' INVALID_FLAVOR_MESSAGE = "Invalid flavor. Use 'sklearn' or 'pyfunc' or 'joblib'." -def force_memory_release(logger: Logger): +def force_memory_release(logger: Logger, metadata: dict[str, Any] | None = None): """Attempt to release memory from the Python process. Executes a garbage collection cycle and calls `malloc_trim(0)` on glibc @@ -61,14 +61,17 @@ def force_memory_release(logger: Logger): Args: logger (Logger): Logger for observability. + metadata (dict[str, Any] | None): Metadata used for structured logging. """ gc.collect() + metadata = metadata or {} + try: ctypes.CDLL('libc.so.6').malloc_trim(0) - logger.info('Memory released') + logger.custom_info('Memory released', metadata) except Exception as e: - logger.info(f'Memory release failed: {e}') + logger.custom_info(f'Memory release failed: {e}', metadata) class MLFlowRepository(SientiaMonitoring): @@ -320,7 +323,7 @@ class MLFlowRepository(SientiaMonitoring): rmtree(full_path) makedirs(output_dir, exist_ok=True) - self.info(f'Downloading artifacts from {run_id} to {output_dir}') + self.info(f'Downloading artifacts from {run_id} to {output_dir}', metadata) core_labels = self.get_core_labels(metadata, operation_type='download_artifacts') @@ -676,7 +679,7 @@ class MLFlowRepository(SientiaMonitoring): model_uri = None if flavor == 'joblib' else f'models:/{model_name}/production' if model_uri is not None: - self.info(f'Loading prediction model {model_name} from {model_uri}') + self.info(f'Loading prediction model {model_name} from {model_uri}', metadata) core_labels = self.get_core_labels(metadata, operation_type='load_predict_model') start_time = time.time() @@ -756,7 +759,7 @@ class MLFlowRepository(SientiaMonitoring): ) if model_uri is not None: - self.info(f'Loading data model {model_name} from {model_uri}') + self.info(f'Loading data model {model_name} from {model_uri}', metadata) core_labels = self.get_core_labels(metadata, operation_type='load_transform_model') start_time = time.time() @@ -822,7 +825,8 @@ class MLFlowRepository(SientiaMonitoring): """ self.info( - f'Downloading {model_type} model {model_name} with flavor {flavor} and load_wrapper {load_wrapper}' + f'Downloading {model_type} model {model_name} with flavor {flavor} and load_wrapper {load_wrapper}', + metadata, ) if model_type not in ['predict', 'transform']: @@ -838,7 +842,10 @@ class MLFlowRepository(SientiaMonitoring): if load_wrapper: core_labels = self.get_core_labels(metadata, operation_type='download_model_wrapper') - self.info(f'Loading wrapper for {model_type} model {model_name} with flavor {flavor}') + self.info( + f'Loading wrapper for {model_type} model {model_name} with flavor {flavor}', + metadata, + ) try: artifact_path = await self.dowload_artifacts( @@ -846,7 +853,8 @@ class MLFlowRepository(SientiaMonitoring): ) self.info( - f'Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}' + f'Model with type {model_type} and name {model_name} is compressed, loading from {artifact_path}', + metadata, ) raw_model = mlflow.pyfunc.load_model(artifact_path) @@ -959,33 +967,37 @@ class MLFlowRepository(SientiaMonitoring): return False return True - def handle_valid_model(self, model_name: str, cache: dict) -> dict: + def handle_valid_model(self, model_name: str, cache: dict, metadata: dict[str, Any]) -> dict: """ Return the cached model configuration when retention is valid. Args: model_name (str): Name of the model (for logging/consistency). cache (dict): Cached model data structure. + metadata (dict[str, Any]): Metadata used for structured logging. Returns: dict: Model configuration. """ - self.debug(f'Model {model_name} is still valid, using cached version') + self.debug(f'Model {model_name} is still valid, using cached version', metadata) return cache['target'] - def handle_outdated_model(self, model_name: str, model_key: str) -> None: + def handle_outdated_model( + self, model_name: str, model_key: str, metadata: dict[str, Any] + ) -> None: """ Clean up outdated cached model and its artifacts. Args: model_name (str): Name of the model. model_key (str): Cache key for the model. + metadata (dict[str, Any]): Metadata used for structured logging. Returns: None """ - self.debug(f'Model {model_name} is outdated, downloading a new one') + self.debug(f'Model {model_name} is outdated, downloading a new one', metadata) del self.model_cache[model_key]['target'] del self.model_cache[model_key] @@ -1031,13 +1043,18 @@ class MLFlowRepository(SientiaMonitoring): # Check if config has changed or is outdated if self.check_cache_retention(cache, retention): - return self.handle_valid_model(model_name=model_name, cache=cache) + return self.handle_valid_model( + model_name=model_name, cache=cache, metadata=metadata + ) else: # Model is outdated, delete old model files - self.handle_outdated_model(model_name=model_name, model_key=model_key) + self.handle_outdated_model( + model_name=model_name, model_key=model_key, metadata=metadata + ) else: self.debug( - f'Model {model_name} is not in {model_type} cache, downloading a new one' + f'Model {model_name} is not in {model_type} cache, downloading a new one', + metadata, ) # Donwload new model (without lock to avoid blocking other threads) @@ -1113,10 +1130,10 @@ class MLFlowRepository(SientiaMonitoring): prediction = model.predict(data) if retention == 0: - self.info(f'Deleting model {model_name}:{operation} from memory') + self.info(f'Deleting model {model_name}:{operation} from memory', metadata) del model - force_memory_release(self.logger) + force_memory_release(self.logger, metadata) return prediction @@ -1550,7 +1567,7 @@ class MLFlowRepository(SientiaMonitoring): del data_model['model'] del data_model - force_memory_release(self.logger) + force_memory_release(self.logger, metadata) return { 'run_id': run_id, diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index e585a2d..d096820 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -136,7 +136,7 @@ async def main(): metadata_runtime = {**metadata, 'runtime': runtime} logger.custom_info('Starting prometheus client...', metadata_runtime) - start_prometheus_server() + start_prometheus_server(logger, metadata_runtime) logger.custom_info('Starting Notification Handler...', metadata_runtime) @@ -313,7 +313,7 @@ async def main(): exit_code = e.code if isinstance(e, SystemExit) and isinstance(e.code, int) else 0 raise except Exception as e: - logger.custom_error(f'An unhandled exception occurred: {e}', metadata) + logger.custom_error(f'An unhandled exception occurred: {e}', metadata_runtime) exit_code = 1 finally: if notification_handler: @@ -324,7 +324,7 @@ async def main(): sys.exit(exit_code) -def start_prometheus_server(): +def start_prometheus_server(logger, metadata: dict): """ Starts the Prometheus metrics server for monitoring and observability. @@ -344,10 +344,10 @@ def start_prometheus_server(): try: port = int(os.getenv('HTTP_METRICS_PORT', 9090)) start_http_server(port) - print(f'Prometheus server started on port {port}.') + logger.custom_info(f'Prometheus server started on port {port}.', metadata) metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP except Exception as e: - print(f'Failed to start Prometheus server: {e}') + logger.custom_critical(f'Failed to start Prometheus server: {e}', metadata) os._exit(1) diff --git a/tests/laborious/utils/repository/test_model_repository.py b/tests/laborious/utils/repository/test_model_repository.py index 0311ef1..454605a 100644 --- a/tests/laborious/utils/repository/test_model_repository.py +++ b/tests/laborious/utils/repository/test_model_repository.py @@ -27,25 +27,27 @@ from laborious.utils.repository.model_repository import ( @patch('laborious.utils.repository.model_repository.gc') def test_force_memory_release_success(gc, ctypes): logger = MagicMock() + metadata = {'model_name': 'my-model'} - force_memory_release(logger) + force_memory_release(logger, metadata) gc.collect.assert_called_once() ctypes.CDLL.return_value.malloc_trim.assert_called_once_with(0) - logger.info.assert_called_once_with('Memory released') + logger.custom_info.assert_called_once_with('Memory released', metadata) @patch('laborious.utils.repository.model_repository.ctypes') @patch('laborious.utils.repository.model_repository.gc') def test_force_memory_release_error(gc, ctypes): logger = MagicMock() + metadata = {'model_name': 'my-model'} ctypes.CDLL.return_value.malloc_trim.side_effect = Exception('error') - force_memory_release(logger) + force_memory_release(logger, metadata) gc.collect.assert_called_once() ctypes.CDLL.return_value.malloc_trim.assert_called_once_with(0) - logger.info.assert_called_once_with('Memory release failed: error') + logger.custom_info.assert_called_once_with('Memory release failed: error', metadata) @pytest.fixture @@ -807,7 +809,7 @@ def test_check_cache_retention_true(datetime_mock, mlflow_repository): def test_handle_valid_model(mlflow_repository): cache = {'target': {'model': 'model', 'artifact_path': 'test_artifact_path'}} - output = mlflow_repository.handle_valid_model('model_name', cache) + output = mlflow_repository.handle_valid_model('model_name', cache, {}) assert output == {'model': 'model', 'artifact_path': 'test_artifact_path'} @@ -817,7 +819,7 @@ def test_handle_outdated_model(mlflow_repository): 'target': {'model': 'model', 'artifact_path': 'test_artifact_path'} } } - mlflow_repository.handle_outdated_model('model_name', 'model_name_transform') + mlflow_repository.handle_outdated_model('model_name', 'model_name_transform', {}) assert mlflow_repository.model_cache == {} @@ -857,7 +859,9 @@ async def test_get_model_cached_valid(mlflow_repository): ) mlflow_repository.handle_valid_model.assert_called_once_with( - model_name='model_name', cache=mlflow_repository.model_cache['model_name_predict'] + model_name='model_name', + cache=mlflow_repository.model_cache['model_name_predict'], + metadata={}, ) mlflow_repository.handle_outdated_model.assert_not_called() @@ -887,7 +891,7 @@ async def test_get_model_cached_outdated(mlflow_repository): ) mlflow_repository.handle_valid_model.assert_not_called() mlflow_repository.handle_outdated_model.assert_called_once_with( - model_name='model_name', model_key='model_name_predict' + model_name='model_name', model_key='model_name_predict', metadata={} ) @@ -916,7 +920,7 @@ async def test_get_cached_operation_retention_0(force_memory_release, mlflow_rep 'model_name', data, 'transform', 0, 'sklearn', {} ) assert output == model.predict.return_value - force_memory_release.assert_called_once_with(mlflow_repository.logger) + force_memory_release.assert_called_once_with(mlflow_repository.logger, {}) @patch('laborious.utils.repository.model_repository.force_memory_release') @@ -1316,7 +1320,7 @@ async def test_create_new_experiment( ] ) - force_memory_release.assert_called_once_with(mlflow_repository.logger) + force_memory_release.assert_called_once_with(mlflow_repository.logger, metadata['metadata']) assert report == { 'run_id': mlflow.start_run.return_value.__enter__.return_value.info.run_id, diff --git a/tests/laborious/worker/test_worker_main.py b/tests/laborious/worker/test_worker_main.py index 92e872f..a222464 100644 --- a/tests/laborious/worker/test_worker_main.py +++ b/tests/laborious/worker/test_worker_main.py @@ -210,7 +210,7 @@ class TestMainHappyPath: @patch('laborious.worker.worker.metrics.APP_UP') @patch('laborious.worker.worker.start_http_server') def test_start_prometheus_server_marks_the_app_up(mock_start_http_server, mock_app_up): - start_prometheus_server() + start_prometheus_server(MagicMock(), {}) mock_start_http_server.assert_called_once() mock_app_up.labels.return_value.set.assert_called_once_with(1) @@ -224,7 +224,7 @@ def test_start_prometheus_server_exits_the_process_on_failure( ): mock_start_http_server.side_effect = OSError('port in use') - start_prometheus_server() + start_prometheus_server(MagicMock(), {}) mock_os_exit.assert_called_once_with(1) mock_app_up.labels.return_value.set.assert_not_called() @@ -238,6 +238,6 @@ def test_start_prometheus_server_reads_the_configured_port( ): monkeypatch.setenv('HTTP_METRICS_PORT', port_env) - start_prometheus_server() + start_prometheus_server(MagicMock(), {}) mock_start_http_server.assert_called_once_with(int(port_env)) From bf48a18c7b46f21fba16175feedba7e39592952f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 10 Sep 2026 14:25:08 -0300 Subject: [PATCH 55/64] QTZPOC-20 QTZPOC-20: Improve worker shutdown handling by allowing SystemExit and KeyboardInterrupt to propagate directly, preserving their original exit codes and ensuring cleanup. --- laborious/worker/worker.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index d096820..3ab3938 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -305,23 +305,23 @@ async def main(): # This will run the workers and wait for them to complete. # If an exception occurs in any of the worker handlers, it will be propagated here. await asyncio.gather(*handlers) - except (SystemExit, KeyboardInterrupt) as e: - # Deliberate shutdown (Ctrl+C, a handler's own sys.exit) — not a failure to log as one. - # `finally` below calls sys.exit(exit_code) unconditionally, which supersedes this raise - # and becomes the exception that actually propagates — the raise is here so this branch - # still honors the reraise-on-SystemExit contract, not because it changes what exits. - exit_code = e.code if isinstance(e, SystemExit) and isinstance(e.code, int) else 0 - raise except Exception as e: logger.custom_error(f'An unhandled exception occurred: {e}', metadata_runtime) exit_code = 1 finally: + # No `except (SystemExit, KeyboardInterrupt)` on purpose: `finally` runs on every path, + # caught or not, so cleanup still happens for a deliberate shutdown (Ctrl+C, a handler's + # own sys.exit) without us having to catch and re-raise it ourselves — which would only + # let us relabel its exit code before `sys.exit(exit_code)` below discards it anyway. + # Letting it propagate on its own keeps the original code (e.g. a handler's sys.exit(N)) + # intact instead of collapsing it to 0. if notification_handler: notification_handler.shutdown() if activities: await activities.shutdown() metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN - sys.exit(exit_code) + + sys.exit(exit_code) def start_prometheus_server(logger, metadata: dict): From 1c32e9d678c8493a110e44ad4e338050b4d7469a Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 10 Sep 2026 15:09:35 -0300 Subject: [PATCH 56/64] QTZPOC-20 QTZPOC-20: Remove trailing newline from sonar-project.properties --- sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index 37d1521..cb92fd5 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -8,4 +8,4 @@ sonar.qualitygate.wait=true sonar.qualitygate.timeout=300 sonar.python.coverage.reportPaths=coverage.xml sonar.python.xunit.reportPath=pytest.xml -sonar.python.version=3.11 +sonar.python.version=3.11 \ No newline at end of file From b65cf2ba6873e82399b5612dcaad2a445bf0a981 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 10 Sep 2026 16:18:18 -0300 Subject: [PATCH 57/64] QTZPOC-20 QTZPOC-20: Pin mlflow dependency to version 2.10.1 in requirements-light.txt --- requirements-light.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-light.txt b/requirements-light.txt index 8bfedaa..6220a72 100644 --- a/requirements-light.txt +++ b/requirements-light.txt @@ -4,7 +4,7 @@ sqlalchemy asyncua==1.0.6 redis sientia_do>=1.12.2 -mlflow +mlflow==2.10.1 prometheus-client botocore boto3 From 85fe4663b1ae6331b53948a1d2a7683ba0f4420b Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Fri, 11 Sep 2026 10:36:06 -0300 Subject: [PATCH 58/64] SIENTIAPDE-2068: feat(cicd): migrate laborious-worker deploy to Flow B (legacy only) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Delete release.yml (tarball-only, no deploy job). - Add deploy.yml calling reusable-deploy-dataops.yml, bumping only the legacy runtime's GITHUB_BRANCH via the new profile/profile_env_list_path mechanism — basic is left untouched for now. workflow_dispatch has no inputs. - Add require-chart-ref-label.yml. --- .github/workflows/deploy.yml | 35 +++++++++++++++++++ .github/workflows/release.yml | 25 ------------- .github/workflows/require-chart-ref-label.yml | 16 +++++++++ 3 files changed, 51 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/deploy.yml delete mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/require-chart-ref-label.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..a0451c9 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,35 @@ +name: Deploy DataOps Worker + +on: + pull_request: + types: [closed] + branches: [main, 'release/**', 'feature/**'] + workflow_dispatch: + +permissions: + contents: write + id-token: write + pull-requests: read + +jobs: + deploy: + name: Deploy + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true + uses: Aignosi/github_workflow_templates/.github/workflows/reusable-deploy-dataops.yml@main + with: + branch_name: ${{ github.event.pull_request.head.ref }} + project_name: 'laborious' + chart_name: 'laborious-worker' + # Only the "legacy" runtime, for now (card 13 correction) — laborious-worker has two + # Deployments (basic, legacy). basic's GITHUB_BRANCH lives in the chart base (.global.env) + # and is deliberately left untouched here. legacy's own override exists ONLY at the profile + # level — profiles/gcp.yaml's laborious-worker.runtimes.legacy.env, not the chart base at + # all — hence profile/profile_env_list_path below instead of env_list_path. + profile: 'gcp' + profile_env_list_path: '[".runtimes.legacy.env"]' + update_version_in: '["pyproject"]' + create_tarball: ${{ github.event_name == 'pull_request' }} + mirror: true + bump_chart: true + deploy: true + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index e051ea9..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Create Release on Merge to Main - -on: - pull_request: - types: [closed] - branches: - - main - workflow_dispatch: - inputs: - version: - description: 'Version to release' - required: false - type: string - -jobs: - release: - if: | - (github.event_name == 'pull_request' && github.event.pull_request.merged == true) || - github.event_name == 'workflow_dispatch' - uses: Aignosi/github_workflow_templates/.github/workflows/dataops-module-release.yml@main - permissions: write-all - with: - project_name: 'laborious' - release_version: ${{ github.event.inputs.version || '' }} - secrets: inherit \ No newline at end of file diff --git a/.github/workflows/require-chart-ref-label.yml b/.github/workflows/require-chart-ref-label.yml new file mode 100644 index 0000000..20f8be3 --- /dev/null +++ b/.github/workflows/require-chart-ref-label.yml @@ -0,0 +1,16 @@ +name: Require chart-ref label + +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled] + branches: + - main + - 'release/**' + - 'feature/**' + +permissions: + pull-requests: write + +jobs: + require-chart-ref-label: + uses: Aignosi/github_workflow_templates/.github/workflows/require-chart-ref-label.yml@main From 76351fdf8a2aa8c6418ffae7399d6c2a992dbe84 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 11 Sep 2026 13:47:07 +0000 Subject: [PATCH 59/64] chore: bump version to 0.7.1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7ffcad0..92d9b66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "laborious" -version = "0.0.0" +version = "0.7.1" description = "Sientia DataOps Laborious - ML Model Orchestration System" readme = "README.md" requires-python = ">=3.11" From 5690e792a99b762c096b54ae0fe5d97b09af98e0 Mon Sep 17 00:00:00 2001 From: "ci-dependencies-fetcher[bot]" <1275848+ci-dependencies-fetcher[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 13:47:36 +0000 Subject: [PATCH 60/64] Revert "chore: bump version to 0.7.1" This reverts commit 76351fdf8a2aa8c6418ffae7399d6c2a992dbe84. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 92d9b66..7ffcad0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "laborious" -version = "0.7.1" +version = "0.0.0" description = "Sientia DataOps Laborious - ML Model Orchestration System" readme = "README.md" requires-python = ">=3.11" From 959c78170ca0332fa5510a113b6acfb605a7d9f8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 11 Sep 2026 14:43:24 +0000 Subject: [PATCH 61/64] chore: bump version to 0.7.2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7ffcad0..7d56283 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "laborious" -version = "0.0.0" +version = "0.7.2" description = "Sientia DataOps Laborious - ML Model Orchestration System" readme = "README.md" requires-python = ">=3.11" From ce91a0bbb67fbe645618cf9aa3aa907858d6790f Mon Sep 17 00:00:00 2001 From: "ci-dependencies-fetcher[bot]" <1275848+ci-dependencies-fetcher[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:49:24 +0000 Subject: [PATCH 62/64] Revert "chore: bump version to 0.7.2" This reverts commit 959c78170ca0332fa5510a113b6acfb605a7d9f8. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7d56283..7ffcad0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "laborious" -version = "0.7.2" +version = "0.0.0" description = "Sientia DataOps Laborious - ML Model Orchestration System" readme = "README.md" requires-python = ">=3.11" From ac67d5b447003a7e7d8aa967bea8cea656f315af Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 11 Sep 2026 15:19:06 +0000 Subject: [PATCH 63/64] chore: bump version to 0.7.2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7ffcad0..7d56283 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "laborious" -version = "0.0.0" +version = "0.7.2" description = "Sientia DataOps Laborious - ML Model Orchestration System" readme = "README.md" requires-python = ">=3.11" From 8ef7ed800a96a95c842de79d4f38861b3a12020c Mon Sep 17 00:00:00 2001 From: Bruno Domingues Date: Tue, 15 Sep 2026 00:08:37 -0300 Subject: [PATCH 64/64] SIENTIAPDE-2071: chore: remove dead Makefile Card 16 of the SIENTIAPDE-2052 epic. This repo is a Flow B dataops worker, it does not build its own image at all anymore, it runs off a shared runtime image with code cloned from Gitea at deploy time. A per repo image build Makefile pointing at aignosi.azurecr.io is dead weight, no workflow calls make. Safe to remove. --- Makefile | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 66aae81..0000000 --- a/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -VERSION = 1.0.8 -name = sientia-laborious -# ENVIRONMENT = production - -docker-hub: - @docker build --no-cache -t aignosi.azurecr.io/$(name):$(VERSION) . - @docker push aignosi.azurecr.io/$(name):$(VERSION) \ No newline at end of file