QTZPOC-20

QTZPOC-20: Enhance system robustness and test coverage across bundle processing, activity error handling, and worker operations.

*   Strengthen bundle reader validation with new tests for archive integrity, security vulnerabilities (e.g., path traversal, duplicate entries, special file types), and I/O error resilience.
*   Improve error handling and status propagation in PI Web API, MLflow, and OPC activities, including partial write scenarios and OPC session management.
*   Add comprehensive tests for worker startup, configuration, graceful shutdown, and Prometheus metrics server initialization.
*   Introduce a new utility for intelligent DataFrame debugging, summarizing large dataframes for concise logging, and refine exception chain summarization.
*   Update code coverage configuration to exclude @overload decorator lines.

tests/laborious/utils/bundle/test_reader.py

@@ -1,7 +1,11 @@
 """The seven gates: what each one rejects, in what order, and what it leaves on disk."""
 
+import io
 import json
+import os
 import random
+import stat
+import zipfile
 from pathlib import Path
 
 import nacl.bindings as sodium
@@ -173,6 +177,16 @@ def test_gate_2_rejects_a_truncated_transfer(work_dir, reader):
     assert error.value.gate == 2
 
 
+def test_gate_2_wraps_an_os_error_reading_the_file(work_dir, reader):
+    """A download that never landed on disk is an OS error, not a format one."""
+    missing = work_dir / 'never-downloaded.sientia'
+
+    with pytest.raises(BundleGateError, match='could not be read') as error:
+        reader.open(missing, factory.DEFAULT_PASSWORD, work_dir / 'extract', work_dir)
+
+    assert error.value.gate == 2
+
+
 def _never_called(message: str):
     def _raise(*_args, **_kwargs):
         raise AssertionError(message)
@@ -220,6 +234,22 @@ def test_gate_3_stream_without_a_final_tag_is_rejected(work_dir, reader):
     assert AMBIGUOUS_REASON in str(error.value)
 
 
+def test_gate_3_wraps_an_os_error_while_writing_the_decrypted_zip(work_dir, reader, monkeypatch):
+    """A write failure while streaming the plaintext to disk is reported as disk trouble, not as
+    a bad password — the AEAD never even got to disagree."""
+    payload = factory.make_bundle()
+
+    def _raise_os_error(*_args, **_kwargs):
+        raise OSError('disk full')
+
+    monkeypatch.setattr(reader, '_pull_stream', _raise_os_error)
+
+    with pytest.raises(BundleGateError, match='could not be written') as error:
+        _open(reader, work_dir, payload)
+
+    assert error.value.gate == 3
+
+
 def test_gate_3_leaves_no_decrypted_zip_behind(work_dir, reader):
     payload = factory.make_bundle()
 
@@ -316,6 +346,27 @@ def test_gate_4_rejects_content_that_is_not_an_archive(work_dir, reader):
     assert error.value.gate == 4
 
 
+def test_gate_4_rejects_an_empty_archive(work_dir, reader):
+    payload = factory.make_bundle(factory.ZipSpec(files={}, include_metadata=False))
+
+    with pytest.raises(BundleGateError, match='archive is empty') as error:
+        _open(reader, work_dir, payload)
+
+    assert error.value.gate == 4
+
+
+def test_gate_4_rejects_an_entry_that_is_neither_a_file_nor_a_directory(work_dir, reader):
+    """A FIFO's mode bits are neither a symlink's, a regular file's, nor a directory's."""
+    payload = factory.make_bundle(
+        factory.ZipSpec(special_mode_entries={'artifacts/pipe': stat.S_IFIFO | 0o644})
+    )
+
+    with pytest.raises(BundleGateError, match='not a regular file') as error:
+        _open(reader, work_dir, payload)
+
+    assert error.value.gate == 4
+
+
 # ------------------------------------------------------------------- gate 5
 
 
@@ -338,6 +389,61 @@ def test_gate_5_extraction_writes_plain_files(work_dir, reader):
             assert path.stat().st_mode & 0o111 == 0, path
 
 
+def test_gate_5_extraction_creates_declared_directory_entries(work_dir, reader):
+    """The producer never writes directory entries, but a bundle from anywhere else might."""
+    payload = factory.make_bundle(factory.ZipSpec(directory_entries=['artifacts/empty_dir']))
+
+    opened = _open(reader, work_dir, payload)
+
+    assert (opened.extracted_dir / 'artifacts' / 'empty_dir').is_dir()
+
+
+def test_gate_5_rejects_a_duplicate_entry_name(work_dir, reader):
+    """Two archive entries claiming the same path: gate 4 checks each name on its own and cannot
+    see the collision, so extraction is where an overwrite is caught — and rejected."""
+    payload = factory.make_bundle(factory.ZipSpec(repeat_file='artifacts/prediction_model/MLmodel'))
+
+    with pytest.raises(BundleGateError, match='overwrite') as error:
+        _open(reader, work_dir, payload)
+
+    assert error.value.gate == 5
+    assert not (work_dir / 'extract').exists()
+
+
+def test_gate_5_wraps_an_os_error_during_unpacking(work_dir, reader, monkeypatch):
+    """A write failure partway through extraction (e.g. disk full) is reported as unpacking
+    trouble, not surfaced as the raw OSError."""
+    payload = factory.make_bundle()
+
+    def _raise_os_error(*_args, **_kwargs):
+        raise OSError('disk full')
+
+    monkeypatch.setattr('laborious.utils.bundle.reader.shutil.copyfileobj', _raise_os_error)
+
+    with pytest.raises(BundleGateError, match='could not be unpacked') as error:
+        _open(reader, work_dir, payload)
+
+    assert error.value.gate == 5
+    assert not (work_dir / 'extract').exists()
+
+
+def test_extract_member_rechecks_containment_on_the_resolved_path(work_dir, reader):
+    """Gate 4 already rejects a traversal by name; extraction re-checks on the *resolved* path as
+    a second, independent line of defense."""
+    root = work_dir / 'root'
+    root.mkdir()
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, 'w') as archive:
+        archive.writestr('safe.txt', b'hello')
+
+    with zipfile.ZipFile(io.BytesIO(buffer.getvalue())) as archive:
+        entry = archive.infolist()[0]
+        with pytest.raises(BundleGateError, match='resolves outside') as error:
+            reader._extract_member(archive, entry, root, work_dir / 'somewhere-else')
+
+    assert error.value.gate == 5
+
+
 # ------------------------------------------------------------------- gate 6
 
 
@@ -375,6 +481,10 @@ def test_gate_5_extraction_writes_plain_files(work_dir, reader):
             ),
             'experiment name',
         ),
+        (
+            factory.ZipSpec(document={'metadata': 'not-a-dict', 'parameters': {}}),
+            'not objects',
+        ),
     ],
 )
 def test_gate_6_rejects_a_malformed_metadata_document(work_dir, reader, spec, match):
@@ -457,6 +567,19 @@ def test_gate_7_rejects_content_a_model_export_should_not_carry(work_dir, reader
     assert not (work_dir / 'extract').exists()
 
 
+def test_gate_7_rejects_a_non_regular_file_left_in_the_tree(work_dir, reader):
+    """Extraction itself only ever writes plain files and directories; this is the check that
+    would catch anything else showing up in the tree by some other means."""
+    root = work_dir / 'weird_root'
+    root.mkdir()
+    os.mkfifo(root / 'pipe')
+
+    with pytest.raises(BundleGateError, match='not a file') as error:
+        reader._gate_7_content_policy(root)
+
+    assert error.value.gate == 7
+
+
 # ------------------------------------------------------------------ ordering
 
 

tests/laborious/utils/models/test_import_errors.py

@@ -536,6 +536,40 @@ def test_a_failure_that_arrives_as_nothing_says_so_rather_than_guessing():
     assert summary == ''
 
 
+def test_summarize_failure_bounds_a_cyclic_cause_chain():
+    """A cycle never hits a `None` cause, so the walk must stop by exhausting its own bound."""
+    first = RuntimeError('first')
+    second = RuntimeError('second')
+    first.__cause__ = second
+    second.__cause__ = first
+
+    kind, _summary = summarize_failure(first)
+
+    assert kind == 'RuntimeError'
+
+
+def test_a_leading_blank_line_is_skipped_for_the_first_real_one():
+    class BlankFirstLineError(Exception):
+        def __str__(self) -> str:
+            return '\n \nreal message'
+
+    _kind, summary = summarize_failure(BlankFirstLineError())
+
+    assert summary == 'real message'
+
+
+def test_describe_failure_chain_reports_a_message_less_link_by_its_type_alone():
+    """`Exception()` has no message at all — the link still has to be readable on its own."""
+    inner = Exception()
+    outer = RuntimeError('outer failure')
+    outer.__cause__ = inner
+
+    kind, detail = describe_failure_chain(outer)
+
+    assert kind == 'Exception'
+    assert detail.splitlines() == ['Exception', 'RuntimeError: outer failure']
+
+
 # ------------------------------------------------- the channel is the only reporter
 
 

tests/laborious/utils/models/test_minio_dataframe_payload.py

@@ -214,6 +214,27 @@ async def test_from_dataframe_offloaded(mock_now):
     minio.upload_file.assert_awaited_once()
 
 
+@pytest.mark.asyncio
+@patch('laborious.utils.models.minio_dataframe_payload.OFFLOAD_THRESHOLD_BYTES', 10**9)
+async def test_from_dataframe_inline_keeps_a_provided_last_timestamp():
+    """An explicit last_timestamp must survive as-is, skipping the max() derivation."""
+    minio = AsyncMock()
+    logger = MagicMock()
+    df = _mock_dataframe({'timestamp': ['2024-01-01'], 'value': [42]})
+
+    result = await MinioDataFramePayload.from_dataframe(
+        dataframe=df,
+        minio_repo=minio,
+        model_name='m',
+        operation='initial',
+        last_timestamp='2030-01-01T00:00:00+00:00',
+        logger=logger,
+    )
+
+    assert result.last_timestamp == '2030-01-01T00:00:00+00:00'
+    logger.custom_debug.assert_called()
+
+
 def test_from_dict_inline():
     raw = {
         'last_timestamp': '2024-01-01T00:00:00+00:00',

tests/laborious/utils/repository/test_opc_repository.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+import time
 from datetime import datetime
 from unittest.mock import ANY, AsyncMock, MagicMock, Mock, patch
 
@@ -8,11 +9,13 @@ from asyncua.crypto.security_policies import SecurityPolicyBasic256
 from asyncua.ua.uaerrors import BadNodeIdUnknown, BadSessionIdInvalid
 from sientia_do.notifications.models import NotificationLevel
 
+from laborious import metrics
 from laborious.utils.repository.opc_repository import (
     OpcClientAlreadyExistsError,
     OpcClientNotInitializedError,
     OpcRepository,
     OpcSessionAlreadyConnectedError,
+    _opc_authentication_token_str,
     is_reconnectable_opcua_bad,
 )
 
@@ -80,6 +83,40 @@ def test_init(opc_repository):
     assert opc_repository.last_reconnection_time is None
 
 
+def test_opc_authentication_token_str_no_protocol():
+    client = MagicMock()
+    client.uaclient.protocol = None
+
+    assert _opc_authentication_token_str(client) == 'unknown'
+
+
+def test_opc_authentication_token_str_no_token():
+    client = MagicMock()
+    client.uaclient.protocol = MagicMock(authentication_token=None)
+
+    assert _opc_authentication_token_str(client) == 'unknown'
+
+
+def test_opc_authentication_token_str_swallows_errors():
+    class _ExplodingClient:
+        @property
+        def uaclient(self):
+            raise RuntimeError('boom')
+
+    assert _opc_authentication_token_str(_ExplodingClient()) == 'unknown'
+
+
+def test_is_session_open_swallows_errors(opc_repository):
+    class _ExplodingClient:
+        @property
+        def uaclient(self):
+            raise RuntimeError('boom')
+
+    opc_repository.client = _ExplodingClient()
+
+    assert opc_repository._is_session_open() is False
+
+
 @pytest.mark.asyncio
 async def test_set_security(opc_repository, mock_client):
     opc_repository.client = mock_client
@@ -160,6 +197,47 @@ async def test_create_client_raises_when_client_exists(opc_repository, mock_clie
         await opc_repository._create_client()
 
 
+@pytest.mark.asyncio
+async def test_create_client_builds_a_secured_client(opc_repository, mock_client):
+    opc_repository.pod_id = 'pod-1'
+    opc_repository.set_security = AsyncMock()
+
+    await opc_repository._create_client()
+
+    assert opc_repository.client is mock_client
+    assert mock_client.name == 'pod-1'
+    assert mock_client.application_name == 'pod-1'
+    assert mock_client.application_uri == 'pod:1'
+    assert mock_client.product_uri == 'pod:1'
+    opc_repository.set_security.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_create_client_skips_security_without_a_cert_path(opc_repository, mock_client):
+    opc_repository.cert_path = None
+    opc_repository.set_security = AsyncMock()
+
+    await opc_repository._create_client()
+
+    assert opc_repository.client is mock_client
+    opc_repository.set_security.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_connect_locked_reuses_an_existing_client(opc_repository, mock_client):
+    """A client already set (e.g. surviving a prior failed open) is not recreated."""
+    opc_repository.client = mock_client
+    opc_repository._is_session_open = MagicMock(return_value=False)
+    opc_repository._create_client = AsyncMock()
+    opc_repository._open_session = AsyncMock(return_value=(True, {}))
+
+    result = await opc_repository._connect_locked()
+
+    opc_repository._create_client.assert_not_called()
+    opc_repository._open_session.assert_called_once()
+    assert result == (True, {})
+
+
 @pytest.mark.asyncio
 async def test_open_session_success(opc_repository):
     closed_proto = MagicMock()
@@ -435,6 +513,82 @@ async def test_write_data_write_value_failed(opc_repository, mock_client):
     assert error_data['attachment_content'] is not None
 
 
+@pytest.mark.asyncio
+async def test_write_data_with_empty_metadata_uses_unknown_labels(opc_repository, mock_client):
+    """No model_id/model_name in the write metadata: the metric labels fall back to 'unknown'."""
+    opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
+    opc_repository.client = mock_client
+    mock_node = AsyncMock()
+    mock_client.get_node = MagicMock(return_value=mock_node)
+
+    result = await opc_repository.write_data('ns=2;s=TestNode', 42.0, 'float', {})
+
+    assert result == (True, {'response_time': ANY})
+    last_call = opc_repository.emit_metric.call_args_list[-1]
+    assert last_call.args[0] is metrics.OPC_WRITE_ATTEMPTS_TOTAL
+    assert last_call.args[1]['model_id'] == 'unknown'
+    assert last_call.args[1]['model_name'] == 'unknown'
+
+
+@pytest.mark.asyncio
+async def test_write_data_get_node_bad_session_id_schedules_reconnect(opc_repository, mock_client):
+    opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
+    opc_repository.client = mock_client
+    opc_repository._start_reconnect = AsyncMock()
+    mock_client.get_node = MagicMock(side_effect=BadSessionIdInvalid())
+
+    is_success, error_data = await opc_repository.write_data(
+        'ns=2;s=TestNode', 42.0, 'float', metadata['metadata']
+    )
+
+    opc_repository._start_reconnect.assert_called_once()
+    assert is_success is False
+    assert error_data['opc_error_kind'] == 'session_bad'
+    assert error_data['opc_status'] == 'BadSessionIdInvalid'
+
+
+@pytest.mark.asyncio
+async def test_write_data_logs_inter_arrival_and_flags_a_stale_session(opc_repository, mock_client):
+    """The second write on the same session logs the gap since the previous one, and flags it
+    when the gap outlives the session's revised timeout."""
+    opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
+    opc_repository.client = mock_client
+    mock_client.session_timeout = 0.001  # 1 microsecond: any real gap will exceed it
+    mock_node = AsyncMock()
+    mock_client.get_node = MagicMock(return_value=mock_node)
+
+    await opc_repository.write_data('ns=2;s=TestNode', 1.0, 'float', metadata['metadata'])
+    await opc_repository.write_data('ns=2;s=TestNode', 2.0, 'float', metadata['metadata'])
+
+    logged = [c.args[0] for c in opc_repository.info.call_args_list]
+    assert any('OPC write inter-arrival' in message for message in logged)
+    emitted_metrics = [c.args[0] for c in opc_repository.emit_metric.call_args_list]
+    assert metrics.OPC_WRITE_INTER_ARRIVAL_OVER_SESSION_TIMEOUT_TOTAL in emitted_metrics
+
+
+@pytest.mark.asyncio
+async def test_log_write_inter_arrival_skips_the_metric_without_a_client(opc_repository):
+    opc_repository._last_write_mono = time.monotonic() - 1
+    opc_repository.client = None
+
+    await opc_repository._log_write_inter_arrival('session-1', 'ns=2;s=Node')
+
+    opc_repository.emit_metric.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_log_write_inter_arrival_skips_the_metric_within_the_session_timeout(
+    opc_repository, mock_client
+):
+    opc_repository._last_write_mono = time.monotonic()
+    opc_repository.client = mock_client
+    mock_client.session_timeout = 600_000  # 600s: comfortably longer than this test takes
+
+    await opc_repository._log_write_inter_arrival('session-1', 'ns=2;s=Node')
+
+    opc_repository.emit_metric.assert_not_called()
+
+
 def test_is_reconnectable_opcua_bad():
     assert is_reconnectable_opcua_bad(BadSessionIdInvalid()) is True
     assert is_reconnectable_opcua_bad(BadNodeIdUnknown()) is False
@@ -491,6 +645,22 @@ async def test_start_reconnect_skips_within_interval(opc_repository):
     assert opc_repository._reconnect_task is None
 
 
+@pytest.mark.asyncio
+async def test_start_reconnect_skips_when_already_in_progress(opc_repository):
+    opc_repository.last_reconnection_time = None
+    in_progress_task = asyncio.create_task(asyncio.sleep(60))
+    opc_repository._reconnect_task = in_progress_task
+
+    await opc_repository._start_reconnect('BadSessionIdInvalid', 'tok')
+
+    assert opc_repository._reconnect_task is in_progress_task
+    opc_repository.warning.assert_called_once()
+
+    in_progress_task.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await in_progress_task
+
+
 @pytest.mark.asyncio
 async def test_write_data_protocol_closed_schedules_reconnect(opc_repository):
     opc_repository.client = MagicMock()
@@ -608,3 +778,23 @@ async def test_reconnect_locked_sets_last_reconnection_time(mock_datetime, opc_r
     opc_repository._connect_locked.assert_called_once()
     assert result == (True, {})
     assert opc_repository.last_reconnection_time == datetime(2025, 1, 1, 12, 0, 0)
+
+
+@pytest.mark.asyncio
+async def test_run_reconnect_failed_without_error_detail_logs_once(opc_repository):
+    """A falsy `error` (no message to add) must not log a second, empty line."""
+    opc_repository._reconnect_locked = AsyncMock(return_value=(False, {}))
+
+    await opc_repository._run_reconnect('BadSessionIdInvalid', 'tok')
+
+    assert opc_repository.error.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_run_reconnect_swallows_an_unexpected_exception(opc_repository):
+    """A bug inside the reconnect itself must not escape the background task."""
+    opc_repository._reconnect_locked = AsyncMock(side_effect=RuntimeError('boom'))
+
+    await opc_repository._run_reconnect('BadSessionIdInvalid', 'tok')
+
+    assert opc_repository.error.call_count >= 2

tests/laborious/utils/test_dataframe_debug.py

@@ -0,0 +1,40 @@
+"""Debug message formatting: full dataframe when small, a summary when it would flood the log."""
+
+from pandas import DataFrame
+
+from laborious.utils.dataframe_debug import (
+    DEFAULT_MAX_DEBUG_DATAFRAME_ROWS,
+    build_dataframe_debug_message,
+)
+
+
+def test_non_dataframe_payloads_are_stringified():
+    assert build_dataframe_debug_message('payload:', {'a': 1}) == "payload: {'a': 1}"
+
+
+def test_a_small_dataframe_is_logged_in_full():
+    data = DataFrame({'a': [1, 2]})
+
+    message = build_dataframe_debug_message('payload:', data)
+
+    assert message.startswith('payload:\n')
+    assert data.to_csv() in message
+
+
+def test_a_dataframe_over_the_row_limit_is_summarized_instead():
+    data = DataFrame({'a': range(DEFAULT_MAX_DEBUG_DATAFRAME_ROWS + 1)})
+
+    message = build_dataframe_debug_message('payload:', data)
+
+    assert 'skipped because dataframe has' in message
+    assert str(data.shape) in message
+    assert data.to_csv() not in message
+
+
+def test_the_row_limit_is_configurable():
+    data = DataFrame({'a': [1, 2, 3]})
+
+    message = build_dataframe_debug_message('payload:', data, max_rows=1)
+
+    assert 'skipped because dataframe has 3 rows' in message
+    assert '(max: 1)' in message

tests/laborious/utils/test_import_password.py

@@ -1,8 +1,11 @@
 """The password envelope: AES-256-GCM in, plaintext password out, nothing in between."""
 
 import base64
+import os
+from unittest.mock import patch
 
 import pytest
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
 
 from laborious.utils.import_password import (
     EnvelopeError,
@@ -89,6 +92,26 @@ def test_rejects_an_unusable_key(raw_key, match):
         load_envelope_key(raw_key)
 
 
+@patch('laborious.utils.import_password.AESGCM')
+def test_a_generic_decrypt_failure_is_reported_as_could_not_be_opened(mock_aesgcm):
+    """Anything AESGCM raises besides InvalidTag (e.g. a malformed nonce) is not authentication."""
+    mock_aesgcm.return_value.decrypt.side_effect = ValueError('bad nonce length')
+    envelope = factory.build_envelope('Correct1Horse!Battery', KEY)
+
+    with pytest.raises(EnvelopeError, match='could not be opened'):
+        decrypt_password_envelope(envelope, KEY)
+
+
+def test_a_non_utf8_plaintext_is_rejected():
+    """The envelope opened but what came out is not text — refuse rather than mangle it."""
+    nonce = os.urandom(12)
+    sealed = AESGCM(KEY).encrypt(nonce, b'\xff\xfe\xfd', None)
+    envelope = base64.b64encode(nonce + sealed).decode('ascii')
+
+    with pytest.raises(EnvelopeError, match='not valid UTF-8'):
+        decrypt_password_envelope(envelope, KEY)
+
+
 def test_no_secret_appears_in_the_error_text():
     password = 'Correct1Horse!Battery'
     envelope = factory.build_envelope(password, KEY)