gitea_admin/sientia-dataops-laborious-legacy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

QTZPOC-20

Browse Source 
QTZPOC-20: Implement explicit joblib model flavor and enhance model import workflow with extensive E2E tests.

This commit introduces a first-class joblib model flavor for direct loading and saving of Python pickle artifacts, including custom code within the artifact's code/ directory, bypassing MLflow's flavor loaders. The model import workflow's status database configuration is now streamlined to default to the POSTGRES_* server, with explicit IMPORT_STATUS_DB_* overrides.

Comprehensive end-to-end tests have been added for the model import workflow, covering success paths, gate rejections, digest enforcement, provisioning failures, status record integrity, cleanup, and retry/recovery scenarios using a real-clock environment. New E2
This commit is contained in:
Vitor Pimentel dos Santos
2026-08-26 15:45:33 -03:00
parent dac4f1718f
commit e95e65b72b
32 changed files with 4247 additions and 1040 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scripts/encode.sh
View File

@@ -1,20 +0,0 @@
#!/bin/bash
# Resolve paths from the script's own location so this works whether it is run
# from the repository root or from inside scripts/.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
source "${REPO_ROOT}/venv/bin/activate"
pip install pathspec
pip install pyyaml
echo "
.git" >> "${REPO_ROOT}/.gitignore"
python "${SCRIPT_DIR}/encrypt.py" "${REPO_ROOT}" "${REPO_ROOT}/code" --ignore "${REPO_ROOT}/.gitignore" --chunk-size 100000
sed -i '/.git/d' "${REPO_ROOT}/.gitignore"
xdg-open "${REPO_ROOT}"

113
scripts/encrypt.py
View File

@@ -1,113 +0,0 @@
import argparse
import os
from typing import Any
import yaml # type: ignore
from pathspec import PathSpec
r"""
Usage:
python .\encrypt.py path_to_dir output_file --ignore ignore_file --chunk-size 100000
"""
def load_ignore_patterns(ignore_file, include_library):
# Ensure the .gitignore file exists
if not os.path.exists(ignore_file):
raise FileNotFoundError(f'Ignore file not found at {ignore_file}')
# Load and parse the .gitignore patterns
with open(ignore_file) as file:
patterns = file.readlines()
if not include_library:
patterns.append('**/deploy/library/')
spec = PathSpec.from_lines('gitwildmatch', patterns)
return spec
def is_ignored(file_path, spec):
"""Check if a file should be ignored based on the ignore patterns."""
return spec.match_file(file_path) if spec else False
def encode_file_tree_to_yaml(directory, ignore_file, include_library):
"""Encode the file tree into a single YAML file."""
ignore_patterns = load_ignore_patterns(ignore_file, include_library) if ignore_file else None
file_tree: dict[str, Any] = {}
for root, dirs, files in os.walk(directory):
# Skip ignored directories
dirs[:] = [d for d in dirs if not is_ignored(os.path.join(root, d), ignore_patterns)]
for file in files:
file_path = os.path.join(root, file)
# Skip ignored files
if is_ignored(file_path, ignore_patterns):
continue
# Read file content
try:
with open(file_path, encoding='utf-8') as f:
content = f.read()
except Exception as e:
print(f'Error reading file {file_path}: {e}')
raise
# Create nested dictionary structure
path_parts = os.path.relpath(file_path, directory).split(os.sep)
current_level = file_tree
# all except the last part (the file name)
for part in path_parts[:-1]:
current_level = current_level.setdefault(part, {})
# Add the file and its content
current_level[path_parts[-1]] = content
return yaml.dump(file_tree, default_flow_style=False)
def chunk_and_write_file_tree_to_yaml(yaml_content, output_file, chunk_size=None):
"""Chunk the YAML content and write it to the output file."""
chunks = (
[yaml_content]
if chunk_size is None
else [yaml_content[i : i + chunk_size] for i in range(0, len(yaml_content), chunk_size)]
)
for i, chunk in enumerate(chunks):
chunk_file = f'{output_file}_{i}.yaml'
# Write the file tree to the output YAML file
with open(chunk_file, 'w', encoding='utf-8') as yaml_file:
yaml_file.write(chunk)
def main():
parser = argparse.ArgumentParser(description='Encrypts file tree to yaml file')
parser.add_argument('input_directory', help='Directory to encode')
parser.add_argument('output_yaml_file', help='Output YAML file')
parser.add_argument('--ignore', default=None, help='Path to the ignore file')
parser.add_argument(
'--chunk-size', type=int, default=None, help='Chunk size for the output YAML file'
)
parser.add_argument(
'--library', type=bool, default=False, help='Incude the library in the output YAML file'
)
# Parse arguments
args = parser.parse_args()
# Example usage
directory_to_encode = args.input_directory
ignore_file_path = args.ignore
output_yaml_file = args.output_yaml_file
include_library = args.library
content = encode_file_tree_to_yaml(directory_to_encode, ignore_file_path, include_library)
chunk_and_write_file_tree_to_yaml(content, output_yaml_file, args.chunk_size)
if __name__ == '__main__':
main()

19
scripts/import_model_manual_run.py
View File

@@ -77,13 +77,18 @@ def _line(label: str, value: object) -> str:
# back to localhost, which is right for a service that must boot; it is wrong for a script that
# writes, because a silent fallback is a write to somewhere nobody read out loud.
#
# The import log database: a second Postgres connection, independent of the POSTGRES_* one above it
# in .env. The schema and table are constants of the worker's code, never configuration.
STATUS_DB_HOST = env_required('IMPORT_STATUS_DB_HOST')
STATUS_DB_PORT = int(env_required('IMPORT_STATUS_DB_PORT'))
# The import log database: the POSTGRES_* server from .env, with only the database name of its own,
# because the log lives in the Spring Boot BFF's database rather than in `sientia`. The schema and
# table are constants of the worker's code, never configuration.
#
# `build_import_status_config` also honours IMPORT_STATUS_DB_{HOST,PORT,USER,PASSWORD} as overrides,
# for a deployment whose log sits on another server. This script does not: it is a development tool
# pointed at a real environment, and the override exists for the e2e suite's two containers.
STATUS_DB_HOST = env_required('POSTGRES_HOST')
STATUS_DB_PORT = int(env_required('POSTGRES_PORT'))
STATUS_DB_NAME = env_required('IMPORT_STATUS_DB_NAME')
STATUS_DB_USER = env_required('IMPORT_STATUS_DB_USER')
STATUS_DB_PASSWORD = os.getenv('IMPORT_STATUS_DB_PASSWORD', '')
STATUS_DB_USER = env_required('POSTGRES_USER')
STATUS_DB_PASSWORD = os.getenv('POSTGRES_PASSWORD', '')
STATUS_TABLE = 'public.experiment_run'
# Where the frontend uploads the bundle. The prefix keeps the worker's own default, which is a
@@ -145,7 +150,7 @@ CONFIRM_TARGET = env('IMPORT_SCRIPT_CONFIRM_TARGET')
if CONFIRM_TARGET != STATUS_DB_HOST:
raise RuntimeError(
'refusing to write: the confirmation does not match the resolved status database host. '
f'confirmation={CONFIRM_TARGET!r}, resolved IMPORT_STATUS_DB_HOST={STATUS_DB_HOST!r}. '
f'confirmation={CONFIRM_TARGET!r}, resolved POSTGRES_HOST={STATUS_DB_HOST!r}. '
'Read the targets cell 1 printed, then restate the host here.'
)

27
scripts/inter_arrival.py
View File

@@ -1,27 +0,0 @@
# %%
# Load logs.txt
with open('logs.txt') as file:
lines = file.readlines()
# %%
import re
# Grep "inter-arrival_s=number" with regex
intervals = []
for line in lines:
match = re.search(r'inter-arrival_s=([0-9.]+)', line)
if match:
intervals.append(float(match.group(1)))
# %%
print(intervals)
# %%
import matplotlib.pyplot as plt
plt.plot(intervals)
plt.ylabel('Inter-arrival time (s)')
plt.xlabel('Sample')
plt.title('Inter-arrival time distribution')
plt.show()
# %%

107
scripts/model_convert.ipynb
View File

@@ -1,107 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 23,
"id": "e838ff21",
"metadata": {},
"outputs": [],
"source": [
"import csv\n",
"\n",
"\n",
"def csv_to_tag_lists(csv_path: str) -> dict:\n",
" read_tags = []\n",
" write_tags = []\n",
"\n",
" def to_float(val):\n",
" try:\n",
" return float(str(val).strip())\n",
" except Exception:\n",
" return None\n",
"\n",
" with open(csv_path, newline='', encoding='utf-8') as f:\n",
" reader = csv.DictReader(f)\n",
" for row in reader:\n",
" # Basic normalization\n",
" op = (row.get('operation') or '').strip()\n",
"\n",
" if op == 'READ':\n",
" # Build common tag payload with required mappings\n",
" tag = {\n",
" 'server_id': '1',\n",
" 'tag_address': row.get('opc_tag'),\n",
" 'tag_name': row.get('name'),\n",
" 'data_range': [to_float(row.get('min_value')), to_float(row.get('max_value'))],\n",
" 'aggr_func': row.get('aggregation_func').lower(),\n",
" # keep other fields with their original names\n",
" 'frequency': row.get('frequency'),\n",
" 'local': row.get('local'),\n",
" 'area': row.get('area'),\n",
" 'description': row.get('description'),\n",
" }\n",
"\n",
" read_tags.append(tag)\n",
"\n",
" else:\n",
" tag = {\n",
" 'server_id': '1',\n",
" 'addr': row.get('opc_tag'),\n",
" 'tag_name': row.get('name'),\n",
" 'local': row.get('local'),\n",
" 'area': row.get('area'),\n",
" 'description': row.get('description'),\n",
" }\n",
"\n",
" if op == 'WRITE_PREDICTION':\n",
" tag['type'] = 'prediction'\n",
" write_tags.append(tag)\n",
" elif op == 'WRITE_CONFIDENCE':\n",
" tag['type'] = 'confidence'\n",
" write_tags.append(tag)\n",
" # ignore any other operation values silently\n",
"\n",
" return {'read_tags': read_tags, 'write_tags': write_tags}"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4621cd43",
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"\n",
"file_names = ['Courier - Página1.csv']\n",
"\n",
"for file_name in file_names:\n",
" write_file = file_name.replace('.csv', '.json')\n",
"\n",
" with open(write_file, 'w', encoding='utf-8') as f:\n",
" json.dump(csv_to_tag_lists(file_name), f, indent=2, ensure_ascii=False)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}