vitor-aignosi
dac4f1718f
QTZPOC-20: Implement model import workflow for .sientia bundles Introduces the import_model workflow and its associated activities to enable importing .sientia bundles into MLflow and MongoDB. This includes: * New core workflow and activity logic for handling bundle download, decryption, validation, MLflow provisioning (experiments, runs, registered versions), and MongoDB model document creation. * Comprehensive E2E and unit test suites, including a dedicated test harness and a manual run script for development. * Updated .env.example and README.md with new configurations and usage instructions. * Ensures correct MLflow model source resolution for legacy loader compatibility and robust MongoDB document management. * Refactors existing utility scripts into a scripts/ directory.
108 lines
3.4 KiB
Plaintext
108 lines
3.4 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 23,
|
|
"id": "e838ff21",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import csv\n",
|
|
"\n",
|
|
"\n",
|
|
"def csv_to_tag_lists(csv_path: str) -> dict:\n",
|
|
" read_tags = []\n",
|
|
" write_tags = []\n",
|
|
"\n",
|
|
" def to_float(val):\n",
|
|
" try:\n",
|
|
" return float(str(val).strip())\n",
|
|
" except Exception:\n",
|
|
" return None\n",
|
|
"\n",
|
|
" with open(csv_path, newline='', encoding='utf-8') as f:\n",
|
|
" reader = csv.DictReader(f)\n",
|
|
" for row in reader:\n",
|
|
" # Basic normalization\n",
|
|
" op = (row.get('operation') or '').strip()\n",
|
|
"\n",
|
|
" if op == 'READ':\n",
|
|
" # Build common tag payload with required mappings\n",
|
|
" tag = {\n",
|
|
" 'server_id': '1',\n",
|
|
" 'tag_address': row.get('opc_tag'),\n",
|
|
" 'tag_name': row.get('name'),\n",
|
|
" 'data_range': [to_float(row.get('min_value')), to_float(row.get('max_value'))],\n",
|
|
" 'aggr_func': row.get('aggregation_func').lower(),\n",
|
|
" # keep other fields with their original names\n",
|
|
" 'frequency': row.get('frequency'),\n",
|
|
" 'local': row.get('local'),\n",
|
|
" 'area': row.get('area'),\n",
|
|
" 'description': row.get('description'),\n",
|
|
" }\n",
|
|
"\n",
|
|
" read_tags.append(tag)\n",
|
|
"\n",
|
|
" else:\n",
|
|
" tag = {\n",
|
|
" 'server_id': '1',\n",
|
|
" 'addr': row.get('opc_tag'),\n",
|
|
" 'tag_name': row.get('name'),\n",
|
|
" 'local': row.get('local'),\n",
|
|
" 'area': row.get('area'),\n",
|
|
" 'description': row.get('description'),\n",
|
|
" }\n",
|
|
"\n",
|
|
" if op == 'WRITE_PREDICTION':\n",
|
|
" tag['type'] = 'prediction'\n",
|
|
" write_tags.append(tag)\n",
|
|
" elif op == 'WRITE_CONFIDENCE':\n",
|
|
" tag['type'] = 'confidence'\n",
|
|
" write_tags.append(tag)\n",
|
|
" # ignore any other operation values silently\n",
|
|
"\n",
|
|
" return {'read_tags': read_tags, 'write_tags': write_tags}"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "4621cd43",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import json\n",
|
|
"\n",
|
|
"file_names = ['Courier - Página1.csv']\n",
|
|
"\n",
|
|
"for file_name in file_names:\n",
|
|
" write_file = file_name.replace('.csv', '.json')\n",
|
|
"\n",
|
|
" with open(write_file, 'w', encoding='utf-8') as f:\n",
|
|
" json.dump(csv_to_tag_lists(file_name), f, indent=2, ensure_ascii=False)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "venv",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.13"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|