gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1430: Introduce comprehensive integration testing with JSON-based scenarios and detailed README documentation. Enhance training workflow to support advanced model configurations, including polynomial regression with mandatory scaler validation. Ensure robust prediction handling by calculating training predictions (y_train_pred) before denormalization and automatically configuring datetime indices for time-series operations.

Browse Source
This commit is contained in:
Bruno Domingues
2025-12-18 17:05:10 -03:00
parent 6e8f87b2a3
commit 06fd08dc70
18 changed files with 631 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
scripts/run_training_test.py
View File

@@ -15,6 +15,7 @@ Prerequisites:
from __future__ import annotations
import argparse
import asyncio
import json
import os
@@ -24,8 +25,8 @@ import uuid
from datetime import datetime, timedelta
from pathlib import Path
from dotenv import load_dotenv
import psycopg2
from dotenv import load_dotenv
from psycopg2.extras import Json
from temporalio import client
@@ -37,7 +38,8 @@ if ENV_PATH.exists():
load_dotenv(dotenv_path=ENV_PATH)
DOCS_PATH = Path('docs/test-model-data.csv')
DEFAULT_CSV_PATH = Path('docs/test-model-data.csv')
TEST_SCENARIOS_DIR = PROJECT_ROOT / 'docs' / 'test-scenarios'
MINIO_ALIAS = 'suse'
MINIO_BUCKET = 'model-training'
@@ -54,26 +56,45 @@ TEMPORAL_NAMESPACE = os.getenv('TEMPORAL_NAMESPACE')
TRAIN_TASK_QUEUE = os.getenv('TRAIN_TASK_QUEUE')
TEMPORAL_WORKFLOW = 'train_model'
BASE_REQUEST_DATA = {
'experimentName': 'model-manager-test-01',
'username': 'bruno.domingues@aignosi.com.br',
'modelType': 'Linear Regression',
'targetVariable': '03CV020/CORRENTE_N_M1_PV(Value)',
'variableColumns': ['303-WIT-200(Value)'],
'lagTrain': 0,
'lagVal': 0,
'remStaticWin': False,
'lowLim': {},
'uppLim': {},
'window': 0,
'useScaler': False,
'includeAr': False,
'trainSize': 80,
'shuffle': True,
'lineSeparator': ',',
'decimalSeparator': '.',
'removedIntervals': [],
}
def list_available_scenarios() -> list[str]:
"""List all available test scenario files."""
if not TEST_SCENARIOS_DIR.exists():
return []
return sorted([f.stem for f in TEST_SCENARIOS_DIR.glob('*.json')])
def load_scenario(scenario_name: str) -> dict:
"""Load a test scenario from JSON file.
Args:
scenario_name: Name of the scenario (without .json extension)
or full path to a JSON file.
Returns:
Dictionary with scenario data.
Raises:
FileNotFoundError: If scenario file doesn't exist.
"""
# Check if it's a full path
scenario_path = Path(scenario_name)
if scenario_path.suffix == '.json' and scenario_path.exists():
with open(scenario_path) as f:
return json.load(f)
# Otherwise, look in the test-scenarios directory
scenario_file = TEST_SCENARIOS_DIR / f'{scenario_name}.json'
if not scenario_file.exists():
available = list_available_scenarios()
available_str = ', '.join(available) if available else 'none'
raise FileNotFoundError(
f"Scenario '{scenario_name}' not found at {scenario_file}.\n"
f'Available scenarios: {available_str}'
)
with open(scenario_file) as f:
return json.load(f)
def _ensure_source_file(path: Path) -> None:
@@ -126,7 +147,7 @@ def insert_experiment_run(file_name: str, request_data: dict) -> int:
(
request_data['experimentName'],
request_data['username'],
'ORCHESTRATOR_REQUEST_SENT',
'ORCHESTRATOR_WAITING_PROC',
now,
now,
MINIO_BUCKET,
@@ -149,7 +170,6 @@ def build_workflow_payload(
'experiment_run_id': experiment_run_id,
'experiment_name': request_data['experimentName'],
'username': request_data['username'],
'model_type': request_data['modelType'],
'target_variable': request_data['targetVariable'],
'variable_columns': request_data['variableColumns'],
'lag_train': request_data['lagTrain'],
@@ -167,6 +187,15 @@ def build_workflow_payload(
'line_separator': request_data['lineSeparator'],
'decimal_separator': request_data['decimalSeparator'],
'removed_intervals': request_data['removedIntervals'],
# New parameters
'model_name': request_data.get('modelName', 'Linear Regression'),
'degree': request_data.get('degree', 1),
'interaction_only': request_data.get('interactionOnly', False),
'nan_treatment': request_data.get('nanTreatment', 'drop'),
'start_date': request_data.get('startDate'),
'end_date': request_data.get('endDate'),
'scaler_name': request_data.get('scalerName', 'None'),
'support_filters': request_data.get('supportFilters', {}),
}
@@ -191,9 +220,79 @@ async def trigger_temporal_workflow(workflow_input: dict) -> str:
return workflow_id
def parse_args() -> argparse.Namespace:
"""Parse command line arguments."""
parser = argparse.ArgumentParser(
description='Run training workflow tests with different scenarios.',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
# List available scenarios
python scripts/run_training_test.py --list
# Run a specific scenario
python scripts/run_training_test.py --scenario linear-regression-basic
# Run with a custom JSON file
python scripts/run_training_test.py --scenario /path/to/custom-scenario.json
# Run with a custom CSV data file
python scripts/run_training_test.py --scenario linear-regression-basic --csv docs/other-data.csv
""",
)
parser.add_argument(
'--scenario',
'-s',
type=str,
help='Name of the test scenario (without .json) or path to a JSON file.',
)
parser.add_argument(
'--csv',
'-c',
type=Path,
default=DEFAULT_CSV_PATH,
help=f'Path to the CSV data file (default: {DEFAULT_CSV_PATH}).',
)
parser.add_argument(
'--list',
'-l',
action='store_true',
help='List all available test scenarios and exit.',
)
return parser.parse_args()
def main() -> None:
args = parse_args()
# List scenarios and exit if requested
if args.list:
scenarios = list_available_scenarios()
if scenarios:
print('Available test scenarios:')
for scenario in scenarios:
print(f' - {scenario}')
else:
print(f'No scenarios found in {TEST_SCENARIOS_DIR}')
sys.exit(0)
# Require scenario argument if not listing
if not args.scenario:
print('Error: --scenario is required. Use --list to see available scenarios.', file=sys.stderr)
sys.exit(1)
# Load scenario
try:
uploaded_file_name = upload_to_minio(DOCS_PATH)
experiment_request = load_scenario(args.scenario)
print(f"Loaded scenario: {args.scenario}")
except FileNotFoundError as exc:
print(str(exc), file=sys.stderr)
sys.exit(1)
# Upload CSV to MinIO
try:
uploaded_file_name = upload_to_minio(args.csv)
print(f"Uploaded CSV to MinIO: {uploaded_file_name}")
except subprocess.CalledProcessError as exc:
print(f'Failed to upload file to MinIO: {exc}', file=sys.stderr)
sys.exit(1)
@@ -201,14 +300,15 @@ def main() -> None:
print(str(exc), file=sys.stderr)
sys.exit(1)
experiment_request = BASE_REQUEST_DATA.copy()
# Insert experiment run
try:
experiment_run_id = insert_experiment_run(uploaded_file_name, experiment_request)
print(f"Created experiment_run with ID: {experiment_run_id}")
except psycopg2.Error as exc:
print(f'Database error while inserting experiment_run: {exc}', file=sys.stderr)
sys.exit(1)
# Build and trigger workflow
workflow_payload = build_workflow_payload(
experiment_run_id=experiment_run_id,
file_name=uploaded_file_name,
@@ -224,6 +324,7 @@ def main() -> None:
print(
json.dumps(
{
'scenario': args.scenario,
'experiment_run_id': experiment_run_id,
's3_object_name': uploaded_file_name,
'workflow_id': workflow_id,