SIENTIAPDE-1430: Add option to run all training test scenarios with summary reporting.
- Introduce --all argument to run_training_test.py to execute all available scenarios sequentially. - Refactor single scenario execution into a run_single_scenario function for better modularity and error handling. - Implement a print_summary function to report results for multiple scenario runs.
This commit is contained in:
@@ -238,6 +238,9 @@ Examples:
|
|||||||
|
|
||||||
# Run with a custom CSV data file
|
# Run with a custom CSV data file
|
||||||
python scripts/run_training_test.py --scenario linear-regression-basic --csv docs/other-data.csv
|
python scripts/run_training_test.py --scenario linear-regression-basic --csv docs/other-data.csv
|
||||||
|
|
||||||
|
# Run all scenarios
|
||||||
|
python scripts/run_training_test.py --all
|
||||||
""",
|
""",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
@@ -259,9 +262,112 @@ Examples:
|
|||||||
action='store_true',
|
action='store_true',
|
||||||
help='List all available test scenarios and exit.',
|
help='List all available test scenarios and exit.',
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--all',
|
||||||
|
'-a',
|
||||||
|
action='store_true',
|
||||||
|
help='Run all available test scenarios sequentially.',
|
||||||
|
)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def run_single_scenario(scenario_name: str, csv_path: Path) -> dict:
|
||||||
|
"""Run a single test scenario and return the result.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
scenario_name: Name of the scenario to run.
|
||||||
|
csv_path: Path to the CSV data file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary with scenario result including success status and details.
|
||||||
|
"""
|
||||||
|
result = {
|
||||||
|
'scenario': scenario_name,
|
||||||
|
'success': False,
|
||||||
|
'error': None,
|
||||||
|
'experiment_run_id': None,
|
||||||
|
's3_object_name': None,
|
||||||
|
'workflow_id': None,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Load scenario
|
||||||
|
try:
|
||||||
|
experiment_request = load_scenario(scenario_name)
|
||||||
|
print(f" Loaded scenario: {scenario_name}")
|
||||||
|
except FileNotFoundError as exc:
|
||||||
|
result['error'] = str(exc)
|
||||||
|
return result
|
||||||
|
|
||||||
|
# Upload CSV to MinIO
|
||||||
|
try:
|
||||||
|
uploaded_file_name = upload_to_minio(csv_path)
|
||||||
|
result['s3_object_name'] = uploaded_file_name
|
||||||
|
print(f" Uploaded CSV to MinIO: {uploaded_file_name}")
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
result['error'] = f'Failed to upload file to MinIO: {exc}'
|
||||||
|
return result
|
||||||
|
except FileNotFoundError as exc:
|
||||||
|
result['error'] = str(exc)
|
||||||
|
return result
|
||||||
|
|
||||||
|
# Insert experiment run
|
||||||
|
try:
|
||||||
|
experiment_run_id = insert_experiment_run(uploaded_file_name, experiment_request)
|
||||||
|
result['experiment_run_id'] = experiment_run_id
|
||||||
|
print(f" Created experiment_run with ID: {experiment_run_id}")
|
||||||
|
except psycopg2.Error as exc:
|
||||||
|
result['error'] = f'Database error while inserting experiment_run: {exc}'
|
||||||
|
return result
|
||||||
|
|
||||||
|
# Build and trigger workflow
|
||||||
|
workflow_payload = build_workflow_payload(
|
||||||
|
experiment_run_id=experiment_run_id,
|
||||||
|
file_name=uploaded_file_name,
|
||||||
|
request_data=experiment_request,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
workflow_id = asyncio.run(trigger_temporal_workflow(workflow_payload))
|
||||||
|
result['workflow_id'] = workflow_id
|
||||||
|
result['success'] = True
|
||||||
|
print(f" Workflow started: {workflow_id}")
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
result['error'] = f'Failed to start Temporal workflow: {exc}'
|
||||||
|
return result
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def print_summary(results: list[dict]) -> None:
|
||||||
|
"""Print a summary of all scenario results.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
results: List of result dictionaries from run_single_scenario.
|
||||||
|
"""
|
||||||
|
passed = [r for r in results if r['success']]
|
||||||
|
failed = [r for r in results if not r['success']]
|
||||||
|
|
||||||
|
print('\n' + '=' * 60)
|
||||||
|
print('SUMMARY')
|
||||||
|
print('=' * 60)
|
||||||
|
print(f"Total: {len(results)} | Passed: {len(passed)} | Failed: {len(failed)}")
|
||||||
|
print('=' * 60)
|
||||||
|
|
||||||
|
if passed:
|
||||||
|
print('\n✓ PASSED:')
|
||||||
|
for r in passed:
|
||||||
|
print(f" - {r['scenario']}")
|
||||||
|
|
||||||
|
if failed:
|
||||||
|
print('\n✗ FAILED:')
|
||||||
|
for r in failed:
|
||||||
|
print(f" - {r['scenario']}")
|
||||||
|
if r['error']:
|
||||||
|
print(f" Error: {r['error']}")
|
||||||
|
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
args = parse_args()
|
args = parse_args()
|
||||||
|
|
||||||
@@ -276,58 +382,49 @@ def main() -> None:
|
|||||||
print(f'No scenarios found in {TEST_SCENARIOS_DIR}')
|
print(f'No scenarios found in {TEST_SCENARIOS_DIR}')
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
# Require scenario argument if not listing
|
# Run all scenarios if requested
|
||||||
|
if args.all:
|
||||||
|
scenarios = list_available_scenarios()
|
||||||
|
if not scenarios:
|
||||||
|
print(f'No scenarios found in {TEST_SCENARIOS_DIR}', file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(f'Running {len(scenarios)} scenarios...\n')
|
||||||
|
results = []
|
||||||
|
|
||||||
|
for i, scenario in enumerate(scenarios, 1):
|
||||||
|
print(f'[{i}/{len(scenarios)}] Running scenario: {scenario}')
|
||||||
|
result = run_single_scenario(scenario, args.csv)
|
||||||
|
results.append(result)
|
||||||
|
status = '✓' if result['success'] else '✗'
|
||||||
|
print(f'[{i}/{len(scenarios)}] {status} {scenario}\n')
|
||||||
|
|
||||||
|
print_summary(results)
|
||||||
|
|
||||||
|
# Exit with error code if any scenario failed
|
||||||
|
failed_count = sum(1 for r in results if not r['success'])
|
||||||
|
sys.exit(1 if failed_count > 0 else 0)
|
||||||
|
|
||||||
|
# Require scenario argument if not listing or running all
|
||||||
if not args.scenario:
|
if not args.scenario:
|
||||||
print('Error: --scenario is required. Use --list to see available scenarios.', file=sys.stderr)
|
print('Error: --scenario or --all is required. Use --list to see available scenarios.', file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
# Load scenario
|
# Run single scenario
|
||||||
try:
|
print(f'Running scenario: {args.scenario}')
|
||||||
experiment_request = load_scenario(args.scenario)
|
result = run_single_scenario(args.scenario, args.csv)
|
||||||
print(f"Loaded scenario: {args.scenario}")
|
|
||||||
except FileNotFoundError as exc:
|
|
||||||
print(str(exc), file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# Upload CSV to MinIO
|
if not result['success']:
|
||||||
try:
|
print(f"Error: {result['error']}", file=sys.stderr)
|
||||||
uploaded_file_name = upload_to_minio(args.csv)
|
|
||||||
print(f"Uploaded CSV to MinIO: {uploaded_file_name}")
|
|
||||||
except subprocess.CalledProcessError as exc:
|
|
||||||
print(f'Failed to upload file to MinIO: {exc}', file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
except FileNotFoundError as exc:
|
|
||||||
print(str(exc), file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# Insert experiment run
|
|
||||||
try:
|
|
||||||
experiment_run_id = insert_experiment_run(uploaded_file_name, experiment_request)
|
|
||||||
print(f"Created experiment_run with ID: {experiment_run_id}")
|
|
||||||
except psycopg2.Error as exc:
|
|
||||||
print(f'Database error while inserting experiment_run: {exc}', file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# Build and trigger workflow
|
|
||||||
workflow_payload = build_workflow_payload(
|
|
||||||
experiment_run_id=experiment_run_id,
|
|
||||||
file_name=uploaded_file_name,
|
|
||||||
request_data=experiment_request,
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
workflow_id = asyncio.run(trigger_temporal_workflow(workflow_payload))
|
|
||||||
except Exception as exc: # noqa: BLE001
|
|
||||||
print(f'Failed to start Temporal workflow: {exc}', file=sys.stderr)
|
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
print(
|
print(
|
||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
'scenario': args.scenario,
|
'scenario': result['scenario'],
|
||||||
'experiment_run_id': experiment_run_id,
|
'experiment_run_id': result['experiment_run_id'],
|
||||||
's3_object_name': uploaded_file_name,
|
's3_object_name': result['s3_object_name'],
|
||||||
'workflow_id': workflow_id,
|
'workflow_id': result['workflow_id'],
|
||||||
},
|
},
|
||||||
indent=2,
|
indent=2,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user