diff --git a/scripts/run_training_test.py b/scripts/run_training_test.py index f90a8a7..2a93c7b 100644 --- a/scripts/run_training_test.py +++ b/scripts/run_training_test.py @@ -238,6 +238,9 @@ Examples: # Run with a custom CSV data file python scripts/run_training_test.py --scenario linear-regression-basic --csv docs/other-data.csv + + # Run all scenarios + python scripts/run_training_test.py --all """, ) parser.add_argument( @@ -259,9 +262,112 @@ Examples: action='store_true', help='List all available test scenarios and exit.', ) + parser.add_argument( + '--all', + '-a', + action='store_true', + help='Run all available test scenarios sequentially.', + ) return parser.parse_args() +def run_single_scenario(scenario_name: str, csv_path: Path) -> dict: + """Run a single test scenario and return the result. + + Args: + scenario_name: Name of the scenario to run. + csv_path: Path to the CSV data file. + + Returns: + Dictionary with scenario result including success status and details. + """ + result = { + 'scenario': scenario_name, + 'success': False, + 'error': None, + 'experiment_run_id': None, + 's3_object_name': None, + 'workflow_id': None, + } + + # Load scenario + try: + experiment_request = load_scenario(scenario_name) + print(f" Loaded scenario: {scenario_name}") + except FileNotFoundError as exc: + result['error'] = str(exc) + return result + + # Upload CSV to MinIO + try: + uploaded_file_name = upload_to_minio(csv_path) + result['s3_object_name'] = uploaded_file_name + print(f" Uploaded CSV to MinIO: {uploaded_file_name}") + except subprocess.CalledProcessError as exc: + result['error'] = f'Failed to upload file to MinIO: {exc}' + return result + except FileNotFoundError as exc: + result['error'] = str(exc) + return result + + # Insert experiment run + try: + experiment_run_id = insert_experiment_run(uploaded_file_name, experiment_request) + result['experiment_run_id'] = experiment_run_id + print(f" Created experiment_run with ID: {experiment_run_id}") + except psycopg2.Error as exc: + result['error'] = f'Database error while inserting experiment_run: {exc}' + return result + + # Build and trigger workflow + workflow_payload = build_workflow_payload( + experiment_run_id=experiment_run_id, + file_name=uploaded_file_name, + request_data=experiment_request, + ) + + try: + workflow_id = asyncio.run(trigger_temporal_workflow(workflow_payload)) + result['workflow_id'] = workflow_id + result['success'] = True + print(f" Workflow started: {workflow_id}") + except Exception as exc: # noqa: BLE001 + result['error'] = f'Failed to start Temporal workflow: {exc}' + return result + + return result + + +def print_summary(results: list[dict]) -> None: + """Print a summary of all scenario results. + + Args: + results: List of result dictionaries from run_single_scenario. + """ + passed = [r for r in results if r['success']] + failed = [r for r in results if not r['success']] + + print('\n' + '=' * 60) + print('SUMMARY') + print('=' * 60) + print(f"Total: {len(results)} | Passed: {len(passed)} | Failed: {len(failed)}") + print('=' * 60) + + if passed: + print('\nāœ“ PASSED:') + for r in passed: + print(f" - {r['scenario']}") + + if failed: + print('\nāœ— FAILED:') + for r in failed: + print(f" - {r['scenario']}") + if r['error']: + print(f" Error: {r['error']}") + + print() + + def main() -> None: args = parse_args() @@ -276,58 +382,49 @@ def main() -> None: print(f'No scenarios found in {TEST_SCENARIOS_DIR}') sys.exit(0) - # Require scenario argument if not listing + # Run all scenarios if requested + if args.all: + scenarios = list_available_scenarios() + if not scenarios: + print(f'No scenarios found in {TEST_SCENARIOS_DIR}', file=sys.stderr) + sys.exit(1) + + print(f'Running {len(scenarios)} scenarios...\n') + results = [] + + for i, scenario in enumerate(scenarios, 1): + print(f'[{i}/{len(scenarios)}] Running scenario: {scenario}') + result = run_single_scenario(scenario, args.csv) + results.append(result) + status = 'āœ“' if result['success'] else 'āœ—' + print(f'[{i}/{len(scenarios)}] {status} {scenario}\n') + + print_summary(results) + + # Exit with error code if any scenario failed + failed_count = sum(1 for r in results if not r['success']) + sys.exit(1 if failed_count > 0 else 0) + + # Require scenario argument if not listing or running all if not args.scenario: - print('Error: --scenario is required. Use --list to see available scenarios.', file=sys.stderr) + print('Error: --scenario or --all is required. Use --list to see available scenarios.', file=sys.stderr) sys.exit(1) - # Load scenario - try: - experiment_request = load_scenario(args.scenario) - print(f"Loaded scenario: {args.scenario}") - except FileNotFoundError as exc: - print(str(exc), file=sys.stderr) - sys.exit(1) + # Run single scenario + print(f'Running scenario: {args.scenario}') + result = run_single_scenario(args.scenario, args.csv) - # Upload CSV to MinIO - try: - uploaded_file_name = upload_to_minio(args.csv) - print(f"Uploaded CSV to MinIO: {uploaded_file_name}") - except subprocess.CalledProcessError as exc: - print(f'Failed to upload file to MinIO: {exc}', file=sys.stderr) - sys.exit(1) - except FileNotFoundError as exc: - print(str(exc), file=sys.stderr) - sys.exit(1) - - # Insert experiment run - try: - experiment_run_id = insert_experiment_run(uploaded_file_name, experiment_request) - print(f"Created experiment_run with ID: {experiment_run_id}") - except psycopg2.Error as exc: - print(f'Database error while inserting experiment_run: {exc}', file=sys.stderr) - sys.exit(1) - - # Build and trigger workflow - workflow_payload = build_workflow_payload( - experiment_run_id=experiment_run_id, - file_name=uploaded_file_name, - request_data=experiment_request, - ) - - try: - workflow_id = asyncio.run(trigger_temporal_workflow(workflow_payload)) - except Exception as exc: # noqa: BLE001 - print(f'Failed to start Temporal workflow: {exc}', file=sys.stderr) + if not result['success']: + print(f"Error: {result['error']}", file=sys.stderr) sys.exit(1) print( json.dumps( { - 'scenario': args.scenario, - 'experiment_run_id': experiment_run_id, - 's3_object_name': uploaded_file_name, - 'workflow_id': workflow_id, + 'scenario': result['scenario'], + 'experiment_run_id': result['experiment_run_id'], + 's3_object_name': result['s3_object_name'], + 'workflow_id': result['workflow_id'], }, indent=2, )