gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1579: Lint fixes and formatting

Browse Source
This commit is contained in:
Kou Kinoshita
2026-02-18 17:31:58 -03:00
parent c6d6f94e05
commit 43cbc31e10
7 changed files with 151 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

192
scripts/run_training_test.py
View File

@@ -39,7 +39,6 @@ from dotenv import load_dotenv
from psycopg2.extras import Json
from temporalio import client
# Carrega variáveis de ambiente do arquivo .env na raiz do projeto
PROJECT_ROOT = Path(__file__).resolve().parent.parent
ENV_PATH = PROJECT_ROOT / '.env'
@@ -325,9 +324,7 @@ def run_local_pipeline(
try:
train_result = training_repository.train(file_content, train_params)
train_result = training_repository.after_train_calculation(
train_params, train_result
)
train_result = training_repository.after_train_calculation(train_params, train_result)
except Exception as e:
result['error'] = str(e)
raise # re-raise so caller gets full traceback for diagnosis
@@ -448,7 +445,7 @@ def run_single_scenario(scenario_name: str, csv_path: Path) -> dict:
# Load scenario
try:
experiment_request = load_scenario(scenario_name)
print(f" Loaded scenario: {scenario_name}")
print(f' Loaded scenario: {scenario_name}')
except FileNotFoundError as exc:
result['error'] = str(exc)
return result
@@ -459,7 +456,7 @@ def run_single_scenario(scenario_name: str, csv_path: Path) -> dict:
try:
uploaded_file_name = upload_to_minio(csv_path)
result['s3_object_name'] = uploaded_file_name
print(f" Uploaded CSV to MinIO: {uploaded_file_name}")
print(f' Uploaded CSV to MinIO: {uploaded_file_name}')
except subprocess.CalledProcessError as exc:
result['error'] = f'Failed to upload file to MinIO: {exc}'
return result
@@ -471,7 +468,7 @@ def run_single_scenario(scenario_name: str, csv_path: Path) -> dict:
try:
experiment_run_id = insert_experiment_run(uploaded_file_name, experiment_request)
result['experiment_run_id'] = experiment_run_id
print(f" Created experiment_run with ID: {experiment_run_id}")
print(f' Created experiment_run with ID: {experiment_run_id}')
except psycopg2.Error as exc:
result['error'] = f'Database error while inserting experiment_run: {exc}'
return result
@@ -487,7 +484,7 @@ def run_single_scenario(scenario_name: str, csv_path: Path) -> dict:
workflow_id = asyncio.run(trigger_temporal_workflow(workflow_payload))
result['workflow_id'] = workflow_id
result['success'] = True
print(f" Workflow started: {workflow_id}")
print(f' Workflow started: {workflow_id}')
except Exception as exc: # noqa: BLE001
result['error'] = f'Failed to start Temporal workflow: {exc}'
return result
@@ -507,109 +504,100 @@ def print_summary(results: list[dict]) -> None:
print('\n' + '=' * 60)
print('SUMMARY')
print('=' * 60)
print(f"Total: {len(results)} | Passed: {len(passed)} | Failed: {len(failed)}")
print(f'Total: {len(results)} | Passed: {len(passed)} | Failed: {len(failed)}')
print('=' * 60)
if passed:
print('\n✓ PASSED:')
for r in passed:
print(f" - {r['scenario']}")
print(f' - {r["scenario"]}')
if failed:
print('\n✗ FAILED:')
for r in failed:
print(f" - {r['scenario']}")
print(f' - {r["scenario"]}')
if r['error']:
print(f" Error: {r['error']}")
print(f' Error: {r["error"]}')
print()
def main() -> None:
args = parse_args()
def _handle_list_scenarios() -> None:
"""Print available scenarios and exit."""
scenarios = list_available_scenarios()
if scenarios:
print('Available test scenarios:')
for scenario in scenarios:
print(f' - {scenario}')
else:
print(f'No scenarios found in {TEST_SCENARIOS_DIR}')
sys.exit(0)
# List scenarios and exit if requested
if args.list:
scenarios = list_available_scenarios()
if scenarios:
print('Available test scenarios:')
for scenario in scenarios:
print(f' - {scenario}')
else:
print(f'No scenarios found in {TEST_SCENARIOS_DIR}')
sys.exit(0)
# Run all scenarios if requested
if args.all:
scenarios = list_available_scenarios()
if not scenarios:
print(f'No scenarios found in {TEST_SCENARIOS_DIR}', file=sys.stderr)
sys.exit(1)
print(f'Running {len(scenarios)} scenarios...\n')
results = []
for i, scenario in enumerate(scenarios, 1):
print(f'[{i}/{len(scenarios)}] Running scenario: {scenario}')
result = run_single_scenario(scenario, args.csv)
results.append(result)
status = '✓' if result['success'] else '✗'
print(f'[{i}/{len(scenarios)}] {status} {scenario}\n')
print_summary(results)
# Exit with error code if any scenario failed
failed_count = sum(1 for r in results if not r['success'])
sys.exit(1 if failed_count > 0 else 0)
# Validate-only mode: no external services
if args.validate_only:
if not args.scenario:
print('Error: --scenario is required with --validate-only.', file=sys.stderr)
sys.exit(1)
result = run_validate_only(args.scenario)
if result['success']:
print(f"Validation OK: {result['scenario']}")
else:
print(f"Validation failed: {result['error']}", file=sys.stderr)
sys.exit(1)
return
# Local pipeline: same code path as worker, full traceback on error
if args.local:
if not args.scenario:
print('Error: --scenario is required with --local.', file=sys.stderr)
sys.exit(1)
print(f'Running local pipeline: {args.scenario} (CSV: {args.csv})')
result = run_local_pipeline(
args.scenario,
args.csv,
save_mlflow=args.local_save_mlflow,
)
if not result['success']:
print(f"Error: {result['error']}", file=sys.stderr)
sys.exit(1)
out = {'scenario': result['scenario'], 'success': True}
if result.get('run_name') is not None:
out['run_name'] = result['run_name']
if result.get('run_dir') is not None:
out['run_dir'] = result['run_dir']
print(json.dumps(out, indent=2))
return
# Require scenario argument if not listing or running all
if not args.scenario:
print('Error: --scenario or --all is required. Use --list to see available scenarios.', file=sys.stderr)
def _handle_run_all(args: argparse.Namespace) -> None:
"""Run all scenarios and exit with appropriate code."""
scenarios = list_available_scenarios()
if not scenarios:
print(f'No scenarios found in {TEST_SCENARIOS_DIR}', file=sys.stderr)
sys.exit(1)
# Run single scenario (MinIO + Postgres + Temporal)
print(f'Running {len(scenarios)} scenarios...\n')
results = []
for i, scenario in enumerate(scenarios, 1):
print(f'[{i}/{len(scenarios)}] Running scenario: {scenario}')
result = run_single_scenario(scenario, args.csv)
results.append(result)
status = '✓' if result['success'] else '✗'
print(f'[{i}/{len(scenarios)}] {status} {scenario}\n')
print_summary(results)
failed_count = sum(1 for r in results if not r['success'])
sys.exit(1 if failed_count > 0 else 0)
def _handle_validate_only(args: argparse.Namespace) -> None:
"""Validate scenario parameters only and exit."""
if not args.scenario:
print('Error: --scenario is required with --validate-only.', file=sys.stderr)
sys.exit(1)
result = run_validate_only(args.scenario)
if result['success']:
print(f'Validation OK: {result["scenario"]}')
else:
print(f'Validation failed: {result["error"]}', file=sys.stderr)
sys.exit(1)
sys.exit(0)
def _handle_local(args: argparse.Namespace) -> None:
"""Run local pipeline and exit."""
if not args.scenario:
print('Error: --scenario is required with --local.', file=sys.stderr)
sys.exit(1)
print(f'Running local pipeline: {args.scenario} (CSV: {args.csv})')
result = run_local_pipeline(
args.scenario,
args.csv,
save_mlflow=args.local_save_mlflow,
)
if not result['success']:
print(f'Error: {result["error"]}', file=sys.stderr)
sys.exit(1)
out = {'scenario': result['scenario'], 'success': True}
if result.get('run_name') is not None:
out['run_name'] = result['run_name']
if result.get('run_dir') is not None:
out['run_dir'] = result['run_dir']
print(json.dumps(out, indent=2))
def _handle_single_scenario(args: argparse.Namespace) -> None:
"""Run one scenario (MinIO + Postgres + Temporal) and print result."""
print(f'Running scenario: {args.scenario}')
result = run_single_scenario(args.scenario, args.csv)
if not result['success']:
print(f"Error: {result['error']}", file=sys.stderr)
print(f'Error: {result["error"]}', file=sys.stderr)
sys.exit(1)
print(
json.dumps(
{
@@ -623,5 +611,31 @@ def main() -> None:
)
def main() -> None:
args = parse_args()
if args.list:
_handle_list_scenarios()
if args.all:
_handle_run_all(args)
if args.validate_only:
_handle_validate_only(args)
if args.local:
_handle_local(args)
return
if not args.scenario:
print(
'Error: --scenario or --all is required. Use --list to see available scenarios.',
file=sys.stderr,
)
sys.exit(1)
_handle_single_scenario(args)
if __name__ == '__main__':
main()