gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1646

Browse Source 
SIENTIAPDE-1646: Remove validate.sh script, refactor TemporalManager for improved readability, and update temporal schedule tests to explicitly define workflow types and align task queue naming conventions.
This commit is contained in:
vitor-aignosi
2026-07-21 10:17:30 -03:00
parent 7b845a7de4
commit a4b6f6aa88
3 changed files with 38 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
orchestrator/activities/temporal_manager.py
View File

@@ -27,6 +27,7 @@ with workflow.unsafe.imports_passed_through():
RUNTIME_WORKFLOWS = ['predictions_batch', 'minimal_retrain', 'drift', 'simple_metrics']
class TemporalManager(SientiaMonitoring):
"""
Temporal workflow and schedule management activity.
@@ -216,7 +217,11 @@ class TemporalManager(SientiaMonitoring):
f'{json.dumps(schedule, indent=4, sort_keys=True)}', metadata=metadata
)
runtime_name = schedule.get('runtime', 'legacy') if workflow_type in RUNTIME_WORKFLOWS else None
runtime_name = (
schedule.get('runtime', 'legacy')
if workflow_type in RUNTIME_WORKFLOWS
else None
)
task_queue_name = build_queue_name(workflow_type, runtime_name)
schedule['task_queue'] = task_queue_name
@@ -322,7 +327,11 @@ class TemporalManager(SientiaMonitoring):
raise ValueError(f'Schedule {schedule_name} not found')
workflow_type = schedule['workflow_type']
runtime_name = schedule.get('runtime', 'legacy') if workflow_type in RUNTIME_WORKFLOWS else None
runtime_name = (
schedule.get('runtime', 'legacy')
if workflow_type in RUNTIME_WORKFLOWS
else None
)
schedule['task_queue'] = build_queue_name(workflow_type, runtime_name)
# fmt: off

38
tests/orchestrator/activities/test_temporal_manager.py
View File

@@ -206,7 +206,7 @@ async def test_create_schedule(
'test-workflow',
input_data['schedules']['scouter']['test-schedule'],
id='test-schedule',
task_queue='test-workflow-legacy-queue',
task_queue='test-workflow-queue',
execution_timeout=timedelta(seconds=100),
run_timeout=timedelta(seconds=100),
task_timeout=timedelta(seconds=100),
@@ -216,7 +216,7 @@ async def test_create_schedule(
'test-workflow',
input_data['schedules']['scouter']['test-schedule-invalid-frequency'],
id='test-schedule-invalid-frequency',
task_queue='test-workflow-legacy-queue',
task_queue='test-workflow-queue',
execution_timeout=timedelta(seconds=400),
run_timeout=timedelta(seconds=400),
task_timeout=timedelta(seconds=400),
@@ -226,7 +226,7 @@ async def test_create_schedule(
'test-workflow',
input_data['schedules']['laborious']['test-schedule-laborious'],
id='test-schedule-laborious',
task_queue='test-workflow-legacy-queue',
task_queue='test-workflow-queue',
execution_timeout=timedelta(seconds=500),
run_timeout=timedelta(seconds=500),
task_timeout=timedelta(seconds=500),
@@ -350,10 +350,24 @@ async def test_update_schedules(
input_data = {
'schedules': {
'scouter': {
'test-schedule': {'frequency': '1m', 'data': {'test': 'test'}},
'test-schedule_no_handler': {'frequency': '1m', 'data': {'test': 'test'}},
'test-schedule': {
'workflow_type': 'scouter',
'frequency': '1m',
'data': {'test': 'test'},
},
'test-schedule_no_handler': {
'workflow_type': 'scouter',
'frequency': '1m',
'data': {'test': 'test'},
},
},
'laborious': {
'test-schedule-laborious': {
'workflow_type': 'laborious',
'frequency': '2m',
'data': {'test': 'test'},
}
},
'laborious': {'test-schedule-laborious': {'frequency': '2m', 'data': {'test': 'test'}}},
}
}
@@ -514,7 +528,7 @@ async def test_create_schedules_default_runtime_legacy_queue(
'test-schedule': {
'model_id': 1,
'model_name': 'test-model-name',
'workflow_type': 'scouter',
'workflow_type': 'drift',
'frequency': '1m',
'data': {'test': 'test'},
}
@@ -527,10 +541,10 @@ async def test_create_schedules_default_runtime_legacy_queue(
await temporal_manager.create_schedules(input_data)
mock_schedule_action_start_workflow.assert_called_once_with(
'scouter',
'drift',
input_data['schedules']['scouter']['test-schedule'],
id='test-schedule',
task_queue='scouter-legacy-queue',
task_queue='drift-legacy-queue',
execution_timeout=timedelta(seconds=300),
run_timeout=timedelta(seconds=300),
task_timeout=timedelta(seconds=300),
@@ -562,7 +576,7 @@ async def test_create_schedules_tenant_runtime_queue(
'test-schedule': {
'model_id': 1,
'model_name': 'test-model-name',
'workflow_type': 'scouter',
'workflow_type': 'drift',
'frequency': '1m',
'runtime': 'tenant-x',
'data': {'test': 'test'},
@@ -576,10 +590,10 @@ async def test_create_schedules_tenant_runtime_queue(
await temporal_manager.create_schedules(input_data)
mock_schedule_action_start_workflow.assert_called_once_with(
'scouter',
'drift',
input_data['schedules']['scouter']['test-schedule'],
id='test-schedule',
task_queue='scouter-tenant-x-queue',
task_queue='drift-tenant-x-queue',
execution_timeout=timedelta(seconds=300),
run_timeout=timedelta(seconds=300),
task_timeout=timedelta(seconds=300),

124
validate.sh
View File

@@ -1,124 +0,0 @@
#!/bin/bash
# Model Manager Code Validation Script
# This script runs all code quality checks before committing or deploying
set -e # Exit on any error
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Args
FIX_MODE=false
while [[ $# -gt 0 ]]; do
case "$1" in
--fix)
FIX_MODE=true
shift
;;
-h|--help)
echo "Usage: $0 [--fix]"
echo " --fix Apply Ruff auto-fixes (format and lint fixes)."
exit 0
;;
*)
echo -e "${RED}Unknown option: $1${NC}"
echo "Usage: $0 [--fix]"
exit 2
;;
esac
done
echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║ Model Manager - Code Validation Suite ║${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
echo ""
# Check if virtual environment is activated
if [[ -z "${VIRTUAL_ENV}" ]] && [[ -z "${CONDA_DEFAULT_ENV}" ]]; then
echo -e "${YELLOW}⚠️ Warning: No virtual environment detected${NC}"
echo -e "${YELLOW} Consider activating your venv/conda environment${NC}"
echo ""
fi
# Function to run a validation step
run_step() {
local step_name=$1
local step_command=$2
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${BLUE}▶ ${step_name}${NC}"
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
if eval "$step_command"; then
echo -e "${GREEN}✅ ${step_name} - PASSED${NC}"
echo ""
return 0
else
echo -e "${RED}❌ ${step_name} - FAILED${NC}"
echo ""
return 1
fi
}
# Track failures
FAILED_STEPS=()
# Step 1: Code Formatting Check (Ruff)
# - default: check only
# - --fix: write changes
if ! run_step "1. Code Formatting (Ruff)" "if \$FIX_MODE; then ruff format orchestrator/ tests/; else ruff format --check orchestrator/ tests/; fi"; then
FAILED_STEPS+=("Code Formatting")
fi
# Step 2: Linting (Ruff)
# - default: check only
# - --fix: apply autofixes
if ! run_step "2. Code Linting (Ruff)" "if \$FIX_MODE; then ruff check --fix orchestrator/ tests/; else ruff check orchestrator/ tests/; fi"; then
FAILED_STEPS+=("Linting")
fi
# Step 3: Type Checking (mypy)
if ! run_step "3. Type Checking (mypy)" "mypy orchestrator/"; then
FAILED_STEPS+=("Type Checking")
fi
# Step 4: Security Analysis (Bandit)
if ! run_step "4. Security Analysis (Bandit)" "bandit -r orchestrator/ -ll -q"; then
FAILED_STEPS+=("Security Analysis")
fi
# Step 5: Unit Tests (pytest)
if ! run_step "5. Unit Tests (pytest)" "pytest tests/ --cov=orchestrator --cov-report=term-missing --cov-report=xml --cov-report=html --cov-fail-under=80 -q"; then
FAILED_STEPS+=("Unit Tests")
fi
# Summary
echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║ Validation Summary ║${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
echo ""
if [ ${#FAILED_STEPS[@]} -eq 0 ]; then
echo -e "${GREEN}✅ All validation checks passed!${NC}"
echo -e "${GREEN} Your code is ready for commit/deployment.${NC}"
echo ""
exit 0
else
echo -e "${RED}❌ Validation failed for the following steps:${NC}"
for step in "${FAILED_STEPS[@]}"; do
echo -e "${RED} • ${step}${NC}"
done
echo ""
echo -e "${YELLOW}💡 Tips:${NC}"
echo -e "${YELLOW} • Run 'ruff format orchestrator/ tests/' to auto-fix formatting${NC}"
echo -e "${YELLOW} • Run 'ruff check --fix orchestrator/ tests/' to auto-fix linting issues${NC}"
echo -e "${YELLOW} • Review mypy errors and add type hints where needed${NC}"
echo -e "${YELLOW} • Check bandit warnings for security issues${NC}"
echo -e "${YELLOW} • Fix failing tests or improve test coverage${NC}"
echo ""
exit 1
fi