vitor-aignosi
0e445de386
QTZPOC-20: Implement .sientia model bundle import workflow Introduces a new Temporal workflow and activities to import .sientia model bundles. This enables: - Secure decryption and validation of bundles through a seven-gate process. - Provisioning imported models in MLflow and updating the MongoDB model listing. - Comprehensive status tracking in the public.experiment_run table with new IMPORT_SUCCESS/IMPORT_ERROR states. - Dedicated metrics, robust error handling, and configurable settings. - Includes updated e2e database fixtures and extensive unit/integration tests for bundle format, reader, and overall workflow.
68 lines
4.0 KiB
SQL
68 lines
4.0 KiB
SQL
-- public.experiment_run as migrations V1 … V12 leave it, for the e2e Postgres container.
|
|
--
|
|
-- Source of truth: sientia-dataops-database-migrations,
|
|
-- postgres/sientia-core-mlops-bff/migrations/ — this file mirrors the *final state* of:
|
|
--
|
|
-- V1 create_table_experiment_run (13 columns, one index)
|
|
-- V2 update_check_constraint_on_status (named the status CHECK)
|
|
-- V3 update_check_constraints (named every other CHECK)
|
|
-- V4 update_status_column_size_and_add_orchestrator_validation_error (status -> VARCHAR(50))
|
|
-- V5 migrate_mage_statuses_to_orchestrator (data only)
|
|
-- V6 rename_mage_response_data_column (-> orchestrator_response_data)
|
|
-- V7 migrate_mlflow_statuses_to_tracking (data only)
|
|
-- V8 remove_orchestrator_request_sent_status (status vocabulary)
|
|
-- V9 remove_file_upload_statuses (status vocabulary)
|
|
-- V10 remove_unused_statuses (status vocabulary)
|
|
-- V11 add_run_type_to_experiment_run (run_type + reindexed)
|
|
-- V12 add_import_statuses_to_experiment_run (IMPORT_SUCCESS / IMPORT_ERROR + pairing)
|
|
--
|
|
-- The same drift trade already accepted for the bundle fixtures: this is a copy, and the tests
|
|
-- that read it are what make the copy auditable. If a migration lands there, update this file and
|
|
-- the version list above. Nothing in this repository owns the DDL — no import ever executes it
|
|
-- outside the test suite.
|
|
|
|
CREATE TABLE IF NOT EXISTS public.experiment_run (
|
|
id BIGSERIAL CONSTRAINT pk_experiment_run PRIMARY KEY,
|
|
experiment_name VARCHAR(50) NOT NULL,
|
|
run_name VARCHAR(50),
|
|
username VARCHAR(320) NOT NULL,
|
|
status VARCHAR(50) NOT NULL,
|
|
error_message VARCHAR(1024),
|
|
created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL,
|
|
updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL,
|
|
bucket_name VARCHAR(63),
|
|
file_name VARCHAR(1024),
|
|
request_data JSONB NOT NULL,
|
|
orchestrator_response_data JSONB,
|
|
run_type VARCHAR(20) NOT NULL DEFAULT 'TRAIN',
|
|
|
|
CONSTRAINT chk_experiment_run_experiment_name_min_length CHECK (LENGTH(experiment_name) >= 3),
|
|
CONSTRAINT chk_experiment_run_run_name_min_length CHECK (LENGTH(run_name) >= 3),
|
|
CONSTRAINT chk_experiment_run_username_min_length CHECK (LENGTH(username) >= 3),
|
|
CONSTRAINT chk_experiment_run_username_pattern
|
|
CHECK (username ~* '^[a-zA-Z0-9_!#$%&''*+/=?`{|}~^.-]+@[a-zA-Z0-9.-]+$'),
|
|
CONSTRAINT chk_experiment_run_error_message_min_length CHECK (LENGTH(error_message) >= 3),
|
|
CONSTRAINT chk_experiment_run_bucket_name_min_length CHECK (LENGTH(bucket_name) >= 3),
|
|
CONSTRAINT chk_experiment_run_bucket_name_pattern CHECK (bucket_name ~* '^[a-z0-9-]+$'),
|
|
CONSTRAINT chk_experiment_run_file_name_min_length CHECK (LENGTH(file_name) >= 3),
|
|
CONSTRAINT chk_experiment_run_file_name_pattern
|
|
CHECK (file_name ~* '^[a-zA-Z0-9 /._-]+$'),
|
|
|
|
-- V12: the two shared stage values plus the training and import terminals.
|
|
CONSTRAINT chk_experiment_run_status CHECK (status IN ('PENDING', 'ORCHESTRATOR_REQUEST_ERROR',
|
|
'ORCHESTRATOR_WAITING_PROC',
|
|
'ORCHESTRATOR_VALIDATION_ERROR',
|
|
'TRAINING_SUCCESS', 'TRAINING_ERROR',
|
|
'IMPORT_SUCCESS', 'IMPORT_ERROR')),
|
|
CONSTRAINT chk_experiment_run_run_type CHECK (run_type IN ('TRAIN', 'IMPORT')),
|
|
|
|
-- V12: each pair of terminal values belongs to its own run_type. The shared values stay out.
|
|
CONSTRAINT chk_experiment_run_terminal_status_run_type CHECK (
|
|
(run_type = 'IMPORT' OR status NOT IN ('IMPORT_SUCCESS', 'IMPORT_ERROR'))
|
|
AND (run_type = 'TRAIN' OR status NOT IN ('TRAINING_SUCCESS', 'TRAINING_ERROR'))
|
|
)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_experiment_run_user_exp_status_date_type
|
|
ON public.experiment_run (username, experiment_name, status, created_at, run_type);
|