vitor-aignosi
dac4f1718f
QTZPOC-20: Implement model import workflow for .sientia bundles Introduces the import_model workflow and its associated activities to enable importing .sientia bundles into MLflow and MongoDB. This includes: * New core workflow and activity logic for handling bundle download, decryption, validation, MLflow provisioning (experiments, runs, registered versions), and MongoDB model document creation. * Comprehensive E2E and unit test suites, including a dedicated test harness and a manual run script for development. * Updated .env.example and README.md with new configurations and usage instructions. * Ensures correct MLflow model source resolution for legacy loader compatibility and robust MongoDB document management. * Refactors existing utility scripts into a scripts/ directory.
66 lines
4.0 KiB
SQL
66 lines
4.0 KiB
SQL
-- public.experiment_run as migrations V1 … V11 leave it, for the e2e Postgres container.
|
|
--
|
|
-- Source of truth: sientia-dataops-database-migrations,
|
|
-- postgres/sientia-core-mlops-bff/migrations/ — this file mirrors the *final state* of:
|
|
--
|
|
-- V1 create_table_experiment_run (13 columns, one index)
|
|
-- V2 update_check_constraint_on_status (named the status CHECK)
|
|
-- V3 update_check_constraints (named every other CHECK)
|
|
-- V4 update_status_column_size_and_add_orchestrator_validation_error (status -> VARCHAR(50))
|
|
-- V5 migrate_mage_statuses_to_orchestrator (data only)
|
|
-- V6 rename_mage_response_data_column (-> orchestrator_response_data)
|
|
-- V7 migrate_mlflow_statuses_to_tracking (data only)
|
|
-- V8 remove_orchestrator_request_sent_status (status vocabulary)
|
|
-- V9 remove_file_upload_statuses (status vocabulary)
|
|
-- V10 remove_unused_statuses (status vocabulary)
|
|
-- V11 add_run_type_to_experiment_run (run_type + reindexed)
|
|
--
|
|
-- V12 (add_import_statuses_to_experiment_run) lives in `experiment_run_v12.sql`, applied on top of
|
|
-- this file. It is separate because one e2e test — the evidence for why the migration gates
|
|
-- enabling the import worker — has to run against a table that does *not* have it, and a second
|
|
-- whole-schema copy with V12 removed would rot the moment either copy were edited.
|
|
--
|
|
-- The same drift trade already accepted for the bundle fixtures: this is a copy, and the tests
|
|
-- that read it are what make the copy auditable. If a migration lands there, update this file and
|
|
-- the version list above. Nothing in this repository owns the DDL — no import ever executes it
|
|
-- outside the test suite.
|
|
|
|
CREATE TABLE IF NOT EXISTS public.experiment_run (
|
|
id BIGSERIAL CONSTRAINT pk_experiment_run PRIMARY KEY,
|
|
experiment_name VARCHAR(50) NOT NULL,
|
|
run_name VARCHAR(50),
|
|
username VARCHAR(320) NOT NULL,
|
|
status VARCHAR(50) NOT NULL,
|
|
error_message VARCHAR(1024),
|
|
created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL,
|
|
updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL,
|
|
bucket_name VARCHAR(63),
|
|
file_name VARCHAR(1024),
|
|
request_data JSONB NOT NULL,
|
|
orchestrator_response_data JSONB,
|
|
run_type VARCHAR(20) NOT NULL DEFAULT 'TRAIN',
|
|
|
|
CONSTRAINT chk_experiment_run_experiment_name_min_length CHECK (LENGTH(experiment_name) >= 3),
|
|
CONSTRAINT chk_experiment_run_run_name_min_length CHECK (LENGTH(run_name) >= 3),
|
|
CONSTRAINT chk_experiment_run_username_min_length CHECK (LENGTH(username) >= 3),
|
|
CONSTRAINT chk_experiment_run_username_pattern
|
|
CHECK (username ~* '^[a-zA-Z0-9_!#$%&''*+/=?`{|}~^.-]+@[a-zA-Z0-9.-]+$'),
|
|
CONSTRAINT chk_experiment_run_error_message_min_length CHECK (LENGTH(error_message) >= 3),
|
|
CONSTRAINT chk_experiment_run_bucket_name_min_length CHECK (LENGTH(bucket_name) >= 3),
|
|
CONSTRAINT chk_experiment_run_bucket_name_pattern CHECK (bucket_name ~* '^[a-z0-9-]+$'),
|
|
CONSTRAINT chk_experiment_run_file_name_min_length CHECK (LENGTH(file_name) >= 3),
|
|
CONSTRAINT chk_experiment_run_file_name_pattern
|
|
CHECK (file_name ~* '^[a-zA-Z0-9 /._-]+$'),
|
|
|
|
-- The V1…V11 status vocabulary: the two shared stage values plus the training terminals.
|
|
-- V12 adds IMPORT_SUCCESS and IMPORT_ERROR to this list.
|
|
CONSTRAINT chk_experiment_run_status CHECK (status IN ('PENDING', 'ORCHESTRATOR_REQUEST_ERROR',
|
|
'ORCHESTRATOR_WAITING_PROC',
|
|
'ORCHESTRATOR_VALIDATION_ERROR',
|
|
'TRAINING_SUCCESS', 'TRAINING_ERROR')),
|
|
CONSTRAINT chk_experiment_run_run_type CHECK (run_type IN ('TRAIN', 'IMPORT'))
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_experiment_run_user_exp_status_date_type
|
|
ON public.experiment_run (username, experiment_name, status, created_at, run_type);
|