feat: update training workflow and repository management

- Replaced synchronous MinIO repository calls with asynchronous counterparts in the Training class for improved performance.
- Enhanced logging throughout the training process to provide better insights into model metadata loading, parameter validation, and training execution.
- Updated the train_test_split function to enforce DataFrame input type, ensuring consistency in data handling.
- Removed the deprecated model_repository.py file to streamline the codebase.
- Adjusted cleanup schedule logic to improve error handling and logging during schedule reconciliation.
- Updated tests to reflect changes in the training workflow and repository interactions.
This commit is contained in:
vitor-aignosi
2026-04-09 12:09:52 -03:00
parent 0ae03b246f
commit 526edcb50e
14 changed files with 114 additions and 503 deletions

View File

@@ -32,6 +32,9 @@ import psycopg2
from dotenv import load_dotenv
from psycopg2.extras import Json
from temporalio import client
from dotenv import load_dotenv
load_dotenv()
# %%
# --- configuration (edit here or use `.env` at repo root) ---
@@ -54,9 +57,14 @@ PG = {
TEMPORAL_HOST = os.getenv('TEMPORAL_HOST')
TEMPORAL_NAMESPACE = os.getenv('TEMPORAL_NAMESPACE')
TRAIN_TASK_QUEUE = os.getenv('TRAIN_TASK_QUEUE')
TRAIN_TASK_QUEUE = "train_model-single-queue"
TEMPORAL_TLS = os.getenv('TEMPORAL_USE_TLS', 'false').lower() in ('1', 'true', 'yes')
print(MINIO_MC_ALIAS, MINIO_BUCKET, OBJECT_NAME, LOCAL_CSV)
print(PG)
print(TEMPORAL_HOST, TEMPORAL_NAMESPACE, TRAIN_TASK_QUEUE, TEMPORAL_TLS)
# %%
# --- 1) database: delete previous row (same id), then insert `experiment_run` ---
# Primary key column is `id` (see `experiment_tracking` updates). `request_data` matches the SQL sample in `input-sample.md`.
@@ -97,7 +105,7 @@ with psycopg2.connect(**PG) as conn:
EXPERIMENT_RUN_ID,
'test-experiment-name',
'test-run-name',
'test-username',
'vitor.santos@aignosi.com.br',
'ORCHESTRATOR_WAITING_PROC',
None,
now,
@@ -142,15 +150,16 @@ _workflow_input = {
'opt_params': {},
}
# %%
c = await client.Client.connect(
target_host=TH,
namespace=TN,
tls=TEMPORAL_TLS,
)
# %%
wid = f'train-model-test-{uuid.uuid4()}'
await c.execute_workflow( # type: ignore[call-overload]
result = await c.execute_workflow( # type: ignore[call-overload]
'train_model',
_workflow_input,
id=wid,
@@ -160,3 +169,6 @@ await c.execute_workflow( # type: ignore[call-overload]
task_timeout=timedelta(minutes=5),
)
print(wid)
print(result)
# %%