feat: update training workflow and repository management
- Replaced synchronous MinIO repository calls with asynchronous counterparts in the Training class for improved performance. - Enhanced logging throughout the training process to provide better insights into model metadata loading, parameter validation, and training execution. - Updated the train_test_split function to enforce DataFrame input type, ensuring consistency in data handling. - Removed the deprecated model_repository.py file to streamline the codebase. - Adjusted cleanup schedule logic to improve error handling and logging during schedule reconciliation. - Updated tests to reflect changes in the training workflow and repository interactions.
This commit is contained in:
@@ -32,6 +32,9 @@ import psycopg2
|
||||
from dotenv import load_dotenv
|
||||
from psycopg2.extras import Json
|
||||
from temporalio import client
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# %%
|
||||
# --- configuration (edit here or use `.env` at repo root) ---
|
||||
@@ -54,9 +57,14 @@ PG = {
|
||||
|
||||
TEMPORAL_HOST = os.getenv('TEMPORAL_HOST')
|
||||
TEMPORAL_NAMESPACE = os.getenv('TEMPORAL_NAMESPACE')
|
||||
TRAIN_TASK_QUEUE = os.getenv('TRAIN_TASK_QUEUE')
|
||||
TRAIN_TASK_QUEUE = "train_model-single-queue"
|
||||
TEMPORAL_TLS = os.getenv('TEMPORAL_USE_TLS', 'false').lower() in ('1', 'true', 'yes')
|
||||
|
||||
|
||||
print(MINIO_MC_ALIAS, MINIO_BUCKET, OBJECT_NAME, LOCAL_CSV)
|
||||
print(PG)
|
||||
print(TEMPORAL_HOST, TEMPORAL_NAMESPACE, TRAIN_TASK_QUEUE, TEMPORAL_TLS)
|
||||
|
||||
# %%
|
||||
# --- 1) database: delete previous row (same id), then insert `experiment_run` ---
|
||||
# Primary key column is `id` (see `experiment_tracking` updates). `request_data` matches the SQL sample in `input-sample.md`.
|
||||
@@ -97,7 +105,7 @@ with psycopg2.connect(**PG) as conn:
|
||||
EXPERIMENT_RUN_ID,
|
||||
'test-experiment-name',
|
||||
'test-run-name',
|
||||
'test-username',
|
||||
'vitor.santos@aignosi.com.br',
|
||||
'ORCHESTRATOR_WAITING_PROC',
|
||||
None,
|
||||
now,
|
||||
@@ -142,15 +150,16 @@ _workflow_input = {
|
||||
'opt_params': {},
|
||||
}
|
||||
|
||||
# %%
|
||||
|
||||
c = await client.Client.connect(
|
||||
target_host=TH,
|
||||
namespace=TN,
|
||||
tls=TEMPORAL_TLS,
|
||||
)
|
||||
|
||||
# %%
|
||||
|
||||
wid = f'train-model-test-{uuid.uuid4()}'
|
||||
await c.execute_workflow( # type: ignore[call-overload]
|
||||
result = await c.execute_workflow( # type: ignore[call-overload]
|
||||
'train_model',
|
||||
_workflow_input,
|
||||
id=wid,
|
||||
@@ -160,3 +169,6 @@ await c.execute_workflow( # type: ignore[call-overload]
|
||||
task_timeout=timedelta(minutes=5),
|
||||
)
|
||||
print(wid)
|
||||
print(result)
|
||||
|
||||
# %%
|
||||
|
||||
Reference in New Issue
Block a user