Merge pull request #23 from Aignosi/fix/SIENTIAPDE-1579
Fix/SIENTIAPDE-1579: Corrigir bug durante o treinamento de certos arquivos no Soft Sensor
This commit is contained in:
33
docs/DB_CV022_WIT230 _double_date_column.csv
Normal file
33
docs/DB_CV022_WIT230 _double_date_column.csv
Normal file
@@ -0,0 +1,33 @@
|
||||
DATA,DATE2,03CV022/CORRENTE_N_M1_PV(Value),303-WIT-230(Value)
|
||||
01/05/2022 00:00:00,07-01-2022 01:00:00,170,33
|
||||
01/05/2022 00:00:10,07-01-2022 01:00:10,169,605
|
||||
01/05/2022 00:00:20,07-01-2022 01:00:20,169,178
|
||||
01/05/2022 00:00:30,07-01-2022 01:00:30,166,468
|
||||
01/05/2022 00:00:40,07-01-2022 01:00:40,162,136
|
||||
01/05/2022 00:00:50,07-01-2022 01:00:50,157,804
|
||||
01/05/2022 00:01:00,07-01-2022 01:01:00,155,883
|
||||
01/05/2022 00:01:10,07-01-2022 01:01:10,155,684
|
||||
01/05/2022 00:01:20,07-01-2022 01:01:20,155,484
|
||||
01/05/2022 00:01:30,07-01-2022 01:01:30,155,284
|
||||
01/05/2022 00:01:40,07-01-2022 01:01:40,155,85
|
||||
01/05/2022 00:01:50,07-01-2022 01:01:50,154,885
|
||||
01/05/2022 00:02:00,09-01-2022 02:02:00,154,685
|
||||
01/05/2022 00:02:10,09-01-2022 02:02:10,154,486
|
||||
01/05/2022 00:02:20,09-01-2022 02:02:20,154,286
|
||||
01/05/2022 00:02:30,09-01-2022 02:02:30,154,86
|
||||
01/05/2022 00:02:40,09-01-2022 02:02:40,152,866
|
||||
01/05/2022 00:02:50,09-01-2022 02:02:50,150,87
|
||||
01/05/2022 00:03:00,09-01-2022 02:03:00,148,874
|
||||
01/05/2022 00:03:10,09-01-2022 02:03:10,148,2134
|
||||
01/05/2022 00:03:20,09-01-2022 02:03:20,148,2068
|
||||
01/05/2022 00:03:30,09-01-2022 02:03:30,148,2022
|
||||
01/05/2022 00:03:40,09-01-2022 02:03:40,148,1976
|
||||
01/05/2022 00:03:50,09-01-2022 02:03:50,154,139
|
||||
01/05/2022 00:04:00,11-01-2022 03:04:00,165,112
|
||||
01/05/2022 00:04:10,11-01-2022 03:04:10,170,42
|
||||
01/05/2022 00:04:20,11-01-2022 03:04:20,170,116
|
||||
01/05/2022 00:04:30,11-01-2022 03:04:30,170,191
|
||||
01/05/2022 00:04:40,11-01-2022 03:04:40,170,266
|
||||
01/05/2022 00:04:50,11-01-2022 03:04:50,170,341
|
||||
01/05/2022 00:05:00,11-01-2022 03:05:00,170,416
|
||||
01/05/2022 00:05:10,11-01-2022 03:05:10,170,491
|
||||
|
647
docs/DB_CV022_WIT230.csv
Normal file
647
docs/DB_CV022_WIT230.csv
Normal file
@@ -0,0 +1,647 @@
|
||||
DATA,03CV022/CORRENTE_N_M1_PV(Value),303-WIT-230(Value)
|
||||
01/05/2022 00:00:00,170,33
|
||||
01/05/2022 00:00:10,169,605
|
||||
01/05/2022 00:00:20,169,178
|
||||
01/05/2022 00:00:30,166,468
|
||||
01/05/2022 00:00:40,162,136
|
||||
01/05/2022 00:00:50,157,804
|
||||
01/05/2022 00:01:00,155,883
|
||||
01/05/2022 00:01:10,155,684
|
||||
01/05/2022 00:01:20,155,484
|
||||
01/05/2022 00:01:30,155,284
|
||||
01/05/2022 00:01:40,155,85
|
||||
01/05/2022 00:01:50,154,885
|
||||
01/05/2022 00:02:00,154,685
|
||||
01/05/2022 00:02:10,154,486
|
||||
01/05/2022 00:02:20,154,286
|
||||
01/05/2022 00:02:30,154,86
|
||||
01/05/2022 00:02:40,152,866
|
||||
01/05/2022 00:02:50,150,87
|
||||
01/05/2022 00:03:00,148,874
|
||||
01/05/2022 00:03:10,148,2134
|
||||
01/05/2022 00:03:20,148,2068
|
||||
01/05/2022 00:03:30,148,2022
|
||||
01/05/2022 00:03:40,148,1976
|
||||
01/05/2022 00:03:50,154,139
|
||||
01/05/2022 00:04:00,165,112
|
||||
01/05/2022 00:04:10,170,42
|
||||
01/05/2022 00:04:20,170,116
|
||||
01/05/2022 00:04:30,170,191
|
||||
01/05/2022 00:04:40,170,266
|
||||
01/05/2022 00:04:50,170,341
|
||||
01/05/2022 00:05:00,170,416
|
||||
01/05/2022 00:05:10,170,491
|
||||
01/05/2022 00:05:20,170,566
|
||||
01/05/2022 00:05:30,170,641
|
||||
01/05/2022 00:05:40,170,716
|
||||
01/05/2022 00:05:50,170,79
|
||||
01/05/2022 00:06:00,170,865
|
||||
01/05/2022 00:06:10,170,94
|
||||
01/05/2022 00:06:20,171,15
|
||||
01/05/2022 00:06:30,171,9
|
||||
01/05/2022 00:06:40,171,165
|
||||
01/05/2022 00:06:50,171,24
|
||||
01/05/2022 00:07:00,171,315
|
||||
01/05/2022 00:07:10,171,39
|
||||
01/05/2022 00:07:20,171,465
|
||||
01/05/2022 00:07:30,171,539
|
||||
01/05/2022 00:07:40,171,614
|
||||
01/05/2022 00:07:50,171,689
|
||||
01/05/2022 00:08:00,171,764
|
||||
01/05/2022 00:08:10,171,839
|
||||
01/05/2022 00:08:20,171,914
|
||||
01/05/2022 00:08:30,171,989
|
||||
01/05/2022 00:08:40,172,64
|
||||
01/05/2022 00:08:50,172,139
|
||||
01/05/2022 00:09:00,172,214
|
||||
01/05/2022 00:09:10,172,288
|
||||
01/05/2022 00:09:20,172,363
|
||||
01/05/2022 00:09:30,172,438
|
||||
01/05/2022 00:09:40,172,513
|
||||
01/05/2022 00:09:50,172,588
|
||||
01/05/2022 00:10:00,172,663
|
||||
01/05/2022 00:10:10,172,738
|
||||
01/05/2022 00:10:20,172,813
|
||||
01/05/2022 00:10:30,172,888
|
||||
01/05/2022 00:10:40,172,962
|
||||
01/05/2022 00:10:50,172,981
|
||||
01/05/2022 00:11:00,172,943
|
||||
01/05/2022 00:11:10,172,905
|
||||
01/05/2022 00:11:20,172,867
|
||||
01/05/2022 00:11:30,172,829
|
||||
01/05/2022 00:11:40,172,79
|
||||
01/05/2022 00:11:50,172,752
|
||||
01/05/2022 00:12:00,172,714
|
||||
01/05/2022 00:12:10,172,676
|
||||
01/05/2022 00:12:20,172,638
|
||||
01/05/2022 00:12:30,172,6
|
||||
01/05/2022 00:12:40,172,562
|
||||
01/05/2022 00:12:50,172,524
|
||||
01/05/2022 00:13:00,172,485
|
||||
01/05/2022 00:13:10,172,447
|
||||
01/05/2022 00:13:20,172,409
|
||||
01/05/2022 00:13:30,172,371
|
||||
01/05/2022 00:13:40,172,333
|
||||
01/05/2022 00:13:50,172,295
|
||||
01/05/2022 00:14:00,172,257
|
||||
01/05/2022 00:14:10,172,219
|
||||
01/05/2022 00:14:20,172,18
|
||||
01/05/2022 00:14:30,172,142
|
||||
01/05/2022 00:14:40,172,104
|
||||
01/05/2022 00:14:50,172,66
|
||||
01/05/2022 00:15:00,172,28
|
||||
01/05/2022 00:15:10,171,99
|
||||
01/05/2022 00:15:20,171,952
|
||||
01/05/2022 00:15:30,171,914
|
||||
01/05/2022 00:15:40,171,876
|
||||
01/05/2022 00:15:50,171,837
|
||||
01/05/2022 00:16:00,171,799
|
||||
01/05/2022 00:16:10,171,761
|
||||
01/05/2022 00:16:20,171,723
|
||||
01/05/2022 00:16:30,171,685
|
||||
01/05/2022 00:16:40,171,647
|
||||
01/05/2022 00:16:50,171,609
|
||||
01/05/2022 00:17:00,171,571
|
||||
01/05/2022 00:17:10,171,533
|
||||
01/05/2022 00:17:20,171,494
|
||||
01/05/2022 00:17:30,171,456
|
||||
01/05/2022 00:17:40,171,418
|
||||
01/05/2022 00:17:50,171,38
|
||||
01/05/2022 00:18:00,171,342
|
||||
01/05/2022 00:18:10,171,304
|
||||
01/05/2022 00:18:20,171,266
|
||||
01/05/2022 00:18:30,171,228
|
||||
01/05/2022 00:18:40,171,189
|
||||
01/05/2022 00:18:50,171,151
|
||||
01/05/2022 00:19:00,171,113
|
||||
01/05/2022 00:19:10,171,75
|
||||
01/05/2022 00:19:20,171,37
|
||||
01/05/2022 00:19:30,170,999
|
||||
01/05/2022 00:19:40,170,961
|
||||
01/05/2022 00:19:50,170,923
|
||||
01/05/2022 00:20:00,170,884
|
||||
01/05/2022 00:20:10,170,846
|
||||
01/05/2022 00:20:20,170,808
|
||||
01/05/2022 00:20:30,170,77
|
||||
01/05/2022 00:20:40,170,732
|
||||
01/05/2022 00:20:50,170,694
|
||||
01/05/2022 00:21:00,170,656
|
||||
01/05/2022 00:21:10,170,618
|
||||
01/05/2022 00:21:20,170,58
|
||||
01/05/2022 00:21:30,170,541
|
||||
01/05/2022 00:21:40,170,503
|
||||
01/05/2022 00:21:50,170,465
|
||||
01/05/2022 00:22:00,170,427
|
||||
01/05/2022 00:22:10,170,389
|
||||
01/05/2022 00:22:20,170,351
|
||||
01/05/2022 00:22:30,170,313
|
||||
01/05/2022 00:22:40,170,275
|
||||
01/05/2022 00:22:50,170,236
|
||||
01/05/2022 00:23:00,170,198
|
||||
01/05/2022 00:23:10,170,16
|
||||
01/05/2022 00:23:20,170,122
|
||||
01/05/2022 00:23:30,170,84
|
||||
01/05/2022 00:23:40,170,46
|
||||
01/05/2022 00:23:50,170,8
|
||||
01/05/2022 00:24:00,169,97
|
||||
01/05/2022 00:24:10,169,932
|
||||
01/05/2022 00:24:20,169,893
|
||||
01/05/2022 00:24:30,169,855
|
||||
01/05/2022 00:24:40,169,817
|
||||
01/05/2022 00:24:50,169,779
|
||||
01/05/2022 00:25:00,169,741
|
||||
01/05/2022 00:25:10,169,703
|
||||
01/05/2022 00:25:20,169,665
|
||||
01/05/2022 00:25:30,169,627
|
||||
01/05/2022 00:25:40,169,588
|
||||
01/05/2022 00:25:50,169,55
|
||||
01/05/2022 00:26:00,169,512
|
||||
01/05/2022 00:26:10,169,474
|
||||
01/05/2022 00:26:20,169,436
|
||||
01/05/2022 00:26:30,169,398
|
||||
01/05/2022 00:26:40,169,36
|
||||
01/05/2022 00:26:50,169,322
|
||||
01/05/2022 00:27:00,169,284
|
||||
01/05/2022 00:27:10,169,245
|
||||
01/05/2022 00:27:20,169,207
|
||||
01/05/2022 00:27:30,169,169
|
||||
01/05/2022 00:27:40,169,131
|
||||
01/05/2022 00:27:50,169,93
|
||||
01/05/2022 00:28:00,169,55
|
||||
01/05/2022 00:28:10,169,17
|
||||
01/05/2022 00:28:20,168,979
|
||||
01/05/2022 00:28:30,168,94
|
||||
01/05/2022 00:28:40,168,902
|
||||
01/05/2022 00:28:50,168,864
|
||||
01/05/2022 00:29:00,168,826
|
||||
01/05/2022 00:29:10,168,788
|
||||
01/05/2022 00:29:20,168,75
|
||||
01/05/2022 00:29:30,168,712
|
||||
01/05/2022 00:29:40,168,674
|
||||
01/05/2022 00:29:50,168,636
|
||||
01/05/2022 00:30:00,168,597
|
||||
01/05/2022 00:30:10,168,559
|
||||
01/05/2022 00:30:20,168,521
|
||||
01/05/2022 00:30:30,168,483
|
||||
01/05/2022 00:30:40,168,445
|
||||
01/05/2022 00:30:50,168,407
|
||||
01/05/2022 00:31:00,168,369
|
||||
01/05/2022 00:31:10,168,331
|
||||
01/05/2022 00:31:20,168,292
|
||||
01/05/2022 00:31:30,168,254
|
||||
01/05/2022 00:31:40,168,216
|
||||
01/05/2022 00:31:50,168,178
|
||||
01/05/2022 00:32:00,168,14
|
||||
01/05/2022 00:32:10,168,102
|
||||
01/05/2022 00:32:20,168,64
|
||||
01/05/2022 00:32:30,168,26
|
||||
01/05/2022 00:32:40,168,26
|
||||
01/05/2022 00:32:50,168,105
|
||||
01/05/2022 00:33:00,168,183
|
||||
01/05/2022 00:33:10,168,262
|
||||
01/05/2022 00:33:20,168,341
|
||||
01/05/2022 00:33:30,168,42
|
||||
01/05/2022 00:33:40,168,499
|
||||
01/05/2022 00:33:50,168,578
|
||||
01/05/2022 00:34:00,168,657
|
||||
01/05/2022 00:34:10,168,735
|
||||
01/05/2022 00:34:20,168,814
|
||||
01/05/2022 00:34:30,168,893
|
||||
01/05/2022 00:34:40,168,972
|
||||
01/05/2022 00:34:50,169,51
|
||||
01/05/2022 00:35:00,169,13
|
||||
01/05/2022 00:35:10,169,209
|
||||
01/05/2022 00:35:20,169,287
|
||||
01/05/2022 00:35:30,169,366
|
||||
01/05/2022 00:35:40,169,445
|
||||
01/05/2022 00:35:50,169,524
|
||||
01/05/2022 00:36:00,169,603
|
||||
01/05/2022 00:36:10,169,682
|
||||
01/05/2022 00:36:20,169,761
|
||||
01/05/2022 00:36:30,169,839
|
||||
01/05/2022 00:36:40,169,918
|
||||
01/05/2022 00:36:50,169,997
|
||||
01/05/2022 00:37:00,170,76
|
||||
01/05/2022 00:37:10,170,155
|
||||
01/05/2022 00:37:20,170,234
|
||||
01/05/2022 00:37:30,170,312
|
||||
01/05/2022 00:37:40,170,391
|
||||
01/05/2022 00:37:50,170,47
|
||||
01/05/2022 00:38:00,170,549
|
||||
01/05/2022 00:38:10,170,628
|
||||
01/05/2022 00:38:20,170,707
|
||||
01/05/2022 00:38:30,170,786
|
||||
01/05/2022 00:38:40,170,864
|
||||
01/05/2022 00:38:50,170,943
|
||||
01/05/2022 00:39:00,170,993
|
||||
01/05/2022 00:39:10,170,967
|
||||
01/05/2022 00:39:20,170,942
|
||||
01/05/2022 00:39:30,170,917
|
||||
01/05/2022 00:39:40,170,891
|
||||
01/05/2022 00:39:50,170,866
|
||||
01/05/2022 00:40:00,170,841
|
||||
01/05/2022 00:40:10,170,815
|
||||
01/05/2022 00:40:20,170,79
|
||||
01/05/2022 00:40:30,170,764
|
||||
01/05/2022 00:40:40,170,739
|
||||
01/05/2022 00:40:50,170,714
|
||||
01/05/2022 00:41:00,170,688
|
||||
01/05/2022 00:41:10,170,663
|
||||
01/05/2022 00:41:20,170,637
|
||||
01/05/2022 00:41:30,170,612
|
||||
01/05/2022 00:41:40,170,587
|
||||
01/05/2022 00:41:50,170,561
|
||||
01/05/2022 00:42:00,170,536
|
||||
01/05/2022 00:42:10,170,51
|
||||
01/05/2022 00:42:20,170,485
|
||||
01/05/2022 00:42:30,170,46
|
||||
01/05/2022 00:42:40,170,434
|
||||
01/05/2022 00:42:50,170,409
|
||||
01/05/2022 00:43:00,170,384
|
||||
01/05/2022 00:43:10,170,358
|
||||
01/05/2022 00:43:20,170,333
|
||||
01/05/2022 00:43:30,170,307
|
||||
01/05/2022 00:43:40,170,282
|
||||
01/05/2022 00:43:50,170,257
|
||||
01/05/2022 00:44:00,170,231
|
||||
01/05/2022 00:44:10,170,206
|
||||
01/05/2022 00:44:20,170,18
|
||||
01/05/2022 00:44:30,170,155
|
||||
01/05/2022 00:44:40,170,13
|
||||
01/05/2022 00:44:50,170,104
|
||||
01/05/2022 00:45:00,170,79
|
||||
01/05/2022 00:45:10,170,54
|
||||
01/05/2022 00:45:20,170,28
|
||||
01/05/2022 00:45:30,170,3
|
||||
01/05/2022 00:45:40,169,977
|
||||
01/05/2022 00:45:50,169,952
|
||||
01/05/2022 00:46:00,169,927
|
||||
01/05/2022 00:46:10,169,901
|
||||
01/05/2022 00:46:20,169,876
|
||||
01/05/2022 00:46:30,169,85
|
||||
01/05/2022 00:46:40,169,825
|
||||
01/05/2022 00:46:50,169,8
|
||||
01/05/2022 00:47:00,169,774
|
||||
01/05/2022 00:47:10,169,749
|
||||
01/05/2022 00:47:20,169,723
|
||||
01/05/2022 00:47:30,169,698
|
||||
01/05/2022 00:47:40,169,673
|
||||
01/05/2022 00:47:50,169,647
|
||||
01/05/2022 00:48:00,169,622
|
||||
01/05/2022 00:48:10,169,597
|
||||
01/05/2022 00:48:20,169,571
|
||||
01/05/2022 00:48:30,169,546
|
||||
01/05/2022 00:48:40,169,52
|
||||
01/05/2022 00:48:50,169,495
|
||||
01/05/2022 00:49:00,169,47
|
||||
01/05/2022 00:49:10,169,444
|
||||
01/05/2022 00:49:20,169,419
|
||||
01/05/2022 00:49:30,169,393
|
||||
01/05/2022 00:49:40,169,368
|
||||
01/05/2022 00:49:50,169,343
|
||||
01/05/2022 00:50:00,169,317
|
||||
01/05/2022 00:50:10,169,292
|
||||
01/05/2022 00:50:20,169,266
|
||||
01/05/2022 00:50:30,169,241
|
||||
01/05/2022 00:50:40,169,216
|
||||
01/05/2022 00:50:50,169,19
|
||||
01/05/2022 00:51:00,169,165
|
||||
01/05/2022 00:51:10,169,14
|
||||
01/05/2022 00:51:20,169,114
|
||||
01/05/2022 00:51:30,169,89
|
||||
01/05/2022 00:51:40,169,63
|
||||
01/05/2022 00:51:50,169,38
|
||||
01/05/2022 00:52:00,169,13
|
||||
01/05/2022 00:52:10,168,987
|
||||
01/05/2022 00:52:20,168,962
|
||||
01/05/2022 00:52:30,168,936
|
||||
01/05/2022 00:52:40,168,911
|
||||
01/05/2022 00:52:50,168,886
|
||||
01/05/2022 00:53:00,168,86
|
||||
01/05/2022 00:53:10,168,835
|
||||
01/05/2022 00:53:20,168,809
|
||||
01/05/2022 00:53:30,168,784
|
||||
01/05/2022 00:53:40,168,759
|
||||
01/05/2022 00:53:50,168,733
|
||||
01/05/2022 00:54:00,168,708
|
||||
01/05/2022 00:54:10,168,683
|
||||
01/05/2022 00:54:20,168,657
|
||||
01/05/2022 00:54:30,168,632
|
||||
01/05/2022 00:54:40,168,606
|
||||
01/05/2022 00:54:50,168,581
|
||||
01/05/2022 00:55:00,168,556
|
||||
01/05/2022 00:55:10,168,53
|
||||
01/05/2022 00:55:20,168,505
|
||||
01/05/2022 00:55:30,168,479
|
||||
01/05/2022 00:55:40,168,454
|
||||
01/05/2022 00:55:50,168,429
|
||||
01/05/2022 00:56:00,168,403
|
||||
01/05/2022 00:56:10,168,378
|
||||
01/05/2022 00:56:20,168,352
|
||||
01/05/2022 00:56:30,168,327
|
||||
01/05/2022 00:56:40,168,302
|
||||
01/05/2022 00:56:50,168,276
|
||||
01/05/2022 00:57:00,168,251
|
||||
01/05/2022 00:57:10,168,226
|
||||
01/05/2022 00:57:20,168,2
|
||||
01/05/2022 00:57:30,168,175
|
||||
01/05/2022 00:57:40,168,149
|
||||
01/05/2022 00:57:50,168,124
|
||||
01/05/2022 00:58:00,168,99
|
||||
01/05/2022 00:58:10,168,73
|
||||
01/05/2022 00:58:20,168,48
|
||||
01/05/2022 00:58:30,168,22
|
||||
01/05/2022 00:58:40,167,76
|
||||
01/05/2022 00:58:50,160,151
|
||||
01/05/2022 00:59:00,161,484
|
||||
01/05/2022 00:59:10,162,817
|
||||
01/05/2022 00:59:20,164,281
|
||||
01/05/2022 00:59:30,166,777
|
||||
01/05/2022 00:59:40,167,148
|
||||
01/05/2022 00:59:50,148,45
|
||||
01/05/2022 01:00:00,117,525
|
||||
01/05/2022 01:00:10,105,3
|
||||
01/05/2022 01:00:20,105,315
|
||||
01/05/2022 01:00:30,105,6
|
||||
01/05/2022 01:00:40,105,886
|
||||
01/05/2022 01:00:50,106,171
|
||||
01/05/2022 01:01:00,106,456
|
||||
01/05/2022 01:01:10,106,742
|
||||
01/05/2022 01:01:20,107,698
|
||||
01/05/2022 01:01:30,115,81
|
||||
01/05/2022 01:01:40,122,464
|
||||
01/05/2022 01:01:50,129,847
|
||||
01/05/2022 01:02:00,137,231
|
||||
01/05/2022 01:02:10,144,138
|
||||
01/05/2022 01:02:20,145,801
|
||||
01/05/2022 01:02:30,147,464
|
||||
01/05/2022 01:02:40,149,835
|
||||
01/05/2022 01:02:50,160,808
|
||||
01/05/2022 01:03:00,171,2
|
||||
01/05/2022 01:03:10,171,36
|
||||
01/05/2022 01:03:20,171,7
|
||||
01/05/2022 01:03:30,171,103
|
||||
01/05/2022 01:03:40,171,137
|
||||
01/05/2022 01:03:50,171,171
|
||||
01/05/2022 01:04:00,171,204
|
||||
01/05/2022 01:04:10,171,238
|
||||
01/05/2022 01:04:20,171,272
|
||||
01/05/2022 01:04:30,171,305
|
||||
01/05/2022 01:04:40,171,339
|
||||
01/05/2022 01:04:50,171,372
|
||||
01/05/2022 01:05:00,171,406
|
||||
01/05/2022 01:05:10,171,44
|
||||
01/05/2022 01:05:20,171,473
|
||||
01/05/2022 01:05:30,171,507
|
||||
01/05/2022 01:05:40,171,541
|
||||
01/05/2022 01:05:50,171,574
|
||||
01/05/2022 01:06:00,171,608
|
||||
01/05/2022 01:06:10,171,642
|
||||
01/05/2022 01:06:20,171,675
|
||||
01/05/2022 01:06:30,171,709
|
||||
01/05/2022 01:06:40,171,743
|
||||
01/05/2022 01:06:50,171,776
|
||||
01/05/2022 01:07:00,171,81
|
||||
01/05/2022 01:07:10,171,844
|
||||
01/05/2022 01:07:20,171,877
|
||||
01/05/2022 01:07:30,171,911
|
||||
01/05/2022 01:07:40,171,944
|
||||
01/05/2022 01:07:50,171,978
|
||||
01/05/2022 01:08:00,172,12
|
||||
01/05/2022 01:08:10,172,45
|
||||
01/05/2022 01:08:20,172,79
|
||||
01/05/2022 01:08:30,172,113
|
||||
01/05/2022 01:08:40,172,146
|
||||
01/05/2022 01:08:50,172,18
|
||||
01/05/2022 01:09:00,172,214
|
||||
01/05/2022 01:09:10,172,247
|
||||
01/05/2022 01:09:20,172,281
|
||||
01/05/2022 01:09:30,172,315
|
||||
01/05/2022 01:09:40,172,348
|
||||
01/05/2022 01:09:50,172,382
|
||||
01/05/2022 01:10:00,172,415
|
||||
01/05/2022 01:10:10,172,449
|
||||
01/05/2022 01:10:20,172,483
|
||||
01/05/2022 01:10:30,172,516
|
||||
01/05/2022 01:10:40,172,55
|
||||
01/05/2022 01:10:50,172,584
|
||||
01/05/2022 01:11:00,172,617
|
||||
01/05/2022 01:11:10,172,651
|
||||
01/05/2022 01:11:20,172,685
|
||||
01/05/2022 01:11:30,172,718
|
||||
01/05/2022 01:11:40,172,752
|
||||
01/05/2022 01:11:50,172,786
|
||||
01/05/2022 01:12:00,172,819
|
||||
01/05/2022 01:12:10,172,853
|
||||
01/05/2022 01:12:20,172,887
|
||||
01/05/2022 01:12:30,172,92
|
||||
01/05/2022 01:12:40,172,954
|
||||
01/05/2022 01:12:50,172,987
|
||||
01/05/2022 01:13:00,173,21
|
||||
01/05/2022 01:13:10,173,55
|
||||
01/05/2022 01:13:20,173,88
|
||||
01/05/2022 01:13:30,173,122
|
||||
01/05/2022 01:13:40,173,156
|
||||
01/05/2022 01:13:50,173,189
|
||||
01/05/2022 01:14:00,173,223
|
||||
01/05/2022 01:14:10,173,257
|
||||
01/05/2022 01:14:20,173,29
|
||||
01/05/2022 01:14:30,173,324
|
||||
01/05/2022 01:14:40,173,358
|
||||
01/05/2022 01:14:50,173,391
|
||||
01/05/2022 01:15:00,173,425
|
||||
01/05/2022 01:15:10,173,458
|
||||
01/05/2022 01:15:20,173,492
|
||||
01/05/2022 01:15:30,173,526
|
||||
01/05/2022 01:15:40,173,559
|
||||
01/05/2022 01:15:50,173,593
|
||||
01/05/2022 01:16:00,173,627
|
||||
01/05/2022 01:16:10,173,66
|
||||
01/05/2022 01:16:20,173,694
|
||||
01/05/2022 01:16:30,173,728
|
||||
01/05/2022 01:16:40,173,761
|
||||
01/05/2022 01:16:50,173,795
|
||||
01/05/2022 01:17:00,173,829
|
||||
01/05/2022 01:17:10,173,862
|
||||
01/05/2022 01:17:20,173,896
|
||||
01/05/2022 01:17:30,173,93
|
||||
01/05/2022 01:17:40,173,963
|
||||
01/05/2022 01:17:50,173,997
|
||||
01/05/2022 01:18:00,174,103
|
||||
01/05/2022 01:18:10,174,218
|
||||
01/05/2022 01:18:20,174,332
|
||||
01/05/2022 01:18:30,174,446
|
||||
01/05/2022 01:18:40,174,56
|
||||
01/05/2022 01:18:50,174,674
|
||||
01/05/2022 01:19:00,174,788
|
||||
01/05/2022 01:19:10,174,902
|
||||
01/05/2022 01:19:20,175,17
|
||||
01/05/2022 01:19:30,175,131
|
||||
01/05/2022 01:19:40,175,245
|
||||
01/05/2022 01:19:50,175,359
|
||||
01/05/2022 01:20:00,175,473
|
||||
01/05/2022 01:20:10,175,587
|
||||
01/05/2022 01:20:20,175,701
|
||||
01/05/2022 01:20:30,175,816
|
||||
01/05/2022 01:20:40,175,93
|
||||
01/05/2022 01:20:50,176,44
|
||||
01/05/2022 01:21:00,176,158
|
||||
01/05/2022 01:21:10,176,272
|
||||
01/05/2022 01:21:20,176,386
|
||||
01/05/2022 01:21:30,176,501
|
||||
01/05/2022 01:21:40,176,615
|
||||
01/05/2022 01:21:50,176,729
|
||||
01/05/2022 01:22:00,176,843
|
||||
01/05/2022 01:22:10,176,957
|
||||
01/05/2022 01:22:20,177,71
|
||||
01/05/2022 01:22:30,177,185
|
||||
01/05/2022 01:22:40,177,3
|
||||
01/05/2022 01:22:50,177,414
|
||||
01/05/2022 01:23:00,177,528
|
||||
01/05/2022 01:23:10,177,642
|
||||
01/05/2022 01:23:20,177,756
|
||||
01/05/2022 01:23:30,177,87
|
||||
01/05/2022 01:23:40,177,984
|
||||
01/05/2022 01:23:50,178,7
|
||||
01/05/2022 01:24:00,178,15
|
||||
01/05/2022 01:24:10,178,24
|
||||
01/05/2022 01:24:20,178,32
|
||||
01/05/2022 01:24:30,178,4
|
||||
01/05/2022 01:24:40,178,48
|
||||
01/05/2022 01:24:50,178,57
|
||||
01/05/2022 01:25:00,178,65
|
||||
01/05/2022 01:25:10,178,73
|
||||
01/05/2022 01:25:20,178,81
|
||||
01/05/2022 01:25:30,178,9
|
||||
01/05/2022 01:25:40,178,98
|
||||
01/05/2022 01:25:50,178,106
|
||||
01/05/2022 01:26:00,178,114
|
||||
01/05/2022 01:26:10,178,123
|
||||
01/05/2022 01:26:20,178,131
|
||||
01/05/2022 01:26:30,178,139
|
||||
01/05/2022 01:26:40,178,147
|
||||
01/05/2022 01:26:50,178,156
|
||||
01/05/2022 01:27:00,178,164
|
||||
01/05/2022 01:27:10,178,172
|
||||
01/05/2022 01:27:20,178,18
|
||||
01/05/2022 01:27:30,178,189
|
||||
01/05/2022 01:27:40,178,197
|
||||
01/05/2022 01:27:50,178,205
|
||||
01/05/2022 01:28:00,178,213
|
||||
01/05/2022 01:28:10,178,222
|
||||
01/05/2022 01:28:20,178,23
|
||||
01/05/2022 01:28:30,178,238
|
||||
01/05/2022 01:28:40,178,246
|
||||
01/05/2022 01:28:50,178,255
|
||||
01/05/2022 01:29:00,178,263
|
||||
01/05/2022 01:29:10,178,271
|
||||
01/05/2022 01:29:20,178,279
|
||||
01/05/2022 01:29:30,178,288
|
||||
01/05/2022 01:29:40,178,296
|
||||
01/05/2022 01:29:50,178,304
|
||||
01/05/2022 01:30:00,178,312
|
||||
01/05/2022 01:30:10,178,321
|
||||
01/05/2022 01:30:20,178,329
|
||||
01/05/2022 01:30:30,178,337
|
||||
01/05/2022 01:30:40,178,345
|
||||
01/05/2022 01:30:50,178,354
|
||||
01/05/2022 01:31:00,178,362
|
||||
01/05/2022 01:31:10,178,37
|
||||
01/05/2022 01:31:20,178,378
|
||||
01/05/2022 01:31:30,178,387
|
||||
01/05/2022 01:31:40,178,395
|
||||
01/05/2022 01:31:50,178,403
|
||||
01/05/2022 01:32:00,178,411
|
||||
01/05/2022 01:32:10,178,42
|
||||
01/05/2022 01:32:20,178,428
|
||||
01/05/2022 01:32:30,178,436
|
||||
01/05/2022 01:32:40,178,444
|
||||
01/05/2022 01:32:50,178,453
|
||||
01/05/2022 01:33:00,178,461
|
||||
01/05/2022 01:33:10,178,469
|
||||
01/05/2022 01:33:20,178,477
|
||||
01/05/2022 01:33:30,178,486
|
||||
01/05/2022 01:33:40,178,494
|
||||
01/05/2022 01:33:50,178,502
|
||||
01/05/2022 01:34:00,178,51
|
||||
01/05/2022 01:34:10,178,519
|
||||
01/05/2022 01:34:20,178,527
|
||||
01/05/2022 01:34:30,178,535
|
||||
01/05/2022 01:34:40,178,543
|
||||
01/05/2022 01:34:50,178,552
|
||||
01/05/2022 01:35:00,178,56
|
||||
01/05/2022 01:35:10,178,568
|
||||
01/05/2022 01:35:20,178,576
|
||||
01/05/2022 01:35:30,178,585
|
||||
01/05/2022 01:35:40,178,593
|
||||
01/05/2022 01:35:50,178,601
|
||||
01/05/2022 01:36:00,178,609
|
||||
01/05/2022 01:36:10,178,618
|
||||
01/05/2022 01:36:20,178,626
|
||||
01/05/2022 01:36:30,178,634
|
||||
01/05/2022 01:36:40,178,642
|
||||
01/05/2022 01:36:50,178,651
|
||||
01/05/2022 01:37:00,178,659
|
||||
01/05/2022 01:37:10,178,667
|
||||
01/05/2022 01:37:20,178,675
|
||||
01/05/2022 01:37:30,178,684
|
||||
01/05/2022 01:37:40,178,692
|
||||
01/05/2022 01:37:50,178,7
|
||||
01/05/2022 01:38:00,178,708
|
||||
01/05/2022 01:38:10,178,717
|
||||
01/05/2022 01:38:20,178,725
|
||||
01/05/2022 01:38:30,178,733
|
||||
01/05/2022 01:38:40,178,741
|
||||
01/05/2022 01:38:50,178,75
|
||||
01/05/2022 01:39:00,178,758
|
||||
01/05/2022 01:39:10,178,766
|
||||
01/05/2022 01:39:20,178,774
|
||||
01/05/2022 01:39:30,178,783
|
||||
01/05/2022 01:39:40,178,791
|
||||
01/05/2022 01:39:50,178,799
|
||||
01/05/2022 01:40:00,178,807
|
||||
01/05/2022 01:40:10,178,816
|
||||
01/05/2022 01:40:20,178,824
|
||||
01/05/2022 01:40:30,178,832
|
||||
01/05/2022 01:40:40,178,84
|
||||
01/05/2022 01:40:50,178,849
|
||||
01/05/2022 01:41:00,178,857
|
||||
01/05/2022 01:41:10,178,865
|
||||
01/05/2022 01:41:20,178,873
|
||||
01/05/2022 01:41:30,178,882
|
||||
01/05/2022 01:41:40,178,89
|
||||
01/05/2022 01:41:50,178,898
|
||||
01/05/2022 01:42:00,178,906
|
||||
01/05/2022 01:42:10,178,915
|
||||
01/05/2022 01:42:20,178,923
|
||||
01/05/2022 01:42:30,178,931
|
||||
01/05/2022 01:42:40,178,939
|
||||
01/05/2022 01:42:50,178,948
|
||||
01/05/2022 01:43:00,178,956
|
||||
01/05/2022 01:43:10,178,964
|
||||
01/05/2022 01:43:20,178,972
|
||||
01/05/2022 01:43:30,178,981
|
||||
01/05/2022 01:43:40,178,989
|
||||
01/05/2022 01:43:50,178,997
|
||||
01/05/2022 01:44:00,169,147
|
||||
01/05/2022 01:44:10,148,934
|
||||
01/05/2022 01:44:20,119,458
|
||||
01/05/2022 01:44:30,108,326
|
||||
01/05/2022 01:44:40,108,825
|
||||
01/05/2022 01:44:50,109,324
|
||||
01/05/2022 01:45:00,109,824
|
||||
01/05/2022 01:45:10,110,323
|
||||
01/05/2022 01:45:20,110,822
|
||||
01/05/2022 01:45:30,123,3
|
||||
01/05/2022 01:45:40,141,628
|
||||
01/05/2022 01:45:50,160,252
|
||||
01/05/2022 01:46:00,169,863
|
||||
01/05/2022 01:46:10,174,353
|
||||
01/05/2022 01:46:20,176,8
|
||||
01/05/2022 01:46:30,176,19
|
||||
01/05/2022 01:46:40,176,31
|
||||
01/05/2022 01:46:50,176,43
|
||||
01/05/2022 01:47:00,176,55
|
||||
01/05/2022 01:47:10,176,67
|
||||
01/05/2022 01:47:20,176,79
|
||||
01/05/2022 01:47:30,176,91
|
||||
|
37097
docs/data-1749222138290.csv
Normal file
37097
docs/data-1749222138290.csv
Normal file
File diff suppressed because it is too large
Load Diff
@@ -17,6 +17,8 @@
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "timestamp",
|
||||
"dateFormat": "yyyy-MM-dd HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 1,
|
||||
"interactionOnly": false,
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "timestamp",
|
||||
"dateFormat": "yyyy-MM-dd HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 1,
|
||||
"interactionOnly": false,
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "timestamp",
|
||||
"dateFormat": "yyyy-MM-dd HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 2,
|
||||
"interactionOnly": false,
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "timestamp",
|
||||
"dateFormat": "yyyy-MM-dd HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 3,
|
||||
"interactionOnly": false,
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "timestamp",
|
||||
"dateFormat": "yyyy-MM-dd HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 1,
|
||||
"interactionOnly": false,
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "timestamp",
|
||||
"dateFormat": "yyyy-MM-dd HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 1,
|
||||
"interactionOnly": false,
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "timestamp",
|
||||
"dateFormat": "yyyy-MM-dd HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 1,
|
||||
"interactionOnly": false,
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "timestamp",
|
||||
"dateFormat": "yyyy-MM-dd HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 1,
|
||||
"interactionOnly": false,
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "timestamp",
|
||||
"dateFormat": "yyyy-MM-dd HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 2,
|
||||
"interactionOnly": false,
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "timestamp",
|
||||
"dateFormat": "yyyy-MM-dd HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 1,
|
||||
"interactionOnly": false,
|
||||
|
||||
@@ -18,6 +18,8 @@
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "timestamp",
|
||||
"dateFormat": "yyyy-MM-dd HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 1,
|
||||
"interactionOnly": false,
|
||||
|
||||
31
docs/test-scenarios/12-angular-test-date-format.json
Normal file
31
docs/test-scenarios/12-angular-test-date-format.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"_description": "Cenário angular-test-01: CV022 WIT230 com lag e intervalo de datas",
|
||||
"experimentName": "angular-test-01",
|
||||
"username": "lucas.kou@aignosi.com.br",
|
||||
"modelName": "Linear Regression",
|
||||
"targetVariable": "03CV022/CORRENTE_N_M1_PV(Value)",
|
||||
"variableColumns": ["303-WIT-230(Value)"],
|
||||
"lagTrain": {"303-WIT-230(Value)": 3},
|
||||
"lagVal": {"303-WIT-230(Value)": 0},
|
||||
"remStaticWin": false,
|
||||
"lowLim": {},
|
||||
"uppLim": {},
|
||||
"window": 0,
|
||||
"useScaler": false,
|
||||
"includeAr": false,
|
||||
"trainSize": 80,
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "DATA",
|
||||
"dateFormat": "dd/MM/yyyy HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 1,
|
||||
"interactionOnly": false,
|
||||
"nanTreatment": "drop",
|
||||
"startDate": "01/05/2022",
|
||||
"endDate": "31/07/2022",
|
||||
"scalerName": "None",
|
||||
"supportFilters": {},
|
||||
"staticThreshold": null
|
||||
}
|
||||
31
docs/test-scenarios/13-angular-test-double-date-column.json
Normal file
31
docs/test-scenarios/13-angular-test-double-date-column.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"_description": "Cenário angular-test: CV022 WIT230 com ficheiro double date column e intervalo curto (00:00 a 00:05)",
|
||||
"experimentName": "angular-test",
|
||||
"username": "lucas.kou@aignosi.com.br",
|
||||
"modelName": "Linear Regression",
|
||||
"targetVariable": "03CV022/CORRENTE_N_M1_PV(Value)",
|
||||
"variableColumns": ["303-WIT-230(Value)"],
|
||||
"lagTrain": {"303-WIT-230(Value)": 0},
|
||||
"lagVal": {"303-WIT-230(Value)": 0},
|
||||
"remStaticWin": false,
|
||||
"lowLim": {},
|
||||
"uppLim": {},
|
||||
"window": 0,
|
||||
"useScaler": false,
|
||||
"includeAr": false,
|
||||
"trainSize": 80,
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "DATA",
|
||||
"dateFormat": "dd/MM/yyyy HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 1,
|
||||
"interactionOnly": false,
|
||||
"nanTreatment": "drop",
|
||||
"startDate": "01/05/2022 00:00:00",
|
||||
"endDate": "01/05/2022 00:05:10",
|
||||
"scalerName": "None",
|
||||
"supportFilters": {},
|
||||
"staticThreshold": null
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"_description": "Cenário angular-test-01: regressão polinomial degree 4, scaler, support filters em 303-WIT-200",
|
||||
"experimentName": "angular-test-01",
|
||||
"username": "lucas.kou@aignosi.com.br",
|
||||
"modelName": "Polynomial Regression",
|
||||
"targetVariable": "03CV020/CORRENTE_N_M1_PV(Value)",
|
||||
"variableColumns": ["303-WIT-200(Value)"],
|
||||
"lagTrain": {"303-WIT-200(Value)": 0},
|
||||
"lagVal": {"303-WIT-200(Value)": 0},
|
||||
"remStaticWin": false,
|
||||
"lowLim": {},
|
||||
"uppLim": {},
|
||||
"window": 0,
|
||||
"useScaler": true,
|
||||
"includeAr": false,
|
||||
"trainSize": 80,
|
||||
"shuffle": true,
|
||||
"lineSeparator": ",",
|
||||
"decimalSeparator": ".",
|
||||
"dateColumn": "timestamp",
|
||||
"dateFormat": "yyyy-MM-dd HH:mm:ss",
|
||||
"removedIntervals": [],
|
||||
"degree": 4,
|
||||
"interactionOnly": false,
|
||||
"nanTreatment": "drop",
|
||||
"startDate": "2025-06-02 00:00:05",
|
||||
"endDate": "2025-06-06 15:02:01",
|
||||
"scalerName": "Standard Scaler",
|
||||
"supportFilters": {
|
||||
"303-WIT-200(Value)": {
|
||||
"upper_line": {
|
||||
"intercept": 40.400002,
|
||||
"angle": 0
|
||||
},
|
||||
"lower_line": {
|
||||
"intercept": 30.5,
|
||||
"angle": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"staticThreshold": null
|
||||
}
|
||||
@@ -6,6 +6,8 @@ in the PostgreSQL database, extending the base Postgres activity with specialize
|
||||
methods for experiment management.
|
||||
"""
|
||||
|
||||
import enum
|
||||
|
||||
from temporalio import activity, workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
@@ -13,7 +15,6 @@ with workflow.unsafe.imports_passed_through():
|
||||
import traceback
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
@@ -24,7 +25,7 @@ with workflow.unsafe.imports_passed_through():
|
||||
from sqlalchemy import text
|
||||
|
||||
|
||||
class UpdateType(str, Enum):
|
||||
class UpdateType(enum.StrEnum):
|
||||
"""Types of experiment run updates."""
|
||||
|
||||
STATUS = 'status'
|
||||
|
||||
@@ -17,6 +17,36 @@ NORMALIZATION = 'Normalization'
|
||||
FEATURE_CREATION = 'Feature Creation'
|
||||
LAG_CREATION = 'Lag Creation'
|
||||
|
||||
# Allowed frontend date formats and their strftime equivalents (single source of truth)
|
||||
FRONTEND_DATE_FORMAT_TO_STRFTIME = {
|
||||
'dd/MM/yyyy HH:mm:ss': '%d/%m/%Y %H:%M:%S',
|
||||
'MM/dd/yyyy HH:mm:ss': '%m/%d/%Y %H:%M:%S',
|
||||
'yyyy/MM/dd HH:mm:ss': '%Y/%m/%d %H:%M:%S',
|
||||
'dd-MM-yyyy HH:mm:ss': '%d-%m-%Y %H:%M:%S',
|
||||
'MM-dd-yyyy HH:mm:ss': '%m-%d-%Y %H:%M:%S',
|
||||
'yyyy-MM-dd HH:mm:ss': '%Y-%m-%d %H:%M:%S',
|
||||
}
|
||||
ALLOWED_FRONTEND_DATE_FORMATS = frozenset(FRONTEND_DATE_FORMAT_TO_STRFTIME.keys())
|
||||
|
||||
|
||||
def validate_frontend_date_format(fmt: str | None) -> None:
|
||||
"""Raise ValueError if fmt is set and not one of the allowed frontend date formats."""
|
||||
if not fmt or not fmt.strip():
|
||||
return
|
||||
if fmt not in ALLOWED_FRONTEND_DATE_FORMATS:
|
||||
allowed = ', '.join(sorted(ALLOWED_FRONTEND_DATE_FORMATS))
|
||||
raise ValueError(
|
||||
f'Invalid date_format "{fmt}". Allowed formats: {allowed}'
|
||||
)
|
||||
|
||||
|
||||
def _frontend_date_format_to_strftime(fmt: str | None) -> str | None:
|
||||
"""Convert front-end date format to Python strftime. Validates format; returns None for empty."""
|
||||
if not fmt:
|
||||
return None
|
||||
validate_frontend_date_format(fmt)
|
||||
return FRONTEND_DATE_FORMAT_TO_STRFTIME[fmt]
|
||||
|
||||
|
||||
class LinearRegressionModel(BaseEstimator, TransformerMixin):
|
||||
"""
|
||||
@@ -244,6 +274,7 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
|
||||
lag_transform: dict[str, int] | None = None,
|
||||
start_date: str | None = None,
|
||||
end_date: str | None = None,
|
||||
date_format: str | None = None,
|
||||
removed_intervals: list[tuple[str, str]] | None = None,
|
||||
static_threshold: int | None = None,
|
||||
low_lim: dict[str, float] | None = None,
|
||||
@@ -270,8 +301,9 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
|
||||
*Format: {'variable_name': lag}*
|
||||
lag_transform (dict): The lags for each variable to be applyed during transformation \\
|
||||
*Format: {'variable_name': lag}*
|
||||
start_date (str): The start date for filtering data (format: 'YYYY-MM-DD HH:MM:SS')
|
||||
end_date (str): The end date for filtering data (format: 'YYYY-MM-DD HH:MM:SS')
|
||||
start_date (str): The start date for filtering data
|
||||
end_date (str): The end date for filtering data
|
||||
date_format (str | None): Frontend date format for start/end (e.g. dd/MM/yyyy HH:mm:ss or MM/dd/yyyy HH:mm:ss). When set, parsing matches the CSV date column.
|
||||
removed_intervals (list): List of tuples with intervals to remove from data \\
|
||||
*Format: [('start_date', 'end_date'), ...]*
|
||||
static_threshold (int): The number of repeated values to be considered as static
|
||||
@@ -314,6 +346,7 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
|
||||
self.lag_transform = lag_transform if lag_transform else {}
|
||||
self.start_date = start_date
|
||||
self.end_date = end_date
|
||||
self.date_format = date_format
|
||||
self.removed_intervals = removed_intervals if removed_intervals else []
|
||||
self.ar_var = ar_var
|
||||
self.self_operations = self_operations
|
||||
@@ -443,10 +476,19 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
|
||||
return input_data
|
||||
|
||||
def _parse_datetime(self, date_str: str | None) -> pd.Timestamp | None:
|
||||
"""Parse a date string to Timestamp, returning None on failure."""
|
||||
"""Parse a date string to Timestamp using date_format when set.
|
||||
|
||||
When date_format is set (e.g. dd/MM/yyyy HH:mm:ss or MM/dd/yyyy HH:mm:ss),
|
||||
parsing matches the CSV date column so start_date/end_date filter correctly.
|
||||
"""
|
||||
if not date_str:
|
||||
return None
|
||||
try:
|
||||
python_fmt = (
|
||||
_frontend_date_format_to_strftime(self.date_format) if self.date_format else None
|
||||
)
|
||||
if python_fmt:
|
||||
return pd.to_datetime(date_str, format=python_fmt)
|
||||
return pd.to_datetime(date_str)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from enum import Enum
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class ExperimentStatus(str, Enum):
|
||||
class ExperimentStatus(StrEnum):
|
||||
"""
|
||||
Status values for experiment run lifecycle.
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from model_manager.sientia.models import validate_frontend_date_format
|
||||
|
||||
# Model name constants
|
||||
MODEL_LINEAR_REGRESSION = 'Linear Regression'
|
||||
MODEL_POLYNOMIAL_REGRESSION = 'Polynomial Regression'
|
||||
@@ -34,6 +36,8 @@ class TrainModelParams:
|
||||
file_name (str): Name of the file in the MinIO bucket.
|
||||
line_separator (str): Line separator used in the CSV file.
|
||||
decimal_separator (str): Decimal separator used in the CSV file.
|
||||
date_column (str | None): Name of the date/time column. If set with date_format, the column is parsed as datetime.
|
||||
date_format (str | None): Format of the date column (e.g. dd/MM/yyyy HH:mm:ss). Used when date_column is set.
|
||||
train_size (int): Percentage of data to use for training (0-100).
|
||||
shuffle (bool): Whether to shuffle the data during train/test split.
|
||||
experiment_run_id (int): Unique identifier for the experiment run.
|
||||
@@ -64,6 +68,8 @@ class TrainModelParams:
|
||||
file_name: str
|
||||
line_separator: str
|
||||
decimal_separator: str
|
||||
date_column: str | None
|
||||
date_format: str | None
|
||||
train_size: int
|
||||
shuffle: bool
|
||||
experiment_run_id: int
|
||||
@@ -120,6 +126,8 @@ class TrainModelParams:
|
||||
decimal_separator=cls._check_none(
|
||||
data.get('decimal_separator'), str, 'decimal_separator'
|
||||
),
|
||||
date_column=data.get('date_column'),
|
||||
date_format=data.get('date_format'),
|
||||
train_size=cls._check_none(data.get('train_size'), int, 'train_size'),
|
||||
shuffle=cls._check_none(data.get('shuffle'), bool, 'shuffle'),
|
||||
experiment_run_id=cls._check_none(
|
||||
@@ -211,6 +219,7 @@ class TrainModelParams:
|
||||
self._validate_intervals_and_dates()
|
||||
self._validate_limits()
|
||||
self._validate_required_strings()
|
||||
self._validate_date_format()
|
||||
|
||||
def _validate_numeric_ranges(self) -> None:
|
||||
"""Validate numeric parameters are within acceptable ranges."""
|
||||
@@ -321,3 +330,8 @@ class TrainModelParams:
|
||||
|
||||
if not self.experiment_name.strip():
|
||||
raise ValueError('experiment_name cannot be empty or whitespace')
|
||||
|
||||
def _validate_date_format(self) -> None:
|
||||
"""Validate date_format is one of the allowed frontend formats when set."""
|
||||
if self.date_format:
|
||||
validate_frontend_date_format(self.date_format)
|
||||
|
||||
@@ -15,12 +15,103 @@ from sientia_do.operations.df_preprocessor import load_data
|
||||
from sientia_do.operations.normalization import MinMaxScaler, Z_Scaler
|
||||
|
||||
from model_manager.sientia.metrics import mae, mse, r2
|
||||
from model_manager.sientia.models import DataPreprocessor, LinearRegressionModel
|
||||
from model_manager.sientia.models import (
|
||||
DataPreprocessor,
|
||||
LinearRegressionModel,
|
||||
)
|
||||
from model_manager.sientia.models import (
|
||||
_frontend_date_format_to_strftime as _frontend_format_to_strftime,
|
||||
)
|
||||
from model_manager.sientia.utils import split_train_test
|
||||
from model_manager.utils.models.train_model_params import TrainModelParams
|
||||
from model_manager.utils.models.train_model_result import TrainModelResult
|
||||
|
||||
|
||||
def _ensure_date_column_parsed(data: pd.DataFrame, params: TrainModelParams) -> pd.DataFrame:
|
||||
"""If date_column and date_format are set, parse the column as datetime to avoid comparison errors downstream."""
|
||||
if not params.date_column or not params.date_format or params.date_column not in data.columns:
|
||||
return data
|
||||
try:
|
||||
python_fmt = _frontend_format_to_strftime(params.date_format)
|
||||
data = data.copy()
|
||||
data[params.date_column] = pd.to_datetime(
|
||||
data[params.date_column], format=python_fmt, errors='coerce'
|
||||
)
|
||||
except Exception as e:
|
||||
raise ValueError(
|
||||
f'Failed to parse date column "{params.date_column}" with format "{params.date_format}": {e}'
|
||||
) from e
|
||||
return data
|
||||
|
||||
|
||||
def _single_variable_support_mask(
|
||||
data_view: pd.DataFrame,
|
||||
var_col: str,
|
||||
target_variable: str,
|
||||
config: dict,
|
||||
) -> np.ndarray | None:
|
||||
"""Compute keep mask for one variable's support lines; None if config is invalid or skipped."""
|
||||
if var_col not in data_view.columns:
|
||||
return None
|
||||
upper = config.get('upper_line') or config.get('upperLine')
|
||||
lower = config.get('lower_line') or config.get('lowerLine')
|
||||
if not upper or not lower:
|
||||
return None
|
||||
|
||||
x_vals = data_view[var_col].astype(float).to_numpy()
|
||||
y_vals = data_view[target_variable].astype(float).to_numpy()
|
||||
xmin, xmax = float(np.nanmin(x_vals)), float(np.nanmax(x_vals))
|
||||
ymin, ymax = float(np.nanmin(y_vals)), float(np.nanmax(y_vals))
|
||||
x_range = (xmax - xmin) if (xmax - xmin) != 0 else 1.0
|
||||
y_range = (ymax - ymin) if (ymax - ymin) != 0 else 1.0
|
||||
scale_ratio = y_range / x_range
|
||||
|
||||
b1 = float(upper.get('intercept', 0))
|
||||
deg1 = float(upper.get('angle', 0))
|
||||
b2 = float(lower.get('intercept', 0))
|
||||
deg2 = float(lower.get('angle', 0))
|
||||
m1 = np.tan(np.deg2rad(deg1)) * scale_ratio
|
||||
m2 = np.tan(np.deg2rad(deg2)) * scale_ratio
|
||||
y1 = m1 * x_vals + b1
|
||||
y2 = m2 * x_vals + b2
|
||||
lower_bound = np.minimum(y1, y2)
|
||||
upper_bound = np.maximum(y1, y2)
|
||||
return (y_vals >= lower_bound) & (y_vals <= upper_bound)
|
||||
|
||||
|
||||
def _apply_support_filters(
|
||||
data_view: pd.DataFrame,
|
||||
target_variable: str,
|
||||
support_filters: dict,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Keep only rows where (var, target) lies between the two guide lines for each variable.
|
||||
|
||||
For each variable in support_filters, the condition is lower(x_var) <= target <= upper(x_var),
|
||||
where lower/upper are the two lines (intercept + slope from angle, scaled by y_range/x_range).
|
||||
Global mask is AND across all variables. Matches DEMO logic in template_01.py.
|
||||
|
||||
Args:
|
||||
data_view: DataFrame after preprocessor transform.
|
||||
target_variable: Name of the target column (y axis).
|
||||
support_filters: Per-variable config with upper_line/lower_line, each {intercept, angle}.
|
||||
|
||||
Returns:
|
||||
data_view filtered to rows satisfying all variable conditions; unchanged if support_filters empty.
|
||||
"""
|
||||
if not support_filters or target_variable not in data_view.columns:
|
||||
return data_view
|
||||
|
||||
combined_keep_mask = np.ones(len(data_view), dtype=bool)
|
||||
n = len(data_view)
|
||||
for var_col, config in support_filters.items():
|
||||
keep_mask = _single_variable_support_mask(data_view, var_col, target_variable, config)
|
||||
if keep_mask is not None and len(keep_mask) == n:
|
||||
combined_keep_mask &= keep_mask
|
||||
|
||||
return data_view.loc[combined_keep_mask]
|
||||
|
||||
|
||||
class TrainingRepository:
|
||||
"""
|
||||
Repository for machine learning model training operations.
|
||||
@@ -67,15 +158,25 @@ class TrainingRepository:
|
||||
Exception: If data loading, preprocessing, or training fails
|
||||
"""
|
||||
data = load_data(uploaded_file, params.line_separator, params.decimal_separator)
|
||||
|
||||
# Configure datetime index if timestamp column exists
|
||||
# Required for TimeSeriesDiscontinuityAnalyzer (static window removal)
|
||||
data = self._configure_datetime_index(data)
|
||||
if data is None:
|
||||
raise ValueError(
|
||||
'Failed to load CSV data: load_data returned None. '
|
||||
'Check file encoding, line separator and decimal separator.'
|
||||
)
|
||||
data = _ensure_date_column_parsed(data, params)
|
||||
data = self._configure_datetime_index(data, params)
|
||||
|
||||
process_data = self._init_data_preprocessor(params)
|
||||
process_data.fit(data)
|
||||
data_view = process_data.transform(data)
|
||||
|
||||
if params.support_filters:
|
||||
data_view = _apply_support_filters(
|
||||
data_view,
|
||||
params.target_variable,
|
||||
params.support_filters,
|
||||
)
|
||||
|
||||
if len(data_view) <= 0:
|
||||
raise ValueError('Data view is empty after transformation')
|
||||
|
||||
@@ -282,6 +383,7 @@ class TrainingRepository:
|
||||
lag_transform=params.lag_val,
|
||||
start_date=params.start_date,
|
||||
end_date=params.end_date,
|
||||
date_format=params.date_format,
|
||||
removed_intervals=removed_intervals,
|
||||
static_threshold=self._get_static_threshold(params),
|
||||
low_lim=params.low_lim,
|
||||
@@ -350,40 +452,41 @@ class TrainingRepository:
|
||||
'original_features': params.variable_columns,
|
||||
}
|
||||
|
||||
def _configure_datetime_index(self, data: pd.DataFrame) -> pd.DataFrame:
|
||||
def _configure_datetime_index(
|
||||
self, data: pd.DataFrame | None, params: TrainModelParams
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Configure datetime index for the DataFrame.
|
||||
|
||||
This method attempts to identify a timestamp column and set it as the
|
||||
DataFrame index with DatetimeIndex type. This is required for
|
||||
TimeSeriesDiscontinuityAnalyzer (used in static window removal).
|
||||
|
||||
The method looks for common timestamp column names and converts the
|
||||
first matching column to datetime, then sets it as the index.
|
||||
|
||||
Args:
|
||||
data: Input DataFrame
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: DataFrame with DatetimeIndex if timestamp column found,
|
||||
otherwise returns original DataFrame unchanged
|
||||
Guards against None to avoid 'NoneType' object has no attribute 'index' downstream.
|
||||
Prefers params.date_column when set; otherwise looks for common timestamp column names.
|
||||
"""
|
||||
# If index is already DatetimeIndex, just ensure it's sorted
|
||||
if data is None:
|
||||
raise ValueError(
|
||||
'Data is None after load_data. '
|
||||
'Check file format, line separator and decimal separator.'
|
||||
)
|
||||
if not isinstance(data, pd.DataFrame):
|
||||
raise TypeError(f'Expected DataFrame, got {type(data).__name__}')
|
||||
|
||||
if isinstance(data.index, pd.DatetimeIndex):
|
||||
self.logger.info('DataFrame already has DatetimeIndex')
|
||||
return data.sort_index()
|
||||
|
||||
# Common timestamp column names
|
||||
timestamp_columns = [
|
||||
common_timestamp_columns = [
|
||||
'timestamp',
|
||||
'Timestamp',
|
||||
'TIMESTAMP',
|
||||
'date',
|
||||
'Date',
|
||||
'DATE',
|
||||
'DATA',
|
||||
'datetime',
|
||||
'DateTime',
|
||||
]
|
||||
timestamp_columns = ([params.date_column] if params.date_column else []) + [
|
||||
c for c in common_timestamp_columns if c != params.date_column
|
||||
]
|
||||
|
||||
for col in timestamp_columns:
|
||||
if col in data.columns:
|
||||
|
||||
@@ -14,18 +14,18 @@ import asyncio
|
||||
import os
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
from dotenv import load_dotenv
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from temporalio.client import Client
|
||||
|
||||
# Ensure project root is on PYTHONPATH when running directly
|
||||
# Ensure project root is on PYTHONPATH when running directly (must run before model_manager import)
|
||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if ROOT_DIR not in sys.path:
|
||||
sys.path.insert(0, ROOT_DIR)
|
||||
|
||||
from model_manager.workflows.cleanup_files import CleanupFiles
|
||||
|
||||
from model_manager.workflows.cleanup_files import CleanupFiles # noqa: E402
|
||||
|
||||
# Carrega variáveis de ambiente do arquivo .env na raiz do projeto
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
@@ -53,7 +53,7 @@ async def main(argv: list[str]) -> None:
|
||||
if argv:
|
||||
bucket_name = argv[0]
|
||||
|
||||
print(f"Connecting to Temporal at {temporal_host} (namespace={temporal_namespace})...")
|
||||
print(f'Connecting to Temporal at {temporal_host} (namespace={temporal_namespace})...')
|
||||
client = await Client.connect(
|
||||
target_host=temporal_host,
|
||||
namespace=temporal_namespace,
|
||||
@@ -64,12 +64,14 @@ async def main(argv: list[str]) -> None:
|
||||
'bucket_name': bucket_name,
|
||||
}
|
||||
|
||||
workflow_id = f"cleanup-files-manual-{int(asyncio.get_event_loop().time())}"
|
||||
workflow_id = f'cleanup-files-manual-{int(asyncio.get_event_loop().time())}'
|
||||
|
||||
print(f"Starting cleanup_files workflow once...\n"
|
||||
f" workflow_id = {workflow_id}\n"
|
||||
f" task_queue = {task_queue}\n"
|
||||
f" bucket_name = {bucket_name}")
|
||||
print(
|
||||
f'Starting cleanup_files workflow once...\n'
|
||||
f' workflow_id = {workflow_id}\n'
|
||||
f' task_queue = {task_queue}\n'
|
||||
f' bucket_name = {bucket_name}'
|
||||
)
|
||||
|
||||
handle = await client.start_workflow(
|
||||
CleanupFiles.run,
|
||||
@@ -79,9 +81,9 @@ async def main(argv: list[str]) -> None:
|
||||
run_timeout=timedelta(minutes=10),
|
||||
)
|
||||
|
||||
print("Workflow started, waiting for completion...")
|
||||
print('Workflow started, waiting for completion...')
|
||||
await handle.result()
|
||||
print("cleanup_files workflow completed successfully.")
|
||||
print('cleanup_files workflow completed successfully.')
|
||||
|
||||
|
||||
if __name__ == '__main__': # pragma: no cover - manual utility script
|
||||
|
||||
@@ -1,12 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Utility script to trigger the training workflow end-to-end for testing.
|
||||
|
||||
Steps performed:
|
||||
Steps performed (default):
|
||||
1. Upload the CSV test dataset to MinIO using the configured `mc` alias.
|
||||
2. Insert a new experiment_run record in Postgres and capture the generated ID.
|
||||
3. Trigger the Temporal `train_model` workflow with the correct payload.
|
||||
|
||||
Prerequisites:
|
||||
Alternatives for local diagnosis (--local / --validate-only):
|
||||
- example: python scripts/run_training_test.py --scenario 01-linear-regression-basic --local --csv docs/test-model-data.csv
|
||||
- --validate-only: Validates scenario parameters only (no MinIO, Postgres, Temporal).
|
||||
- --local: Runs the same training pipeline locally (validate + load CSV + train +
|
||||
after_train_calculation). Use to get full Python tracebacks for debugging.
|
||||
Does not upload to MinIO, insert DB, or start Temporal.
|
||||
By default skips MLflow save; use --local-save-mlflow to also test saving to MLflow.
|
||||
|
||||
Prerequisites (default flow):
|
||||
- `mc` CLI configured with alias defined in MINIO_ALIAS.
|
||||
- PostgreSQL accessible with credentials in environment variables or defaults.
|
||||
- Temporal server reachable without TLS on TEMPORAL_HOST / TEMPORAL_NAMESPACE.
|
||||
@@ -23,6 +31,7 @@ import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg2
|
||||
@@ -30,7 +39,6 @@ from dotenv import load_dotenv
|
||||
from psycopg2.extras import Json
|
||||
from temporalio import client
|
||||
|
||||
|
||||
# Carrega variáveis de ambiente do arquivo .env na raiz do projeto
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
ENV_PATH = PROJECT_ROOT / '.env'
|
||||
@@ -97,6 +105,19 @@ def load_scenario(scenario_name: str) -> dict:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _resolve_csv_path(csv_path: Path) -> Path:
|
||||
"""Resolve CSV path; if not found in project root, try docs/."""
|
||||
if csv_path.is_absolute():
|
||||
return csv_path
|
||||
resolved = PROJECT_ROOT / csv_path
|
||||
if resolved.exists():
|
||||
return resolved
|
||||
docs_path = PROJECT_ROOT / 'docs' / csv_path.name
|
||||
if docs_path.exists():
|
||||
return docs_path
|
||||
return resolved
|
||||
|
||||
|
||||
def _ensure_source_file(path: Path) -> None:
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f'Test dataset not found at {path.resolve()}')
|
||||
@@ -186,6 +207,8 @@ def build_workflow_payload(
|
||||
'file_name': file_name,
|
||||
'line_separator': request_data['lineSeparator'],
|
||||
'decimal_separator': request_data['decimalSeparator'],
|
||||
'date_column': request_data.get('dateColumn'),
|
||||
'date_format': request_data.get('dateFormat'),
|
||||
'removed_intervals': request_data['removedIntervals'],
|
||||
# New parameters
|
||||
'model_name': request_data.get('modelName', 'Linear Regression'),
|
||||
@@ -196,6 +219,7 @@ def build_workflow_payload(
|
||||
'end_date': request_data.get('endDate'),
|
||||
'scaler_name': request_data.get('scalerName', 'None'),
|
||||
'support_filters': request_data.get('supportFilters', {}),
|
||||
'static_threshold': request_data.get('staticThreshold'),
|
||||
}
|
||||
|
||||
|
||||
@@ -220,6 +244,110 @@ async def trigger_temporal_workflow(workflow_input: dict) -> str:
|
||||
return workflow_id
|
||||
|
||||
|
||||
def _build_local_payload(request_data: dict, csv_path: Path) -> dict:
|
||||
"""Build workflow payload for local run (no real experiment_run_id)."""
|
||||
return build_workflow_payload(
|
||||
experiment_run_id=0,
|
||||
file_name=csv_path.name,
|
||||
request_data=request_data,
|
||||
)
|
||||
|
||||
|
||||
def run_validate_only(scenario_name: str) -> dict:
|
||||
"""Validate scenario parameters only. No MinIO, Postgres, or Temporal.
|
||||
|
||||
Returns:
|
||||
dict: {'success': bool, 'error': str | None, 'scenario': str}
|
||||
"""
|
||||
from model_manager.utils.models.train_model_params import TrainModelParams
|
||||
|
||||
result = {'scenario': scenario_name, 'success': False, 'error': None}
|
||||
try:
|
||||
request_data = load_scenario(scenario_name)
|
||||
except FileNotFoundError as exc:
|
||||
result['error'] = str(exc)
|
||||
return result
|
||||
|
||||
payload = _build_local_payload(request_data, Path('local.csv'))
|
||||
try:
|
||||
train_params = TrainModelParams.from_dict(payload)
|
||||
train_params.validate_business_rules()
|
||||
result['success'] = True
|
||||
except (ValueError, TypeError, KeyError) as e:
|
||||
result['error'] = str(e)
|
||||
return result
|
||||
|
||||
|
||||
def run_local_pipeline(
|
||||
scenario_name: str,
|
||||
csv_path: Path,
|
||||
save_mlflow: bool = False,
|
||||
) -> dict:
|
||||
"""Run the same training pipeline locally (validate + train + after_train).
|
||||
|
||||
Reads CSV from disk, runs TrainingRepository.train and after_train_calculation.
|
||||
Optionally saves to MLflow if save_mlflow is True (requires MLflow env).
|
||||
|
||||
Returns:
|
||||
dict: {'success': bool, 'error': str | None, 'scenario': str, ...}
|
||||
"""
|
||||
from model_manager.utils.logger_helper import get_logger
|
||||
from model_manager.utils.models.train_model_params import TrainModelParams
|
||||
from model_manager.utils.repository.training_repository import TrainingRepository
|
||||
|
||||
result = {
|
||||
'scenario': scenario_name,
|
||||
'success': False,
|
||||
'error': None,
|
||||
}
|
||||
try:
|
||||
request_data = load_scenario(scenario_name)
|
||||
except FileNotFoundError as exc:
|
||||
result['error'] = str(exc)
|
||||
return result
|
||||
|
||||
csv_path = _resolve_csv_path(csv_path)
|
||||
_ensure_source_file(csv_path)
|
||||
payload = _build_local_payload(request_data, csv_path)
|
||||
try:
|
||||
train_params = TrainModelParams.from_dict(payload)
|
||||
train_params.validate_business_rules()
|
||||
except (ValueError, TypeError, KeyError) as e:
|
||||
result['error'] = f'Validation failed: {e}'
|
||||
return result
|
||||
|
||||
logger = get_logger(__name__)
|
||||
training_repository = TrainingRepository(logger)
|
||||
|
||||
with open(csv_path, 'rb') as f:
|
||||
file_content = BytesIO(f.read())
|
||||
|
||||
try:
|
||||
train_result = training_repository.train(file_content, train_params)
|
||||
train_result = training_repository.after_train_calculation(train_params, train_result)
|
||||
except Exception as e:
|
||||
result['error'] = str(e)
|
||||
raise # re-raise so caller gets full traceback for diagnosis
|
||||
|
||||
if save_mlflow:
|
||||
from model_manager.utils.connectors_config import build_mlflow_config
|
||||
from model_manager.utils.repository.model_repository import ModelRepository
|
||||
|
||||
mlflow_config = build_mlflow_config()
|
||||
model_repository = ModelRepository(
|
||||
url=mlflow_config['url'],
|
||||
username=mlflow_config['username'],
|
||||
password=mlflow_config['password'],
|
||||
logger=logger,
|
||||
)
|
||||
train_result = model_repository.save_model(train_result)
|
||||
|
||||
result['success'] = True
|
||||
result['run_name'] = getattr(train_result, 'run_name', None)
|
||||
result['run_dir'] = getattr(train_result, 'run_dir', None)
|
||||
return result
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""Parse command line arguments."""
|
||||
parser = argparse.ArgumentParser(
|
||||
@@ -241,6 +369,15 @@ Examples:
|
||||
|
||||
# Run all scenarios
|
||||
python scripts/run_training_test.py --all
|
||||
|
||||
# Validate scenario parameters only (no external services)
|
||||
python scripts/run_training_test.py --scenario linear-regression-basic --validate-only
|
||||
|
||||
# Run training pipeline locally to diagnose errors (full traceback)
|
||||
python scripts/run_training_test.py --scenario linear-regression-basic --local --csv docs/test-model-data.csv
|
||||
|
||||
# Local run and save to MLflow (requires MLflow env)
|
||||
python scripts/run_training_test.py --scenario linear-regression-basic --local --local-save-mlflow
|
||||
""",
|
||||
)
|
||||
parser.add_argument(
|
||||
@@ -268,6 +405,21 @@ Examples:
|
||||
action='store_true',
|
||||
help='Run all available test scenarios sequentially.',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--validate-only',
|
||||
action='store_true',
|
||||
help='Only validate scenario parameters (no MinIO, Postgres, Temporal).',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--local',
|
||||
action='store_true',
|
||||
help='Run training pipeline locally (validate + train from CSV) to get full tracebacks.',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--local-save-mlflow',
|
||||
action='store_true',
|
||||
help='With --local, also save the model to MLflow (requires MLflow env).',
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -293,16 +445,18 @@ def run_single_scenario(scenario_name: str, csv_path: Path) -> dict:
|
||||
# Load scenario
|
||||
try:
|
||||
experiment_request = load_scenario(scenario_name)
|
||||
print(f" Loaded scenario: {scenario_name}")
|
||||
print(f' Loaded scenario: {scenario_name}')
|
||||
except FileNotFoundError as exc:
|
||||
result['error'] = str(exc)
|
||||
return result
|
||||
|
||||
csv_path = _resolve_csv_path(csv_path)
|
||||
|
||||
# Upload CSV to MinIO
|
||||
try:
|
||||
uploaded_file_name = upload_to_minio(csv_path)
|
||||
result['s3_object_name'] = uploaded_file_name
|
||||
print(f" Uploaded CSV to MinIO: {uploaded_file_name}")
|
||||
print(f' Uploaded CSV to MinIO: {uploaded_file_name}')
|
||||
except subprocess.CalledProcessError as exc:
|
||||
result['error'] = f'Failed to upload file to MinIO: {exc}'
|
||||
return result
|
||||
@@ -314,7 +468,7 @@ def run_single_scenario(scenario_name: str, csv_path: Path) -> dict:
|
||||
try:
|
||||
experiment_run_id = insert_experiment_run(uploaded_file_name, experiment_request)
|
||||
result['experiment_run_id'] = experiment_run_id
|
||||
print(f" Created experiment_run with ID: {experiment_run_id}")
|
||||
print(f' Created experiment_run with ID: {experiment_run_id}')
|
||||
except psycopg2.Error as exc:
|
||||
result['error'] = f'Database error while inserting experiment_run: {exc}'
|
||||
return result
|
||||
@@ -330,7 +484,7 @@ def run_single_scenario(scenario_name: str, csv_path: Path) -> dict:
|
||||
workflow_id = asyncio.run(trigger_temporal_workflow(workflow_payload))
|
||||
result['workflow_id'] = workflow_id
|
||||
result['success'] = True
|
||||
print(f" Workflow started: {workflow_id}")
|
||||
print(f' Workflow started: {workflow_id}')
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result['error'] = f'Failed to start Temporal workflow: {exc}'
|
||||
return result
|
||||
@@ -350,74 +504,100 @@ def print_summary(results: list[dict]) -> None:
|
||||
print('\n' + '=' * 60)
|
||||
print('SUMMARY')
|
||||
print('=' * 60)
|
||||
print(f"Total: {len(results)} | Passed: {len(passed)} | Failed: {len(failed)}")
|
||||
print(f'Total: {len(results)} | Passed: {len(passed)} | Failed: {len(failed)}')
|
||||
print('=' * 60)
|
||||
|
||||
if passed:
|
||||
print('\n✓ PASSED:')
|
||||
for r in passed:
|
||||
print(f" - {r['scenario']}")
|
||||
print(f' - {r["scenario"]}')
|
||||
|
||||
if failed:
|
||||
print('\n✗ FAILED:')
|
||||
for r in failed:
|
||||
print(f" - {r['scenario']}")
|
||||
print(f' - {r["scenario"]}')
|
||||
if r['error']:
|
||||
print(f" Error: {r['error']}")
|
||||
print(f' Error: {r["error"]}')
|
||||
|
||||
print()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
def _handle_list_scenarios() -> None:
|
||||
"""Print available scenarios and exit."""
|
||||
scenarios = list_available_scenarios()
|
||||
if scenarios:
|
||||
print('Available test scenarios:')
|
||||
for scenario in scenarios:
|
||||
print(f' - {scenario}')
|
||||
else:
|
||||
print(f'No scenarios found in {TEST_SCENARIOS_DIR}')
|
||||
sys.exit(0)
|
||||
|
||||
# List scenarios and exit if requested
|
||||
if args.list:
|
||||
scenarios = list_available_scenarios()
|
||||
if scenarios:
|
||||
print('Available test scenarios:')
|
||||
for scenario in scenarios:
|
||||
print(f' - {scenario}')
|
||||
else:
|
||||
print(f'No scenarios found in {TEST_SCENARIOS_DIR}')
|
||||
sys.exit(0)
|
||||
|
||||
# Run all scenarios if requested
|
||||
if args.all:
|
||||
scenarios = list_available_scenarios()
|
||||
if not scenarios:
|
||||
print(f'No scenarios found in {TEST_SCENARIOS_DIR}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f'Running {len(scenarios)} scenarios...\n')
|
||||
results = []
|
||||
|
||||
for i, scenario in enumerate(scenarios, 1):
|
||||
print(f'[{i}/{len(scenarios)}] Running scenario: {scenario}')
|
||||
result = run_single_scenario(scenario, args.csv)
|
||||
results.append(result)
|
||||
status = '✓' if result['success'] else '✗'
|
||||
print(f'[{i}/{len(scenarios)}] {status} {scenario}\n')
|
||||
|
||||
print_summary(results)
|
||||
|
||||
# Exit with error code if any scenario failed
|
||||
failed_count = sum(1 for r in results if not r['success'])
|
||||
sys.exit(1 if failed_count > 0 else 0)
|
||||
|
||||
# Require scenario argument if not listing or running all
|
||||
if not args.scenario:
|
||||
print('Error: --scenario or --all is required. Use --list to see available scenarios.', file=sys.stderr)
|
||||
def _handle_run_all(args: argparse.Namespace) -> None:
|
||||
"""Run all scenarios and exit with appropriate code."""
|
||||
scenarios = list_available_scenarios()
|
||||
if not scenarios:
|
||||
print(f'No scenarios found in {TEST_SCENARIOS_DIR}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Run single scenario
|
||||
print(f'Running {len(scenarios)} scenarios...\n')
|
||||
results = []
|
||||
for i, scenario in enumerate(scenarios, 1):
|
||||
print(f'[{i}/{len(scenarios)}] Running scenario: {scenario}')
|
||||
result = run_single_scenario(scenario, args.csv)
|
||||
results.append(result)
|
||||
status = '✓' if result['success'] else '✗'
|
||||
print(f'[{i}/{len(scenarios)}] {status} {scenario}\n')
|
||||
|
||||
print_summary(results)
|
||||
failed_count = sum(1 for r in results if not r['success'])
|
||||
sys.exit(1 if failed_count > 0 else 0)
|
||||
|
||||
|
||||
def _handle_validate_only(args: argparse.Namespace) -> None:
|
||||
"""Validate scenario parameters only and exit."""
|
||||
if not args.scenario:
|
||||
print('Error: --scenario is required with --validate-only.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
result = run_validate_only(args.scenario)
|
||||
if result['success']:
|
||||
print(f'Validation OK: {result["scenario"]}')
|
||||
else:
|
||||
print(f'Validation failed: {result["error"]}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def _handle_local(args: argparse.Namespace) -> None:
|
||||
"""Run local pipeline and exit."""
|
||||
if not args.scenario:
|
||||
print('Error: --scenario is required with --local.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f'Running local pipeline: {args.scenario} (CSV: {args.csv})')
|
||||
result = run_local_pipeline(
|
||||
args.scenario,
|
||||
args.csv,
|
||||
save_mlflow=args.local_save_mlflow,
|
||||
)
|
||||
if not result['success']:
|
||||
print(f'Error: {result["error"]}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
out = {'scenario': result['scenario'], 'success': True}
|
||||
if result.get('run_name') is not None:
|
||||
out['run_name'] = result['run_name']
|
||||
if result.get('run_dir') is not None:
|
||||
out['run_dir'] = result['run_dir']
|
||||
print(json.dumps(out, indent=2))
|
||||
|
||||
|
||||
def _handle_single_scenario(args: argparse.Namespace) -> None:
|
||||
"""Run one scenario (MinIO + Postgres + Temporal) and print result."""
|
||||
print(f'Running scenario: {args.scenario}')
|
||||
result = run_single_scenario(args.scenario, args.csv)
|
||||
|
||||
if not result['success']:
|
||||
print(f"Error: {result['error']}", file=sys.stderr)
|
||||
print(f'Error: {result["error"]}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
@@ -431,5 +611,31 @@ def main() -> None:
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
|
||||
if args.list:
|
||||
_handle_list_scenarios()
|
||||
|
||||
if args.all:
|
||||
_handle_run_all(args)
|
||||
|
||||
if args.validate_only:
|
||||
_handle_validate_only(args)
|
||||
|
||||
if args.local:
|
||||
_handle_local(args)
|
||||
return
|
||||
|
||||
if not args.scenario:
|
||||
print(
|
||||
'Error: --scenario or --all is required. Use --list to see available scenarios.',
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
_handle_single_scenario(args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@@ -7,11 +7,58 @@ import pandas as pd
|
||||
from pytest import raises
|
||||
|
||||
from model_manager.sientia.models import (
|
||||
ALLOWED_FRONTEND_DATE_FORMATS,
|
||||
FRONTEND_DATE_FORMAT_TO_STRFTIME,
|
||||
DataPreprocessor,
|
||||
LinearRegressionModel,
|
||||
_frontend_date_format_to_strftime,
|
||||
validate_frontend_date_format,
|
||||
)
|
||||
|
||||
|
||||
class TestFrontendDateFormatToStrftime:
|
||||
"""Tests for _frontend_date_format_to_strftime (models module)."""
|
||||
|
||||
def test_none_or_empty_returns_none(self):
|
||||
"""None or empty string returns None."""
|
||||
assert _frontend_date_format_to_strftime(None) is None
|
||||
assert _frontend_date_format_to_strftime('') is None
|
||||
|
||||
def test_all_six_allowed_formats_convert_correctly(self):
|
||||
"""All allowed frontend formats map to expected strftime."""
|
||||
for frontend_fmt, strftime_fmt in FRONTEND_DATE_FORMAT_TO_STRFTIME.items():
|
||||
assert _frontend_date_format_to_strftime(frontend_fmt) == strftime_fmt
|
||||
|
||||
def test_invalid_format_raises(self):
|
||||
"""Invalid format raises ValueError with allowed list in message."""
|
||||
with raises(ValueError, match='Invalid date_format'):
|
||||
_frontend_date_format_to_strftime('yyyy-MM-dd')
|
||||
with raises(ValueError, match='Allowed formats'):
|
||||
_frontend_date_format_to_strftime('invalid')
|
||||
|
||||
|
||||
class TestValidateFrontendDateFormat:
|
||||
"""Tests for validate_frontend_date_format."""
|
||||
|
||||
def test_none_or_empty_does_not_raise(self):
|
||||
"""None or empty string does not raise."""
|
||||
validate_frontend_date_format(None)
|
||||
validate_frontend_date_format('')
|
||||
validate_frontend_date_format(' ')
|
||||
|
||||
def test_allowed_formats_do_not_raise(self):
|
||||
"""All allowed formats pass validation."""
|
||||
for fmt in ALLOWED_FRONTEND_DATE_FORMATS:
|
||||
validate_frontend_date_format(fmt)
|
||||
|
||||
def test_invalid_format_raises(self):
|
||||
"""Invalid format raises ValueError."""
|
||||
with raises(ValueError, match='Invalid date_format'):
|
||||
validate_frontend_date_format('yyyy-MM-dd')
|
||||
with raises(ValueError, match='Invalid date_format'):
|
||||
validate_frontend_date_format('custom-bad-format')
|
||||
|
||||
|
||||
class _IterableWithContains:
|
||||
def __init__(self, iterable, contains_values):
|
||||
self._iterable = iterable
|
||||
|
||||
@@ -555,6 +555,27 @@ def test_validate_business_rules_valid_start_and_end_date(valid_train_params_dic
|
||||
params.validate_business_rules() # Should not raise
|
||||
|
||||
|
||||
def test_validate_business_rules_invalid_date_format(valid_train_params_dict):
|
||||
"""Test validate_business_rules raises when date_format is not allowed."""
|
||||
from model_manager.utils.models.train_model_params import TrainModelParams
|
||||
|
||||
params = TrainModelParams.from_dict(valid_train_params_dict)
|
||||
params.date_format = 'yyyy-MM-dd'
|
||||
|
||||
with pytest.raises(ValueError, match='Invalid date_format'):
|
||||
params.validate_business_rules()
|
||||
|
||||
|
||||
def test_validate_business_rules_valid_date_format(valid_train_params_dict):
|
||||
"""Test validate_business_rules accepts allowed date_format."""
|
||||
from model_manager.utils.models.train_model_params import TrainModelParams
|
||||
|
||||
params = TrainModelParams.from_dict(valid_train_params_dict)
|
||||
params.date_format = 'yyyy-MM-dd HH:mm:ss'
|
||||
|
||||
params.validate_business_rules() # Should not raise
|
||||
|
||||
|
||||
def test_validate_business_rules_polynomial_regression_valid(valid_train_params_dict):
|
||||
"""Test validate_business_rules accepts valid Polynomial Regression config."""
|
||||
from model_manager.utils.models.train_model_params import TrainModelParams
|
||||
|
||||
@@ -41,6 +41,8 @@ def sample_params():
|
||||
scaler_name='Standard Scaler',
|
||||
support_filters={},
|
||||
static_threshold=None,
|
||||
date_column=None,
|
||||
date_format=None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -344,9 +344,14 @@ def test_create_run_directory_success(
|
||||
|
||||
result = repo._create_run_directory('/tmp/reports', 'test_run') # noqa: S108
|
||||
|
||||
expected_path = os.path.join('/tmp/reports/temp', 'test_run_20240101_120000_123456') # noqa: S108
|
||||
assert result == expected_path
|
||||
mock_makedirs.assert_called_once_with(expected_path, exist_ok=True)
|
||||
expected_path = os.path.normpath(
|
||||
os.path.join('/tmp/reports', 'temp', 'test_run_20240101_120000_123456') # noqa: S108
|
||||
)
|
||||
assert os.path.normpath(result) == expected_path
|
||||
mock_makedirs.assert_called_once()
|
||||
call_path = mock_makedirs.call_args[0][0]
|
||||
assert os.path.normpath(call_path) == expected_path
|
||||
assert mock_makedirs.call_args[1] == {'exist_ok': True}
|
||||
|
||||
|
||||
@patch('model_manager.utils.repository.model_repository.ModelServing')
|
||||
|
||||
@@ -10,7 +10,12 @@ import pytest
|
||||
from model_manager.sientia.models import LinearRegressionModel
|
||||
from model_manager.utils.models.train_model_params import TrainModelParams
|
||||
from model_manager.utils.models.train_model_result import TrainModelResult
|
||||
from model_manager.utils.repository.training_repository import TrainingRepository
|
||||
from model_manager.utils.repository.training_repository import (
|
||||
TrainingRepository,
|
||||
_apply_support_filters,
|
||||
_ensure_date_column_parsed,
|
||||
_frontend_format_to_strftime,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -57,6 +62,8 @@ def sample_params():
|
||||
scaler_name='None',
|
||||
support_filters={},
|
||||
static_threshold=None,
|
||||
date_column=None,
|
||||
date_format=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -139,6 +146,8 @@ class TestExtractModelEquation:
|
||||
scaler_name='None',
|
||||
support_filters={},
|
||||
static_threshold=None,
|
||||
date_column=None,
|
||||
date_format=None,
|
||||
)
|
||||
|
||||
# Mock model with single coefficient
|
||||
@@ -585,7 +594,7 @@ class TestTrain:
|
||||
"""
|
||||
return BytesIO(csv_content.encode('utf-8'))
|
||||
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df)
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df, params: df)
|
||||
@patch('model_manager.utils.repository.training_repository.split_train_test')
|
||||
@patch('model_manager.utils.repository.training_repository.load_data')
|
||||
def test_train_basic_workflow(
|
||||
@@ -631,7 +640,7 @@ class TestTrain:
|
||||
# Verify split was called
|
||||
assert mock_split_train_test.called
|
||||
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df)
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df, params: df)
|
||||
@patch('model_manager.utils.repository.training_repository.split_train_test')
|
||||
@patch('model_manager.utils.repository.training_repository.load_data')
|
||||
def test_train_with_scaler(
|
||||
@@ -661,7 +670,7 @@ class TestTrain:
|
||||
assert result is not None
|
||||
assert result.scaler_dict is not None
|
||||
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df)
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df, params: df)
|
||||
@patch('model_manager.utils.repository.training_repository.split_train_test')
|
||||
@patch('model_manager.utils.repository.training_repository.load_data')
|
||||
def test_train_with_shuffle_enabled(
|
||||
@@ -692,7 +701,7 @@ class TestTrain:
|
||||
call_kwargs = mock_split_train_test.call_args[1]
|
||||
assert call_kwargs['shuffle'] is True
|
||||
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df)
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df, params: df)
|
||||
@patch('model_manager.utils.repository.training_repository.split_train_test')
|
||||
@patch('model_manager.utils.repository.training_repository.load_data')
|
||||
def test_train_with_different_train_size(
|
||||
@@ -723,7 +732,7 @@ class TestTrain:
|
||||
call_kwargs = mock_split_train_test.call_args[1]
|
||||
assert call_kwargs['train_size'] == 0.7
|
||||
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df)
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df, params: df)
|
||||
@patch('model_manager.utils.repository.training_repository.split_train_test')
|
||||
@patch('model_manager.utils.repository.training_repository.load_data')
|
||||
def test_train_raises_on_empty_data_after_transform(
|
||||
@@ -744,7 +753,7 @@ class TestTrain:
|
||||
with pytest.raises(ValueError, match='Data view is empty after transformation'):
|
||||
training_repo.train(sample_csv_data, sample_params)
|
||||
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df)
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df, params: df)
|
||||
@patch('model_manager.utils.repository.training_repository.split_train_test')
|
||||
@patch('model_manager.utils.repository.training_repository.load_data')
|
||||
def test_train_logs_success(
|
||||
@@ -781,7 +790,7 @@ class TestTrain:
|
||||
'Model trained successfully' in str(call) for call in mock_logger.info.call_args_list
|
||||
)
|
||||
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df)
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df, params: df)
|
||||
@patch('model_manager.utils.repository.training_repository.split_train_test')
|
||||
@patch('model_manager.utils.repository.training_repository.load_data')
|
||||
def test_train_with_custom_separators(
|
||||
@@ -812,7 +821,7 @@ class TestTrain:
|
||||
# Verify load_data was called with custom separators
|
||||
mock_load_data.assert_called_once_with(sample_csv_data, ';', ',')
|
||||
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df: df)
|
||||
@patch.object(TrainingRepository, '_configure_datetime_index', lambda self, df, params: df)
|
||||
@patch('model_manager.utils.repository.training_repository.split_train_test')
|
||||
@patch('model_manager.utils.repository.training_repository.load_data')
|
||||
def test_train_result_contains_all_fields(
|
||||
@@ -862,19 +871,55 @@ class TestConfigureDatetimeIndex:
|
||||
"""Create a TrainingRepository instance."""
|
||||
return TrainingRepository(logger=mock_logger)
|
||||
|
||||
def test_configure_datetime_index_already_datetime(self, training_repo):
|
||||
@pytest.fixture
|
||||
def datetime_params(self):
|
||||
"""Minimal params for _configure_datetime_index (date_column/date_format can be None)."""
|
||||
return TrainModelParams(
|
||||
experiment_run_id=1,
|
||||
experiment_name='test',
|
||||
target_variable='y',
|
||||
variable_columns=['x'],
|
||||
lag_train={'x': 0},
|
||||
lag_val={'x': 0},
|
||||
rem_static_win=False,
|
||||
low_lim={},
|
||||
upp_lim={},
|
||||
window=0,
|
||||
use_scaler=False,
|
||||
include_ar=False,
|
||||
train_size=80,
|
||||
shuffle=True,
|
||||
bucket_name='b',
|
||||
file_name='f.csv',
|
||||
line_separator=',',
|
||||
decimal_separator='.',
|
||||
removed_intervals=[],
|
||||
model_name='Linear Regression',
|
||||
degree=1,
|
||||
interaction_only=False,
|
||||
nan_treatment='drop',
|
||||
start_date=None,
|
||||
end_date=None,
|
||||
scaler_name='None',
|
||||
support_filters={},
|
||||
static_threshold=None,
|
||||
date_column=None,
|
||||
date_format=None,
|
||||
)
|
||||
|
||||
def test_configure_datetime_index_already_datetime(self, training_repo, datetime_params):
|
||||
"""Test _configure_datetime_index when index is already DatetimeIndex."""
|
||||
data = pd.DataFrame(
|
||||
{'var1': [1, 2, 3], 'var2': [4, 5, 6]},
|
||||
index=pd.to_datetime(['2023-01-01', '2023-01-02', '2023-01-03']),
|
||||
)
|
||||
|
||||
result = training_repo._configure_datetime_index(data)
|
||||
result = training_repo._configure_datetime_index(data, datetime_params)
|
||||
|
||||
assert isinstance(result.index, pd.DatetimeIndex)
|
||||
assert len(result) == 3
|
||||
|
||||
def test_configure_datetime_index_with_timestamp_column(self, training_repo):
|
||||
def test_configure_datetime_index_with_timestamp_column(self, training_repo, datetime_params):
|
||||
"""Test _configure_datetime_index with 'timestamp' column."""
|
||||
data = pd.DataFrame(
|
||||
{
|
||||
@@ -884,12 +929,12 @@ class TestConfigureDatetimeIndex:
|
||||
}
|
||||
)
|
||||
|
||||
result = training_repo._configure_datetime_index(data)
|
||||
result = training_repo._configure_datetime_index(data, datetime_params)
|
||||
|
||||
assert isinstance(result.index, pd.DatetimeIndex)
|
||||
assert 'timestamp' not in result.columns
|
||||
|
||||
def test_configure_datetime_index_with_date_column(self, training_repo):
|
||||
def test_configure_datetime_index_with_date_column(self, training_repo, datetime_params):
|
||||
"""Test _configure_datetime_index with 'date' column."""
|
||||
data = pd.DataFrame(
|
||||
{
|
||||
@@ -898,12 +943,12 @@ class TestConfigureDatetimeIndex:
|
||||
}
|
||||
)
|
||||
|
||||
result = training_repo._configure_datetime_index(data)
|
||||
result = training_repo._configure_datetime_index(data, datetime_params)
|
||||
|
||||
assert isinstance(result.index, pd.DatetimeIndex)
|
||||
assert 'date' not in result.columns
|
||||
|
||||
def test_configure_datetime_index_with_datetime_column(self, training_repo):
|
||||
def test_configure_datetime_index_with_datetime_column(self, training_repo, datetime_params):
|
||||
"""Test _configure_datetime_index with 'datetime' column."""
|
||||
data = pd.DataFrame(
|
||||
{
|
||||
@@ -912,12 +957,12 @@ class TestConfigureDatetimeIndex:
|
||||
}
|
||||
)
|
||||
|
||||
result = training_repo._configure_datetime_index(data)
|
||||
result = training_repo._configure_datetime_index(data, datetime_params)
|
||||
|
||||
assert isinstance(result.index, pd.DatetimeIndex)
|
||||
assert 'datetime' not in result.columns
|
||||
|
||||
def test_configure_datetime_index_first_column_datetime(self, training_repo):
|
||||
def test_configure_datetime_index_first_column_datetime(self, training_repo, datetime_params):
|
||||
"""Test _configure_datetime_index when first column looks like datetime."""
|
||||
data = pd.DataFrame(
|
||||
{
|
||||
@@ -926,13 +971,13 @@ class TestConfigureDatetimeIndex:
|
||||
}
|
||||
)
|
||||
|
||||
result = training_repo._configure_datetime_index(data)
|
||||
result = training_repo._configure_datetime_index(data, datetime_params)
|
||||
|
||||
assert isinstance(result.index, pd.DatetimeIndex)
|
||||
assert 'my_date' not in result.columns
|
||||
|
||||
@pytest.mark.filterwarnings('ignore::UserWarning')
|
||||
def test_configure_datetime_index_no_timestamp_column(self, training_repo):
|
||||
def test_configure_datetime_index_no_timestamp_column(self, training_repo, datetime_params):
|
||||
"""Test _configure_datetime_index when no timestamp column found."""
|
||||
data = pd.DataFrame(
|
||||
{
|
||||
@@ -941,14 +986,14 @@ class TestConfigureDatetimeIndex:
|
||||
}
|
||||
)
|
||||
|
||||
result = training_repo._configure_datetime_index(data)
|
||||
result = training_repo._configure_datetime_index(data, datetime_params)
|
||||
|
||||
# Should return original data unchanged (no valid datetime columns)
|
||||
assert 'var1' in result.columns
|
||||
assert 'var2' in result.columns
|
||||
|
||||
@pytest.mark.filterwarnings('ignore::UserWarning')
|
||||
def test_configure_datetime_index_invalid_timestamp_column(self, training_repo):
|
||||
def test_configure_datetime_index_invalid_timestamp_column(self, training_repo, datetime_params):
|
||||
"""Test _configure_datetime_index with invalid timestamp values."""
|
||||
data = pd.DataFrame(
|
||||
{
|
||||
@@ -957,13 +1002,13 @@ class TestConfigureDatetimeIndex:
|
||||
}
|
||||
)
|
||||
|
||||
result = training_repo._configure_datetime_index(data)
|
||||
result = training_repo._configure_datetime_index(data, datetime_params)
|
||||
|
||||
# Should skip invalid column and try first column
|
||||
assert 'var1' in result.columns
|
||||
|
||||
@pytest.mark.filterwarnings('ignore::UserWarning')
|
||||
def test_configure_datetime_index_invalid_first_column(self, training_repo):
|
||||
def test_configure_datetime_index_invalid_first_column(self, training_repo, datetime_params):
|
||||
"""Test _configure_datetime_index when first column is not datetime."""
|
||||
data = pd.DataFrame(
|
||||
{
|
||||
@@ -972,13 +1017,13 @@ class TestConfigureDatetimeIndex:
|
||||
}
|
||||
)
|
||||
|
||||
result = training_repo._configure_datetime_index(data)
|
||||
result = training_repo._configure_datetime_index(data, datetime_params)
|
||||
|
||||
# Should return original data unchanged
|
||||
assert 'var1' in result.columns
|
||||
assert 'var2' in result.columns
|
||||
|
||||
def test_configure_datetime_index_first_column_all_nan(self, training_repo):
|
||||
def test_configure_datetime_index_first_column_all_nan(self, training_repo, datetime_params):
|
||||
"""Test _configure_datetime_index when first column has all NaN values."""
|
||||
data = pd.DataFrame(
|
||||
{
|
||||
@@ -987,7 +1032,7 @@ class TestConfigureDatetimeIndex:
|
||||
}
|
||||
)
|
||||
|
||||
result = training_repo._configure_datetime_index(data)
|
||||
result = training_repo._configure_datetime_index(data, datetime_params)
|
||||
|
||||
# Should return original data unchanged (first column has no valid values)
|
||||
assert 'first_col' in result.columns
|
||||
@@ -1072,3 +1117,201 @@ class TestExtractModelEquationPolynomial:
|
||||
|
||||
assert 'equation_string' in result
|
||||
assert 'latex_equation' in result
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests for _frontend_format_to_strftime and _ensure_date_column_parsed
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestFrontendFormatToStrftime:
|
||||
"""Tests for _frontend_format_to_strftime (alias from models)."""
|
||||
|
||||
def test_empty_or_none_returns_none(self):
|
||||
"""Empty string or None returns None (same behavior as models module)."""
|
||||
assert _frontend_format_to_strftime('') is None
|
||||
assert _frontend_format_to_strftime(None) is None
|
||||
|
||||
def test_allowed_formats_convert_correctly(self):
|
||||
"""Allowed frontend formats convert to strftime via mapper."""
|
||||
assert _frontend_format_to_strftime('dd/MM/yyyy HH:mm:ss') == '%d/%m/%Y %H:%M:%S'
|
||||
assert _frontend_format_to_strftime('yyyy-MM-dd HH:mm:ss') == '%Y-%m-%d %H:%M:%S'
|
||||
assert _frontend_format_to_strftime('MM/dd/yyyy HH:mm:ss') == '%m/%d/%Y %H:%M:%S'
|
||||
|
||||
def test_invalid_format_raises(self):
|
||||
"""Invalid date format raises ValueError."""
|
||||
with pytest.raises(ValueError, match='Invalid date_format'):
|
||||
_frontend_format_to_strftime('yyyy-MM-dd')
|
||||
|
||||
|
||||
class TestEnsureDateColumnParsed:
|
||||
"""Tests for _ensure_date_column_parsed."""
|
||||
|
||||
@pytest.fixture
|
||||
def date_params(self):
|
||||
"""Params with date_column and date_format set."""
|
||||
return TrainModelParams(
|
||||
experiment_run_id=1,
|
||||
experiment_name='test',
|
||||
target_variable='y',
|
||||
variable_columns=['x'],
|
||||
lag_train={'x': 0},
|
||||
lag_val={'x': 0},
|
||||
rem_static_win=False,
|
||||
low_lim={},
|
||||
upp_lim={},
|
||||
window=0,
|
||||
use_scaler=False,
|
||||
include_ar=False,
|
||||
train_size=80,
|
||||
shuffle=True,
|
||||
bucket_name='b',
|
||||
file_name='f.csv',
|
||||
line_separator=',',
|
||||
decimal_separator='.',
|
||||
removed_intervals=[],
|
||||
model_name='Linear Regression',
|
||||
degree=1,
|
||||
interaction_only=False,
|
||||
nan_treatment='drop',
|
||||
start_date=None,
|
||||
end_date=None,
|
||||
scaler_name='None',
|
||||
support_filters={},
|
||||
static_threshold=None,
|
||||
date_column='ts',
|
||||
date_format='yyyy-MM-dd HH:mm:ss',
|
||||
)
|
||||
|
||||
def test_returns_unchanged_when_no_date_column(self, date_params):
|
||||
"""When params.date_column is None, data is returned unchanged."""
|
||||
date_params.date_column = None
|
||||
date_params.date_format = None
|
||||
data = pd.DataFrame({'ts': ['2023-01-01'], 'x': [1]})
|
||||
result = _ensure_date_column_parsed(data, date_params)
|
||||
pd.testing.assert_frame_equal(result, data)
|
||||
|
||||
def test_returns_unchanged_when_column_missing(self, date_params):
|
||||
"""When date_column not in data columns, data is returned unchanged."""
|
||||
data = pd.DataFrame({'other': [1], 'x': [2]})
|
||||
result = _ensure_date_column_parsed(data, date_params)
|
||||
pd.testing.assert_frame_equal(result, data)
|
||||
|
||||
def test_parses_column_with_format(self, date_params):
|
||||
"""When date_column and date_format set, column is parsed as datetime."""
|
||||
data = pd.DataFrame({
|
||||
'ts': ['2023-01-01 10:00:00', '2023-06-15 14:30:00'],
|
||||
'x': [1, 2],
|
||||
})
|
||||
result = _ensure_date_column_parsed(data, date_params)
|
||||
assert result['ts'].dtype == 'datetime64[ns]'
|
||||
assert result['ts'].iloc[0].year == 2023
|
||||
assert result['ts'].iloc[0].month == 1
|
||||
assert result['ts'].iloc[1].month == 6
|
||||
|
||||
def test_invalid_values_coerced_to_nat(self, date_params):
|
||||
"""Invalid date strings are coerced to NaT when format is set."""
|
||||
date_params.date_format = 'yyyy-MM-dd HH:mm:ss'
|
||||
data = pd.DataFrame({
|
||||
'ts': ['2023-01-01 00:00:00', 'not-a-date', '2023-12-31 00:00:00'],
|
||||
'x': [1, 2, 3],
|
||||
})
|
||||
result = _ensure_date_column_parsed(data, date_params)
|
||||
assert pd.isna(result['ts'].iloc[1])
|
||||
assert result['ts'].iloc[0].year == 2023
|
||||
assert result['ts'].iloc[2].month == 12
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests for _apply_support_filters
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestApplySupportFilters:
|
||||
"""Tests for _apply_support_filters function."""
|
||||
|
||||
def test_empty_support_filters_returns_unchanged(self):
|
||||
"""When support_filters is empty, data_view is returned unchanged."""
|
||||
data = pd.DataFrame({'x': [1, 2, 3], 'target': [10, 20, 30]})
|
||||
result = _apply_support_filters(data, 'target', {})
|
||||
pd.testing.assert_frame_equal(result, data)
|
||||
|
||||
def test_target_not_in_columns_returns_unchanged(self):
|
||||
"""When target_variable is not in data_view columns, return unchanged."""
|
||||
data = pd.DataFrame({'x': [1, 2, 3], 'y': [10, 20, 30]})
|
||||
result = _apply_support_filters(data, 'target', {'x': {}})
|
||||
pd.testing.assert_frame_equal(result, data)
|
||||
|
||||
def test_variable_not_in_columns_skipped(self):
|
||||
"""When a filter variable is not in data_view, that variable is skipped."""
|
||||
data = pd.DataFrame({'x': [1, 2, 3], 'target': [10, 20, 30]})
|
||||
support_filters = {
|
||||
'missing_var': {
|
||||
'upper_line': {'intercept': 100, 'angle': 10},
|
||||
'lower_line': {'intercept': 0, 'angle': -10},
|
||||
},
|
||||
}
|
||||
result = _apply_support_filters(data, 'target', support_filters)
|
||||
pd.testing.assert_frame_equal(result, data)
|
||||
|
||||
def test_snake_case_upper_lower_line(self):
|
||||
"""Support filters with upper_line/lower_line (snake_case) filter rows."""
|
||||
data = pd.DataFrame({
|
||||
'x': [1.0, 2.0, 3.0, 4.0],
|
||||
'target': [2.0, 4.0, 6.0, 8.0],
|
||||
})
|
||||
support_filters = {
|
||||
'x': {
|
||||
'upper_line': {'intercept': 1.0, 'angle': 50},
|
||||
'lower_line': {'intercept': -1.0, 'angle': -50},
|
||||
},
|
||||
}
|
||||
result = _apply_support_filters(data, 'target', support_filters)
|
||||
assert len(result) <= 4
|
||||
assert list(result.columns) == ['x', 'target']
|
||||
|
||||
def test_camel_case_upper_lower_line(self):
|
||||
"""Support filters with upperLine/lowerLine (camelCase) are accepted."""
|
||||
data = pd.DataFrame({
|
||||
'x': [1.0, 2.0, 3.0],
|
||||
'target': [1.0, 2.0, 3.0],
|
||||
})
|
||||
support_filters = {
|
||||
'x': {
|
||||
'upperLine': {'intercept': 2, 'angle': 5},
|
||||
'lowerLine': {'intercept': 0, 'angle': -5},
|
||||
},
|
||||
}
|
||||
result = _apply_support_filters(data, 'target', support_filters)
|
||||
assert len(result) <= 3
|
||||
assert list(result.columns) == ['x', 'target']
|
||||
|
||||
def test_two_variables_ands_masks(self):
|
||||
"""Two variables apply AND of both masks."""
|
||||
data = pd.DataFrame({
|
||||
'a': [1.0, 2.0, 3.0],
|
||||
'b': [1.0, 2.0, 3.0],
|
||||
'target': [2.0, 2.0, 2.0],
|
||||
})
|
||||
support_filters = {
|
||||
'a': {
|
||||
'upper_line': {'intercept': 10, 'angle': 45},
|
||||
'lower_line': {'intercept': -10, 'angle': -45},
|
||||
},
|
||||
'b': {
|
||||
'upper_line': {'intercept': 10, 'angle': 45},
|
||||
'lower_line': {'intercept': -10, 'angle': -45},
|
||||
},
|
||||
}
|
||||
result = _apply_support_filters(data, 'target', support_filters)
|
||||
assert len(result) <= 3
|
||||
assert list(result.columns) == ['a', 'b', 'target']
|
||||
|
||||
def test_missing_upper_or_lower_skips_variable(self):
|
||||
"""If upper_line or lower_line is missing, that variable is skipped."""
|
||||
data = pd.DataFrame({'x': [1, 2, 3], 'target': [10, 20, 30]})
|
||||
support_filters = {
|
||||
'x': {'upper_line': {'intercept': 100, 'angle': 0}},
|
||||
}
|
||||
result = _apply_support_filters(data, 'target', support_filters)
|
||||
pd.testing.assert_frame_equal(result, data)
|
||||
|
||||
Reference in New Issue
Block a user