From 673fc79df33a92a7490e90f9165595d88f224ddc Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Sep 2025 16:02:54 -0300 Subject: [PATCH] SIENTIAPDE-1222 Refactor model configuration handling in MLFlow and workflows - Replaced 'model_retention' with 'model_config' to encapsulate retention settings and improve consistency across various components. - Updated test cases to reflect changes in argument structure, ensuring compatibility with the new model configuration format. - Added 'prediction_store_policy' to input data handling in workflows for enhanced configuration management. --- ## Problemas no Courier:.md | 9 -- tests/laborious/activities/test_mlflow.py | 45 +++++-- .../subworkflows/test_prediction_process.py | 115 +++++++++++------- .../workflows/test_predictions_batch.py | 11 +- 4 files changed, 111 insertions(+), 69 deletions(-) delete mode 100644 ## Problemas no Courier:.md diff --git a/## Problemas no Courier:.md b/## Problemas no Courier:.md deleted file mode 100644 index 48083e8..0000000 --- a/## Problemas no Courier:.md +++ /dev/null @@ -1,9 +0,0 @@ -## Problemas no Courier: -1. Enviamos a coluna timestamp do index para fazer o transform, para poder sincronizar a predição com o pacote que gerou ela, visto que vários modelos podem retornar uma lista de predições em vários casos. No caso do Courier, está vindo um timestamp que começa em 0, estando dessincronizado com os dados que enviamos. Seria possível alterar o comportamento do modelo para retornar o mesmo index que enviamos? - - Segue uma output do transform de exemplo: -``` -'303-WIT-230_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '303-WIT-230_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '303-WIT-230_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '303-WIT-230_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3185.43310546875}, '305-WIT-135_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-135_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-WIT-135_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-135_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1302.29638671875}, '305-WIT-160_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-WIT-160_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-WIT-160_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-WIT-160_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1596.55419921875}, '305-PIT-170_median': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-170_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-PIT-170_min': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-170_max': {Timestamp('1970-01-01 00:00:01.732971600'): 12.885445594787598}, '305-PIT-175_median': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-PIT-175_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-PIT-175_min': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-PIT-175_max': {Timestamp('1970-01-01 00:00:01.732971600'): 13.401863098144531}, '305-FIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-FIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3252.680419921875}, '305-FIT-013_median': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '305-FIT-013_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '305-FIT-013_min': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '305-FIT-013_max': {Timestamp('1970-01-01 00:00:01.732971600'): 3466.790771484375}, '306-PIT-101_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-PIT-101_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-101_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-PIT-101_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.72174072265625}, '306-FIT-051_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-FIT-051_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-051_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-FIT-051_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2337.706298828125}, '306-DIT-001_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-DIT-001_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-DIT-001_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-DIT-001_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3684569597244265}, '306-PIT-105_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-PIT-105_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-105_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-PIT-105_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.64784049987793}, '306-FIT-052_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-FIT-052_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-052_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-FIT-052_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2382.291748046875}, '306-DIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-DIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-DIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-DIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1.3873101472854614}, '306-PIT-115_median': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-PIT-115_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-115_min': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-PIT-115_max': {Timestamp('1970-01-01 00:00:01.732971600'): 30.8940544128418}, '306-FIT-004_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-FIT-004_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-004_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-FIT-004_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2135.0361328125}, '306-PIT-110_median': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-PIT-110_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-110_min': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-PIT-110_max': {Timestamp('1970-01-01 00:00:01.732971600'): 32.04661560058594}, '306-FIT-003_median': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-FIT-003_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-003_min': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-FIT-003_max': {Timestamp('1970-01-01 00:00:01.732971600'): 2279.009521484375}, '306-PIT-125_median': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-PIT-125_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-125_min': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-PIT-125_max': {Timestamp('1970-01-01 00:00:01.732971600'): 42.22250747680664}, '306-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1839.752197265625}, '306-PIT-130_median': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-PIT-130_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-PIT-130_min': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-PIT-130_max': {Timestamp('1970-01-01 00:00:01.732971600'): 42.87420654296875}, '306-FIT-006_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '306-FIT-006_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '306-FIT-006_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '306-FIT-006_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1681.57421875}, '307-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 35.0}, '307-FIT-003_median': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIT-003_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-003_min': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIT-003_max': {Timestamp('1970-01-01 00:00:01.732971600'): 0.0}, '307-FIC-022_median': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '307-FIC-022_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIC-022_min': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '307-FIC-022_max': {Timestamp('1970-01-01 00:00:01.732971600'): 600.6909790039062}, '310-FIT-005_median': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '310-FIT-005_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '310-FIT-005_min': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '310-FIT-005_max': {Timestamp('1970-01-01 00:00:01.732971600'): 678.303955078125}, '307-FIT-008_median': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-008_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-008_min': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-008_max': {Timestamp('1970-01-01 00:00:01.732971600'): 662.0567016601562}, '307-FIT-009_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '307-FIT-009_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '307-FIT-009_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '307-FIT-009_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1018.8775024414062}, '309-PIT-101_median': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-101_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-101_min': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-101_max': {Timestamp('1970-01-01 00:00:01.732971600'): 22.887086868286133}, '309-PIT-105_median': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-105_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-105_min': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-105_max': {Timestamp('1970-01-01 00:00:01.732971600'): 26.76431655883789}, '309-PIT-110_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-110_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-110_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-110_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.158727645874023}, '309-PIT-185_median': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-185_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-185_min': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-185_max': {Timestamp('1970-01-01 00:00:01.732971600'): 22.906055450439453}, '309-PIT-190_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-190_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-190_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-190_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.419971466064453}, '309-PIT-195_median': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-PIT-195_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-195_min': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-PIT-195_max': {Timestamp('1970-01-01 00:00:01.732971600'): 27.01349449157715}, '309-FIT-051_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-051_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-FIT-051_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-051_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1651.6611328125}, '309-FIT-052_median': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-FIT-052_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-FIT-052_min': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-FIT-052_max': {Timestamp('1970-01-01 00:00:01.732971600'): 1567.1781005859375}, '309-PIT-001_median': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-001_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-001_min': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-001_max': {Timestamp('1970-01-01 00:00:01.732971600'): 0.1696880310773849}, '309-PIT-002_median': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '309-PIT-002_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '309-PIT-002_min': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '309-PIT-002_max': {Timestamp('1970-01-01 00:00:01.732971600'): 4.956284046173096}, '317AIT003.3_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.3_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.5_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.1_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT003.2_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.37_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.49_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.61_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.38_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.50_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.62_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.39_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.51_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.63_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.40_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.52_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.64_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.41_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.53_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.65_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.42_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.54_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.66_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.43_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.55_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.67_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.44_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.56_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.68_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.45_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.57_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.69_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.46_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.58_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.70_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.47_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.59_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.71_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.48_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.60_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_median': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_std': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_min': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, '317AIT002.72_max': {Timestamp('1970-01-01 00:00:01.732971600'): nan}, 'SiO2_conc': {Timestamp('1970-01-01 00:00:01.732971600'): 5.15}}} -``` - -2. No modelo do transform (data_model), o nome do método que faz o transform de fato é "transform", sendo que em nossos modelos, por padrão esse nome é "predict". Seria possível alterar o nome do método manta manter a compatibilidade e o padrão que já temos? diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index ed68729..666ec7f 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -4,6 +4,7 @@ from unittest.mock import ANY, MagicMock, patch import numpy as np from pandas import DataFrame, Timestamp from pytest import fixture, mark, raises +from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ from laborious.activities.mlflow import MLFlow from sientia_do.notifications.models import NotificationLevel @@ -79,7 +80,7 @@ async def test_request_transform_success(mock_max, mock_dataframe, mlflow): 'value': 1.0, 'created_at': '2024-01-01 12:00:00'} ], 'model_name': 'test_model', - 'model_retention': 30 + 'model_config': {} } # Mock the transform response @@ -108,26 +109,35 @@ async def test_request_transform_success(mock_max, mock_dataframe, mlflow): # Verify the repository was called with correct arguments mlflow.model_monitoring_repository.transform.assert_called_once_with( - 'test_model', mock_dataframe, 30 + 'test_model', mock_dataframe, {}, metadata['metadata'] ) @mark.asyncio @patch("laborious.activities.mlflow.DataFrame") +@patch("laborious.activities.mlflow.to_datetime") @patch("laborious.activities.mlflow.max") -async def test_request_predict(mock_max, mock_dataframe, mlflow): +async def test_request_predict(mock_max, mock_to_datetime, mock_dataframe, mlflow): mock_max.return_value = '2024-01-02' # Mock input data input_data = { **metadata, - 'data': [ - {'timestamp': '2024-01-01', 'variable': 'var1', 'value': 1.0}, - {'timestamp': '2024-01-01', 'variable': 'var2', 'value': 2.0}, - {'timestamp': '2024-01-02', 'variable': 'var1', 'value': 3.0}, - {'timestamp': '2024-01-02', 'variable': 'var2', 'value': 4.0} - ], + 'data': { + "variable": { + "2024-01-01": "var1", + "2024-01-02": "var2", + "2024-01-03": "var1", + "2024-01-04": "var2" + }, + "value": { + "2024-01-01": 1.0, + "2024-01-02": 2.0, + "2024-01-03": 3.0, + "2024-01-04": 4.0 + } + }, 'model_name': 'test_model', - 'model_retention': 30 + 'model_config': {} } # Mock the predict response @@ -141,13 +151,26 @@ async def test_request_predict(mock_max, mock_dataframe, mlflow): mock_dataframe.return_value.replace.assert_called_once_with( np.nan, None, inplace=True ) + mock_dataframe.return_value.__setitem__.assert_any_call( + 'timestamp', mock_to_datetime.return_value.dt.strftime.return_value + ) + mock_dataframe.return_value.__setitem__.assert_any_call( + 'timestamp', mock_to_datetime.return_value.dt.strftime.return_value + ) + + mock_to_datetime.assert_called_once_with( + mock_dataframe.return_value.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ + ) + mock_to_datetime.return_value.dt.strftime.assert_called_once_with( + DATETIME_FORMAT + ) # Verify the response assert response_data == expected_response # Verify the repository was called with correct arguments mlflow.model_monitoring_repository.predict.assert_called_once_with( - 'test_model', mock_dataframe.return_value, 30 + 'test_model', mock_dataframe.return_value, {}, metadata['metadata'] ) diff --git a/tests/laborious/workflows/subworkflows/test_prediction_process.py b/tests/laborious/workflows/subworkflows/test_prediction_process.py index eea8c53..df60ada 100644 --- a/tests/laborious/workflows/subworkflows/test_prediction_process.py +++ b/tests/laborious/workflows/subworkflows/test_prediction_process.py @@ -34,7 +34,9 @@ async def test_run(workflow_mock, prediction_process): 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'}, 'prediction_store_policy': 'lts:1' @@ -62,54 +64,54 @@ async def test_run(workflow_mock, prediction_process): workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { + **metadata, 'data': input_data['data'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.input_gate, { + **metadata, 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_transform, { + **metadata, 'data': input_data['data'], 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], - **metadata + 'model_config': input_data['model_config'], }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_response_gate, { + **metadata, 'filters': input_data['mlflow_transform_filters'], 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_content_gate, { + **metadata, 'filters': input_data['mlflow_transform_filters'], 'data': 'transformed_data', 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_predict, { + **metadata, 'data': 'transformed_data', 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], - **metadata + 'model_config': input_data['model_config'], }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_response_gate, { + **metadata, 'filters': input_data['mlflow_predict_filters'], 'data': {'content': 'predicted_data', 'timestamp': '2024-01-01'}, 'type': 'predict', 'path_priority': input_data['path_priority'], - **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_child_workflow.assert_called_once_with( @@ -122,11 +124,12 @@ async def test_run(workflow_mock, prediction_process): 'timestamp': '2024-01-01', 'model_id': 1, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': input_data['model_config'], 'opc_output_config': input_data['opc_output_config'], 'schema': input_data['schema'], 'table_name': input_data['table_name'], - 'comment': 'Error' + 'comment': 'Error', + 'prediction_store_policy': input_data['prediction_store_policy'] } ) @@ -146,7 +149,9 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process): 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'} } @@ -165,13 +170,13 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process): workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { 'data': input_data['data'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY), call(Activities.input_gate, { 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY) ]) workflow_mock.execute_child_workflow.assert_not_called() @@ -192,7 +197,9 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'} } @@ -213,7 +220,7 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { 'data': input_data['data'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -221,14 +228,14 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_ 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_transform, { 'data': input_data['data'], 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], + 'model_config': input_data['model_config'], **metadata }, retry_policy=ANY, start_to_close_timeout=ANY) @@ -261,7 +268,9 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'} } @@ -286,7 +295,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { 'data': input_data['data'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -294,14 +303,14 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_transform, { 'data': input_data['data'], 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], + 'model_config': input_data['model_config'], **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -310,7 +319,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_content_gate, { @@ -318,7 +327,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process 'data': 'transformed_data', 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_child_workflow.assert_not_called() @@ -339,7 +348,9 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'mlflow_transform_filters': {'test': 'filter'}, 'mlflow_predict_filters': {'test': 'filter'}, 'model_name': 'test_model_name', - 'model_retention': '30', + 'model_config': { + 'retention': '30' + }, 'path_priority': ['continue', 'repeat', 'stop'], 'opc_output_config': {'test': 'config'} } @@ -365,7 +376,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.get_last_timestamp, { 'data': input_data['data'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -373,14 +384,14 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'filters': input_data['input_filters'], 'data': input_data['data'], 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_transform, { 'data': input_data['data'], 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], + 'model_config': input_data['model_config'], **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -389,7 +400,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'data': {'content': 'transformed_data', 'timestamp': '2024-01-01'}, 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.mlflow_content_gate, { @@ -397,13 +408,13 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'data': 'transformed_data', 'type': 'transform', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ call(Activities.request_predict, { 'data': 'transformed_data', 'model_name': input_data['model_name'], - 'model_retention': input_data['model_retention'], + 'model_config': input_data['model_config'], **metadata }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_local_activity_method.assert_has_calls([ @@ -412,7 +423,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p 'data': {'content': 'predicted_data', 'timestamp': '2024-01-01'}, 'type': 'predict', 'path_priority': input_data['path_priority'], - **metadata + **metadata, }, retry_policy=ANY, start_to_close_timeout=ANY)]) workflow_mock.execute_child_workflow.assert_not_called() @@ -429,7 +440,9 @@ async def test_path_flag_handler_stop(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_retention = '30' + model_config = { + 'retention': '30' + } # Act result = await prediction_process.path_flag_handler( @@ -440,7 +453,7 @@ async def test_path_flag_handler_stop(workflow_mock, prediction_process): 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_retention': model_retention + 'model_config': model_config }, confidence, last_timestamp, "" ) @@ -462,7 +475,9 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_retention = '30' + model_config = { + 'retention': '30' + } # Act result = await prediction_process.path_flag_handler( @@ -473,7 +488,7 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process): 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_retention': model_retention + 'model_config': model_config }, confidence, last_timestamp, "" ) @@ -506,7 +521,10 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_retention = '30' + model_config = { + 'retention': '30' + } + prediction_store_policy = 'erl:1' # Act result = await prediction_process.path_flag_handler( @@ -517,8 +535,9 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_retention': model_retention, - 'opc_output_config': {'test': 'config'} + 'model_config': model_config, + 'opc_output_config': {'test': 'config'}, + 'prediction_store_policy': prediction_store_policy }, confidence, last_timestamp, 'Prediction Process' ) @@ -535,11 +554,12 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process): 'timestamp': last_timestamp, 'model_id': model, 'model_name': model_name, - 'model_retention': model_retention, + 'model_config': model_config, 'schema': schema, 'table_name': table_name, 'comment': 'Prediction Process', - 'opc_output_config': {'test': 'config'} + 'opc_output_config': {'test': 'config'}, + 'prediction_store_policy': prediction_store_policy } ) @@ -556,8 +576,10 @@ async def test_path_flag_handler_unknown(workflow_mock, prediction_process): model = 'test_model' last_timestamp = '2024-01-01' model_name = 'test_model_name' - model_retention = '30' - + model_config = { + 'retention': '30' + } + prediction_store_policy = 'erl:1' # Act result = await prediction_process.path_flag_handler( data, path_flag, { @@ -567,8 +589,9 @@ async def test_path_flag_handler_unknown(workflow_mock, prediction_process): 'model_id': model, 'last_timestamp': last_timestamp, 'model_name': model_name, - 'model_retention': model_retention, - 'opc_output_config': {'test': 'config'} + 'model_config': model_config, + 'opc_output_config': {'test': 'config'}, + 'prediction_store_policy': prediction_store_policy }, confidence, last_timestamp, "" ) diff --git a/tests/laborious/workflows/test_predictions_batch.py b/tests/laborious/workflows/test_predictions_batch.py index e9b9bb6..90d7d21 100644 --- a/tests/laborious/workflows/test_predictions_batch.py +++ b/tests/laborious/workflows/test_predictions_batch.py @@ -33,7 +33,11 @@ async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch 'schema': 'test_schema', 'table_name': 'test_table', 'opc_output_config': 'test_opc_output_config', - 'datetime_columns': ['timestamp', 'created_at'] + 'datetime_columns': ['timestamp', 'created_at'], + 'prediction_store_policy': 'erl:1', + 'model_config': { + 'retention': '30' + } } await predictions_batch.run(input_data) @@ -72,9 +76,10 @@ async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch 'POLICY': 'STOP' } }), - 'model_retention': input_data.get('model_retention', 60), + 'model_config': input_data.get('model_config', {}), 'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']), - 'opc_output_config': input_data.get('opc_output_config', {}) + 'opc_output_config': input_data.get('opc_output_config', {}), + 'prediction_store_policy': input_data.get('prediction_store_policy', 'erl:1') } workflow_mock.execute_child_workflow.assert_has_calls([