SIENTIAPDE-1222
SIENTIAPDE-1214: Enhance MLFlow and tests with datetime index handling and logging improvements - Added a new method in MLFlow to detect and parse datetime indices in DataFrames, ensuring proper format and raising errors for invalid types. - Updated prediction workflows to utilize the new datetime index handling, improving data integrity during transformations. - Enhanced logging in model_repository to include detailed data outputs for better traceability. - Adjusted timeout settings in prediction workflows for improved execution time management. - Updated tests.ipynb to include additional checks for index types and outputs for better validation of functionality.
This commit is contained in:
vitor-aignosi
11
data.csv
Normal file
11
data.csv
Normal file
@@ -0,0 +1,11 @@
|
||||
timestamp,07BP012/VEL_M1_PV,07BP013/VEL_M1_PV,07BP014/VEL_M1_PV,07FT001_COR_B,07FT001_COR_G,07FT001_COR_R,07FT001_TEXT,07FT007_COR_B,07FT007_COR_G,07FT007_COR_R,07FT007_TEXT,07FT012_COR_B,07FT012_COR_G,07FT012_COR_R,07FT012_TEXT,09BP023/VEL_PV,09BP024/VEL_PV,303-WIT-230,305-AIC-001_PV,305-AIC-002_PV,305-CALC-001,305-FIC-001_PV,305-FIC-003_PV,305-FIC-005_PV,305-FIC-006_PV,305-FIT-002,305-FIT-009,305-FIT-010,305-FIT-011,305-FIT-012,305-FIT-013,305-LIC-001_PV,305-LIC-002_PV,305-PIT-170,305-PIT-175,305-SIC-001_SP,305-SIC-002_SP,305-WIT-135,305-WIT-160,306-CALC-018,306-DIT-001,306-DIT-002,306-FIT-003,306-FIT-004,306-FIT-005,306-FIT-006,306-FIT-051,306-FIT-052,306-LIC-001_PV,306-LIC-002_PV,306-LIC-003_PV,306-PIT-101,306-PIT-105,306-PIT-110,306-PIT-115,306-PIT-125,306-PIT-130,307-CALC-001,307-CALC-002,307-FIC-001_PV,307-FIC-005_PV,307-FIC-006_PV,307-FIC-019_PV,307-FIC-022,307-FIC-101_PV,307-FIC-105_PV,307-FIC-110_PV,307-FIC-115_PV,307-FIC-120_PV,307-FIC-130_PV,307-FIC-135_PV,307-FIC-140_PV,307-FIC-145_PV,307-FIC-150_PV,307-FIC-155_PV,307-FIC-160_PV,307-FIT-003,307-FIT-005,307-FIT-008,307-FIT-009,307-LIC-003_PV,307-LIC-004_PV,307-LIC-101_PV,307-LIC-105_PV,307-LIC-110_PV,307-LIC-115_PV,307-LIC-120_PV,307-LIC-130_PV,307-LIC-135_PV,307-LIC-140_PV,307-LIC-145_PV,307-LIC-150_PV,307-LIC-155_PV,307-LIC-160_PV,307-SIC-006_OUT,307-SIC-007_OUT,309-FIC-013,309-FIC-014,309-FIT-051,309-FIT-052,309-LIC-001_PV,309-LIC-002_PV,309-PIT-001,309-PIT-002,309-PIT-101,309-PIT-105,309-PIT-110,309-PIT-185,309-PIT-190,309-PIT-195,310-AIC-001_PV,310-AIC-002,310-CALC-001,310-CALC-002,310-DIC-002_PV,310-FIC-004_PV,310-FIC-010_PV,310-FIC-110_PV,310-FIC-120_PV,310-FIC-160_PV,310-FIC-170_PV,310-FIT-004,310-FIT-005,310-FIT-006,310-FIT-010,310-FV-032,310-LIC-110_PV,310-LIC-120_PV,310-LIC-160_PV,310-LIC-170_PV,310-LIT-003_PV,310-LIT-004_PV,310-SIC-003_OUT,310-SIC-004_OUT,310-SIC-005_OUT,310-SIC-006_OUT,311-FIC-029_PV,311-FIC-033_PV,311-FIT-033,312-CALC-001,312-CALC-005,312-CALC-006,312-DIC-001_PV,312-DIC-002_PV,312-FIC-001_PV,312-FIC-002_PV,313-CALC-001,313-DIC-001_PV,313-DIC-002_SP,313-FIC-006_PV,317AIT001.2,317AIT002.1,317AIT002.10,317AIT002.11,317AIT002.12,317AIT002.13,317AIT002.14,317AIT002.15,317AIT002.16,317AIT002.17,317AIT002.18,317AIT002.19,317AIT002.2,317AIT002.20,317AIT002.21,317AIT002.22,317AIT002.23,317AIT002.24,317AIT002.25,317AIT002.26,317AIT002.27,317AIT002.28,317AIT002.29,317AIT002.3,317AIT002.30,317AIT002.31,317AIT002.32,317AIT002.33,317AIT002.34,317AIT002.35,317AIT002.36,317AIT002.37,317AIT002.38,317AIT002.39,317AIT002.4,317AIT002.40,317AIT002.41,317AIT002.42,317AIT002.43,317AIT002.44,317AIT002.45,317AIT002.46,317AIT002.47,317AIT002.48,317AIT002.49,317AIT002.5,317AIT002.50,317AIT002.51,317AIT002.52,317AIT002.53,317AIT002.54,317AIT002.55,317AIT002.56,317AIT002.57,317AIT002.58,317AIT002.59,317AIT002.6,317AIT002.60,317AIT002.61,317AIT002.62,317AIT002.63,317AIT002.64,317AIT002.65,317AIT002.66,317AIT002.67,317AIT002.68,317AIT002.69,317AIT002.7,317AIT002.70,317AIT002.71,317AIT002.72,317AIT002.8,317AIT002.9,317AIT003.1,317AIT003.2,317AIT003.3,317AIT003.5,319-CALC-001,319-CALC-013,319-DIC-001_PV,319-DIC-001_SP,319-DIC-002_PV,319-DIC-002_SP,319-FIC-006_PV,319-FIC-006_SP,319-FIC-007_PV,319-FIC-007_SP,319-FIQ-CALC-009_DAY,319-FIT-004,319-FIT-005,319-LIT-201-R,319-PIT-003,319-PIT-004,319-SIC-001_OUT,319-SIC-002_OUT,Fe_conc,G03-07BP102_M1,G03-07BP103_M1,G03-08BP107_M1,G03-10BP104_M1,G03-19BP101_M1,G03-19BP106_M1,G03-19BP110_M1,SOL-CALC-005,SOL-CALC-006,SiO2_conc
|
||||
2024-12-05 02:16:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2185.6279296875,8.909636497497559,8.824532508850098,2628.087646484375,233.4102020263672,289.2882995605469,2.2411208152771,2.740176200866699,3158.830322265625,687.0074462890625,0.0,287.7219543457031,0.0,3312.36962890625,89.0501937866211,83.04390716552734,12.823511123657228,13.373872756958008,,,1364.865478515625,1239.42919921875,3.005729913711548,1.3840404748916626,1.3935494422912598,2170.908447265625,2189.05615234375,1866.9991455078125,1715.1016845703125,2254.970703125,2276.34033203125,83.4413070678711,94.47754669189452,89.85079956054688,34.401798248291016,33.98549270629883,31.97739601135254,34.75410461425781,43.70286178588867,44.13051223754883,1894.9356689453125,2017.797607421875,900.0048217773438,30.6091537475586,375.9732971191406,0.0,652.0335693359375,765.3798217773438,763.6800537109375,769.5640258789062,739.6097412109375,719.44775390625,706.0026245117188,703.8912353515625,567.9400634765625,593.1154174804688,629.06787109375,595.2117919921875,625.0,1.425487995147705,27.0,1194.873046875,998.1407470703124,105.17456817626952,99.1063003540039,28.07830810546875,21.692842483520508,34.955604553222656,12.697091102600098,40.85791778564453,21.66144752502441,17.217727661132812,47.85460662841797,57.36534118652344,59.91471481323242,48.2706184387207,39.44066619873047,83.20423126220703,66.5,0.0,0.0,1768.509765625,1827.390380859375,85.6709213256836,96.77904510498048,0.0998583808541297,4.900833606719971,23.81103706359864,26.791275024414062,26.24358367919922,23.354522705078125,27.844594955444336,27.89141845703125,10.390382766723633,10.471061706542969,223.273666381836,209.89990234375,1.1763592958450315,2.309485912322998,103.65160369873048,450.9481201171875,427.9828186035156,450.59222412109375,448.7367858886719,17.0,700.9956665039062,0.0,1200.0,1.0,39.16987991333008,37.460086822509766,19.4918155670166,11.98000144958496,95.19344329833984,86.53215789794922,,,86.0,99.30213165283205,22.59608268737793,0.0,400.0,1351.7703857421875,0.0,1459.112548828125,2.163416624069214,1.6798019409179688,997.8304443359376,0.0240168757736682,203.056381225586,1.369994044303894,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.4978864192962646,8.562295913696289,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.253314971923828,13.389976501464844,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.01197052001953,1.7381054162979126,1.1206940412521362,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,59.35650634765625,1.4764769077301023,1.526507019996643,0.4823205173015594,0.7710468769073486,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3361487984657287,0.428571492433548,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,7.89943265914917,5.800000190734863,6.465214729309082,86.5999984741211,830.4485473632812,849.3055419921875,,,1.5839204788208008,1.572644829750061,0.0,600.0,920.1913452148438,921.7091064453124,0.0,10.023720741271973,0.0,11.650277137756348,4.348147869110107,0.0748394280672073,71.11778259277344,100.0,64.71,0.0,4.533299922943115,5.666272640228272,8.773231506347656,4.993200302124023,4.861800193786621,6.432722568511963,50.96606063842773,67.95718383789062,3.52
|
||||
2024-12-05 02:18:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2300.107421875,8.874933242797852,8.814615249633789,2379.468017578125,205.70236206054688,299.0367736816406,2.237759590148926,2.737307071685791,3149.414306640625,705.9229125976562,0.0,287.82403564453125,0.0,3319.203857421875,88.97682189941406,82.33074951171875,13.035848617553713,13.303841590881348,,,801.2099609375,1577.420166015625,3.18897008895874,1.3844648599624634,1.3944939374923706,2137.277099609375,2091.429931640625,1866.9801025390625,1714.9752197265625,2248.451904296875,2272.76025390625,85.66764831542969,88.35608673095703,87.84507751464844,34.351341247558594,33.97214126586914,31.98730659484864,32.71706008911133,43.68099594116211,44.0351676940918,1898.191162109375,1618.9393310546875,898.7774047851562,30.54461669921875,374.732666015625,0.0,648.9142456054688,761.4437866210938,761.818115234375,728.4386596679688,741.0633544921875,659.6345825195312,707.1867065429688,684.1427612304688,568.2828369140625,594.7976684570312,633.2908325195312,595.1723022460938,640.0,1.245144605636597,27.0,1194.7637939453125,998.0435791015624,103.9820556640625,99.07530975341795,28.694652557373047,22.18309211730957,31.824302673339844,12.703847885131836,38.318546295166016,21.9743595123291,16.507755279541016,47.80324935913086,57.33625793457031,59.91497421264648,48.0490837097168,40.25405502319336,83.34803771972656,66.5,0.0,0.0,1746.5220947265625,1811.0615234375,85.64175415039062,96.97100067138672,0.0999280512332916,4.900284290313721,23.82648658752441,26.91034507751465,26.227479934692383,23.32929229736328,27.45798110961914,27.736440658569336,10.399251937866213,10.47207736968994,175.54986572265625,211.37567138671875,1.174445867538452,2.112058639526367,103.86756896972656,450.9291076660156,427.931396484375,450.5900268554688,448.73138427734375,17.0,700.9699096679688,10.513471603393556,1200.0,1.0,40.18443298339844,37.6434326171875,19.451520919799805,11.894670486450195,95.58326721191406,86.0557632446289,,,86.0,98.96598815917967,22.82772254943848,0.0,400.0,968.2177734375,0.0,1464.4488525390625,2.163362979888916,1.679788589477539,1003.175537109375,0.0238033775240182,184.37130737304688,1.3698077201843262,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,8.57795238494873,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,13.411721229553224,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1217609643936155,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,59.35650634765625,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7702381610870361,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4283382296562195,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,8.100000381469727,5.800000190734863,6.435592651367188,86.5999984741211,725.95458984375,858.4331665039062,,,1.5835398435592651,1.5726622343063354,0.0,600.0,917.3837280273438,925.9874877929688,0.0,10.049739837646484,0.0,11.572147369384766,4.3472161293029785,0.0748393461108207,71.409423828125,100.0,64.71,0.0,4.533299922943115,5.762217998504639,8.699999809265137,4.993200302124023,4.861800193786621,6.558172702789307,50.968997955322266,67.96240234375,3.52
|
||||
2024-12-05 02:20:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2063.14990234375,8.86483097076416,8.804698944091797,2637.2822265625,186.7984161376953,274.3067932128906,2.234398603439331,2.734437942504883,3139.637939453125,738.9068603515625,0.0,287.9261474609375,0.0,3328.472412109375,84.00076293945312,86.40196990966797,12.14862060546875,13.639047622680664,,,1477.9354248046875,1141.748291015625,2.891580104827881,1.3848892450332642,1.3954384326934814,2059.924560546875,2129.599853515625,1866.961181640625,1714.848876953125,2241.93310546875,2266.62158203125,85.53884887695312,87.77095031738281,89.44815063476562,34.300880432128906,33.95878982543945,31.997217178344727,30.499488830566406,43.65913391113281,44.02417755126953,1907.386474609375,1944.6339111328125,903.0270385742188,30.480079650878903,374.2165832519531,0.0,649.119384765625,757.221435546875,759.9561767578125,811.8240356445312,748.2200927734375,740.3225708007812,708.370849609375,673.0714721679688,573.4213256835938,596.4798583984375,631.8908081054688,595.1328735351562,635.0,1.0720911026000977,27.0,1194.6546630859375,997.9464111328124,105.5190887451172,99.0443115234375,28.1169376373291,21.61070251464844,32.5518684387207,12.71060562133789,37.97523498535156,22.30083274841309,17.719982147216797,47.75189208984375,57.30717849731445,59.91522979736328,47.62279891967773,41.22840881347656,83.33912658691406,66.5,0.0,0.0,1749.242431640625,1821.6829833984373,85.61257934570312,97.1629638671875,0.0999977141618728,4.899734973907471,23.841936111450195,26.64358901977539,26.1314697265625,23.415620803833008,27.687856674194336,27.292091369628903,10.39584255218506,10.473093032836914,208.32379150390625,211.2283935546875,1.1736302375793457,1.0507607460021973,104.0835418701172,450.91009521484375,427.87994384765625,450.58782958984375,448.7259826660156,17.0,700.9442138671875,0.0,1200.0,1.0,40.472747802734375,38.068607330322266,19.54105758666992,12.325950622558594,95.0851821899414,85.76668548583984,,,86.0,98.85044860839844,22.84588432312012,0.0,400.0,1176.9923095703125,0.0,1459.0894775390625,2.1633095741271973,1.6797752380371094,997.1138916015624,0.0235898792743682,204.41549682617188,1.3696213960647583,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,5.633681297302246,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,37.54674530029297,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.708251953125,-9999.0,0.1042194217443466,1.934388875961304,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.3545424044132232,-9999.0,1.715789794921875,0.3691616058349609,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3020144402980804,-9999.0,0.7188712954521179,0.2959806621074676,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.69066619873047,8.100000381469727,5.400000095367432,6.405970096588135,86.5999984741211,802.98876953125,856.4476318359375,,,1.583159327507019,1.57267963886261,0.0,600.0,935.9249267578124,930.265869140625,0.0,10.131684303283691,0.0,11.596683502197266,4.34628438949585,0.0748392716050148,71.35333251953125,100.0,64.71,0.0,4.533299922943115,5.731375694274902,8.699999809265137,4.993200302124023,4.861800193786621,6.488824367523193,50.97193908691406,67.96761322021484,3.52
|
||||
2024-12-05 02:22:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2344.354736328125,8.854728698730469,8.794782638549805,2443.620849609375,235.85362243652344,289.5343017578125,2.231037378311157,2.7315685749053955,3128.458251953125,678.6727905273438,0.0,288.0282287597656,0.0,3352.32080078125,82.41395568847656,88.41472625732422,12.335658073425291,13.590455055236816,,,1179.0438232421875,1241.806396484375,2.9580600261688232,1.3853135108947754,1.3963829278945925,2020.5482177734373,2175.6435546875,1866.942138671875,1714.722412109375,2241.09228515625,2263.143798828125,85.41004943847656,89.18098449707031,91.12660217285156,34.372066497802734,33.94544219970703,32.00712585449219,30.11072158813477,43.63727188110352,44.04359436035156,1842.406005859375,1904.2274169921875,902.5571899414062,30.415542602539062,374.7893676757813,0.0,651.5109252929688,768.8901977539062,758.0942993164062,720.6381225585938,742.55078125,685.2013549804688,709.5549926757812,730.93359375,572.37548828125,598.1620483398438,634.2003173828125,595.0933837890625,632.0,0.9020777940750122,27.0,1194.54541015625,997.8492431640624,104.99007415771484,99.01332092285156,28.318359375,21.52416229248047,31.58490943908692,12.717362403869627,38.4969482421875,21.7913761138916,17.436378479003906,47.70053482055664,57.278099060058594,59.915489196777344,46.977535247802734,42.46583938598633,83.07581329345703,66.5,0.0,0.0,1748.264892578125,1827.9678955078125,85.58340454101562,97.35491943359376,0.1000673845410347,4.899185180664063,23.85738754272461,26.20622062683105,26.06732177734375,23.50194931030273,27.590068817138672,27.46240234375,10.392244338989258,10.47410774230957,214.98492431640625,205.6874847412109,1.173506498336792,2.398390531539917,104.2995147705078,450.8910827636719,427.8284912109375,450.5856323242188,448.7205810546875,17.0,700.9185180664062,0.0,1200.0,1.0,40.27254486083984,38.66741561889648,19.67565155029297,12.237468719482422,93.86233520507812,85.74674987792969,,,86.0,98.91876983642578,22.30738639831543,0.0,400.0,1275.9793701171875,0.0,1467.3343505859375,2.1632559299468994,1.6797618865966797,998.2872314453124,0.0233763810247182,187.3364105224609,1.3694350719451904,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,36.86557769775391,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,62.86636352539063,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.708251953125,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.3545424044132232,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.23477554321289,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3020144402980804,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.100000381469727,5.400000095367432,6.699999809265137,85.80000305175781,747.72314453125,830.0194091796875,,,1.5827786922454834,1.5726970434188845,0.0,600.0,924.1300048828124,934.5442504882812,0.0,10.00216579437256,0.0,11.625198364257812,4.345353126525879,0.0748391896486282,71.52710723876953,100.0,64.71,0.0,4.533299922943115,5.767271518707275,8.699999809265137,4.993200302124023,4.861800193786621,6.6877641677856445,50.97750854492188,67.97283172607422,3.52
|
||||
2024-12-05 02:24:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2862.970703125,8.844626426696777,8.774948120117188,2980.736572265625,237.5259704589844,291.2091369628906,2.2276761531829834,2.728699445724488,3153.987548828125,682.5768432617188,0.0,288.1303405761719,0.0,3345.968505859375,87.24137878417969,86.74526977539062,12.552753448486328,13.396173477172852,,,1370.993896484375,1611.220947265625,2.8518500328063965,1.3857378959655762,1.397327542304993,2030.3245849609373,2142.251953125,1866.9232177734373,1714.6802978515625,2248.769775390625,2268.414794921875,85.06763458251953,91.27864837646484,90.73202514648438,34.44520568847656,33.932090759277344,32.01703643798828,31.01426696777344,43.61540985107422,44.06300735473633,1867.2406005859373,2242.3193359375,898.3688354492188,30.811418533325195,375.3621520996094,0.0,647.7444458007812,756.3349609375,759.6863403320312,795.988525390625,743.1278076171875,681.9320678710938,710.7391357421875,760.3604736328125,576.6845092773438,599.8442993164062,620.6749267578125,595.053955078125,640.0,0.7632204294204712,27.0,1194.4361572265625,997.7520751953124,106.02984619140624,98.98233032226562,28.794273376464844,21.658815383911133,32.04315185546875,12.724120140075684,38.84426498413086,21.82027244567871,16.058849334716797,47.64917755126953,57.24901580810547,59.915748596191406,46.44183349609375,41.52559280395508,83.30157470703125,66.5,0.0,0.0,1750.5181884765625,1845.0968017578125,85.55422973632812,97.546875,0.1001370549201965,4.8986358642578125,23.87283706665039,26.4957332611084,26.47050094604492,23.5178451538086,27.697071075439453,27.838388442993164,10.388647079467772,10.475123405456545,243.6897125244141,207.8060302734375,1.1728342771530151,1.2688955068588257,104.5154800415039,450.8720397949219,427.7770690917969,450.58343505859375,448.7151794433594,17.0,700.892822265625,0.0,1200.0,1.0,40.07234191894531,37.74433898925781,19.810243606567383,12.158288955688477,91.9724578857422,85.08606719970703,,,86.0,98.60803985595705,22.23607635498047,0.0,400.0,1401.1199951171875,0.0,1445.149658203125,2.1632025241851807,1.67974853515625,1001.5813598632812,0.0231628827750682,230.4506072998047,1.3692487478256226,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.899999618530273,5.400000095367432,6.699999809265137,85.80000305175781,927.35986328125,843.9348754882812,,,1.5823981761932373,1.5727144479751587,0.0,600.0,943.0855712890624,939.0200805664062,0.0,10.032031059265137,0.0,11.4324369430542,4.34442138671875,0.0748391151428222,71.41386413574219,100.0,64.71,0.0,4.533299922943115,5.681782245635986,8.699999809265137,4.993200302124023,4.861800193786621,6.551279544830322,50.98702239990234,67.97804260253906,3.52
|
||||
2024-12-05 02:26:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2654.10888671875,8.834524154663086,8.75146198272705,2392.275390625,235.610580444336,295.8012084960937,2.2243149280548096,2.725830078125,3172.66748046875,694.5460205078125,0.0,288.2324523925781,0.0,3339.616455078125,89.53855895996094,85.1905746459961,12.741495132446287,13.875475883483888,,,1323.8238525390625,1067.496337890625,2.916759967803955,1.386162281036377,1.3982720375061035,2102.00048828125,2161.18212890625,1866.9041748046875,1714.821044921875,2259.66064453125,2275.4189453125,84.60967254638672,90.81041717529295,90.06828308105469,34.51834487915039,33.918739318847656,32.026947021484375,32.243064880371094,43.593544006347656,44.08242416381836,1909.4129638671875,1879.74658203125,902.9876708984376,31.21604347229004,375.9349365234375,0.0,650.2518310546875,751.2490234375,761.6719970703125,715.2727661132812,747.89599609375,747.788818359375,706.99658203125,774.9991455078125,583.5380249023438,601.5264892578125,614.032958984375,595.0144653320312,615.0,0.7071666121482849,27.0,1194.326904296875,997.6549682617188,104.1503677368164,98.95133209228516,28.68854713439941,21.7934684753418,32.87420654296875,12.730876922607422,38.9033317565918,22.107240676879883,17.046525955200195,47.59782028198242,57.21993637084961,59.9160041809082,47.52975463867188,40.58535003662109,83.56719970703125,66.5,0.0,0.0,1742.1806640625,1809.867919921875,85.52505493164062,96.9273681640625,0.1002067178487777,4.8980865478515625,23.88828659057617,26.306615829467773,26.495014190673828,23.27698135375977,27.35708808898925,27.72958755493164,10.385048866271973,10.4761381149292,216.2808380126953,213.6409606933593,1.1716774702072144,2.5998244285583496,104.73145294189452,450.85302734375,427.793212890625,450.5812377929688,448.7097778320313,17.0,700.8671264648438,0.0148877017199993,1200.0,1.0,39.87213897705078,37.95595169067383,19.94483757019043,12.006688117980955,91.47502899169922,85.28714752197266,,,86.0,98.45233917236328,22.66000938415528,0.0,400.0,1256.264892578125,0.0,1452.758544921875,2.163148880004883,1.6797351837158203,1000.3143920898438,0.0229493845254182,190.5460968017578,1.369062423706055,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.27360534667969,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,8.002630233764648,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042194217443466,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.715789794921875,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7188712954521179,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.880759239196777,65.73210906982422,8.899999618530273,3.299999952316284,6.699999809265137,85.80000305175781,762.7355346679688,865.4536743164062,,,1.5816150903701782,1.572731852531433,0.0,600.0,938.32177734375,943.602783203125,0.0,10.15860080718994,0.0,11.596125602722168,4.343489646911621,0.0748390331864357,71.58973693847656,100.0,64.71,0.0,4.533299922943115,5.802553176879883,8.699999809265137,4.993200302124023,4.861800193786621,6.558778762817383,50.99653625488281,67.98326110839844,3.52
|
||||
2024-12-05 02:28:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2227.77099609375,8.836018562316895,8.75161361694336,2456.804931640625,238.3328552246093,283.34686279296875,2.220953941345215,2.722960948944092,3188.196533203125,818.465087890625,0.0,288.33453369140625,0.0,3333.26416015625,90.52090454101562,83.3249282836914,12.903297424316406,13.22053337097168,,,1122.2493896484375,1331.5670166015625,3.0864999294281006,1.3865865468978882,1.3992165327072144,2095.321533203125,2129.77685546875,1866.88525390625,1714.9619140625,2259.91357421875,2288.202880859375,84.55709075927734,89.37647247314453,88.79876708984375,34.55312728881836,33.905391693115234,32.03685760498047,31.44712448120117,43.62167739868164,44.101837158203125,2035.30908203125,1930.71044921875,899.136474609375,32.6287956237793,376.5077209472656,0.0,653.2977294921875,750.5552368164062,760.983642578125,770.3090209960938,738.3396606445312,653.3170776367188,701.0360107421875,730.8182983398438,566.891845703125,588.6995239257812,629.2593383789062,594.9750366210938,622.0,0.6648255586624146,27.0,1194.2176513671875,997.5578002929688,105.18731689453124,98.92034149169922,28.36734771728516,21.92812156677246,32.240821838378906,12.737634658813477,38.52233505249024,21.57743263244629,17.07017707824707,47.54646301269531,57.19085311889648,59.916263580322266,48.34513473510742,39.64510345458984,83.13888549804688,66.5,0.0,0.0,1753.1981201171875,1814.801513671875,85.49588012695312,96.34169006347656,0.1002763882279396,4.8975372314453125,23.903738021850582,26.488170623779297,26.35142517089844,23.256576538085938,27.712129592895508,27.556394577026367,10.381451606750488,10.477153778076172,218.41183471679688,225.77491760253903,1.1706732511520386,1.818994283676148,104.94741821289062,450.8340148925781,428.7974853515625,450.57904052734375,448.7043762207031,17.0,700.8414306640625,0.5039713978767395,1200.0,1.0,39.67193984985352,38.82048416137695,20.07943153381348,12.295125961303713,90.39128875732422,85.13963317871094,,,86.0,98.48783111572266,24.05027961730957,0.0,400.0,1260.049560546875,0.0,1470.87890625,2.163095474243164,1.6797218322753906,1004.2244873046876,0.0227358862757682,194.2509765625,1.3688760995864868,1.3799999952316284,0.0,,61.264404296875,17.026304244995117,5.435695171356201,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.86962890625,4.85367488861084,72.85436248779297,91.8195343017578,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,7.830216407775879,38.73592758178711,19.121501922607425,16.98356819152832,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042373403906822,1.970276951789856,0.4393351674079895,0.0910589918494224,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.7188185453414917,0.3608308732509613,1.1838626861572266,1.6499500274658203,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7221007943153381,0.2941087782382965,-9999.0,0.5613290071487427,0.7046993374824524,0.8771045207977295,8.49777603149414,65.73210906982422,8.899999618530273,3.299999952316284,6.5,86.30000305175781,790.286376953125,913.74462890625,,,1.5807862281799316,1.572749376296997,0.0,600.0,945.046875,945.7421264648438,0.0,10.257745742797852,0.0,11.62919807434082,4.342557907104492,0.0748389586806297,71.7236099243164,100.0,64.71,0.0,4.533299922943115,5.762265205383301,8.699999809265137,4.993200302124023,4.861800193786621,6.415340900421143,51.00605010986328,67.98847961425781,3.52
|
||||
2024-12-05 02:30:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2773.8876953125,8.840205192565918,8.76478099822998,2499.578125,233.58840942382807,273.2201843261719,2.217592716217041,2.720091819763184,3169.754638671875,718.6017456054688,0.0,288.4366455078125,0.0,3293.60546875,88.42021942138672,80.45106506347656,12.661704063415527,13.48155403137207,,,1310.0872802734375,1188.9476318359375,3.5530900955200195,1.387010931968689,1.4001611471176147,2047.4097900390625,2157.358642578125,1866.8662109375,1715.1026611328125,2255.59716796875,2283.900390625,84.08345794677734,86.43714904785156,89.38142395019531,34.57453536987305,34.07208633422852,32.04676818847656,29.560300827026367,43.759544372558594,44.12125396728516,2028.601806640625,1870.660400390625,895.9712524414062,32.81827163696289,377.08050537109375,0.0,649.1517333984375,770.4885864257812,759.2982788085938,734.395751953125,743.6987915039062,678.6212158203125,700.5547485351562,663.7391967773438,576.5570678710938,589.0308837890625,628.8489990234375,594.935546875,631.0,0.6459924578666687,27.0,1194.1085205078125,997.4606323242188,105.77910614013672,98.88935089111328,28.33451271057129,22.062774658203125,32.428157806396484,12.744391441345217,38.12985610961914,22.12809181213379,17.180856704711914,47.4951057434082,57.16177368164063,59.91652297973633,49.058895111083984,39.42089080810547,83.53347778320312,66.5,0.0,0.0,1742.7498779296875,1810.687255859375,85.46670532226562,97.0500717163086,0.1003460586071014,4.896987915039063,23.919187545776367,26.559722900390625,26.710390090942383,23.56723022460937,27.740917205810547,27.745880126953125,10.377854347229004,10.478169441223145,205.0146484375,225.3446502685547,1.1705199480056765,2.214766025543213,105.16339111328124,450.8149719238281,429.5928039550781,450.5768432617188,448.698974609375,17.0,700.815673828125,0.239432543516159,1200.0,1.0,37.73193740844727,38.19848251342773,20.21402359008789,11.93883228302002,88.80018615722656,84.96267700195312,,,86.0,98.27005767822266,24.427764892578125,0.0,400.0,1261.719482421875,0.0,1463.796630859375,2.163041830062866,1.679708480834961,993.51123046875,0.0225223880261182,192.6799774169922,1.368689775466919,1.3799999952316284,0.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,761.5707397460938,910.4165649414062,,,1.5799574851989746,1.5727667808532717,0.0,600.0,953.1422119140624,947.8602294921876,0.0,10.270873069763184,0.0,11.646549224853516,4.341626167297363,0.0748388767242431,71.84945678710938,100.0,64.71,0.0,4.533299922943115,5.71589994430542,8.699999809265137,4.993200302124023,4.861800193786621,6.311936378479004,51.01556396484375,67.99369049072266,3.52
|
||||
2024-12-05 02:32:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,3052.78662109375,8.844391822814941,8.777948379516602,2515.992431640625,229.52468872070312,265.2821044921875,2.214231491088867,2.7172224521636963,3156.99365234375,701.08251953125,0.0,288.5387268066406,0.0,3317.0087890625,86.63294982910156,82.62700653076172,12.9373197555542,12.94904613494873,,,1328.0621337890625,1193.599609375,3.41225004196167,1.3874353170394895,1.4011056423187256,1862.8626708984373,2177.4072265625,1866.84716796875,1715.2435302734375,2251.281005859375,2277.60693359375,84.24079895019531,84.25446319580078,89.87091827392578,34.595943450927734,34.107933044433594,32.05667495727539,24.58193588256836,43.89741134643555,44.14066696166992,2033.9571533203125,1965.69189453125,902.446533203125,32.7397346496582,377.6492614746094,0.0,647.1126708984375,740.5242309570312,757.6129760742188,745.9622192382812,750.0130004882812,749.8059692382812,700.0735473632812,701.1631469726562,570.8465576171875,589.3622436523438,632.1378173828125,594.8961181640625,628.0,0.6863186955451965,27.0,1193.999267578125,997.3634643554688,103.6092300415039,98.8583526611328,28.94314765930176,22.19742774963379,32.61549377441406,12.75114917755127,37.71183013916016,22.179346084594727,17.744470596313477,47.44374847412109,57.132694244384766,59.916778564453125,48.76108932495117,40.08521270751953,83.61792755126953,66.5,0.0,0.0,1746.4300537109375,1854.5777587890625,85.64894104003906,97.2160415649414,0.1004157289862632,4.8964385986328125,23.925472259521484,26.834339141845703,26.46601295471192,23.520160675048828,27.76211738586425,27.71584892272949,10.374256134033203,10.4791841506958,218.3724822998047,225.1285858154297,1.1713463068008425,2.5177488327026367,105.52578735351562,450.79595947265625,429.6829833984375,450.57464599609375,448.6935729980469,17.0,700.7899780273438,1.7021775245666504,1200.0,1.0,36.82808303833008,38.11102294921875,20.06475257873535,11.937515258789062,89.86893463134766,84.6894760131836,,,86.0,98.14891815185548,24.32128524780273,0.0,400.0,1259.343017578125,0.0,1462.4814453125,2.1629884243011475,1.6796951293945312,998.4035034179688,0.0223088879138231,198.8683013916016,1.368503451347351,1.3799999952316284,0.0,,61.264404296875,16.665037155151367,5.333080291748047,40.14448165893555,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,-9999.0,12.386075973510742,86.86962890625,4.85367488861084,73.36161041259766,91.94583892822266,39.49971771240234,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.54771423339844,-9999.0,7.830216407775879,38.73592758178711,18.59975242614746,17.211471557617188,-9999.0,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,1.7066189050674438,-9999.0,0.1042373403906822,1.970276951789856,0.4222961962223053,0.0943225920200347,0.0354578979313373,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,0.358340710401535,-9999.0,1.7188185453414917,0.3608308732509613,1.2047821283340454,1.6332342624664309,42.07585144042969,2.292947769165039,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,0.3026709258556366,-9999.0,0.7221007943153381,0.2941087782382965,-9999.0,0.5667175650596619,0.7008373737335205,0.8771045207977295,8.49777603149414,65.73210906982422,12.899999618530272,5.300000190734863,6.515639781951904,86.30000305175781,790.82958984375,907.0885620117188,,,1.5791287422180176,1.572784185409546,0.0,600.0,950.763671875,949.9783325195312,0.0,10.146312713623049,0.0,11.57530403137207,4.340694427490234,0.0748387947678566,71.91187286376953,100.0,64.71,0.0,4.533299922943115,5.70755672454834,8.692553520202637,4.993200302124023,4.861800193786621,6.372900485992432,51.02507781982422,67.99890899658203,3.52
|
||||
2024-12-05 02:34:00+0000,83.99459838867188,0.0,83.99459838867188,,,,,,,,,,,,,85.49629974365234,86.9979019165039,2978.102783203125,8.848578453063965,8.862582206726074,2573.7099609375,234.7885284423828,289.6545104980469,2.2108702659606934,2.714353322982788,3147.3876953125,705.0564575195312,0.0,288.6408386230469,0.0,3398.574462890625,86.58733367919922,89.01890563964844,12.617484092712402,13.247730255126951,,,1347.5814208984375,1234.0552978515625,3.514620065689087,1.387859582901001,1.4020501375198364,1821.7401123046875,2152.696533203125,1866.8282470703125,1715.38427734375,2248.69091796875,2274.983642578125,84.39221954345703,88.17390441894531,90.19397735595705,34.61734771728516,34.1109504699707,32.32330322265625,23.76468849182129,43.90478515625,44.16008377075195,1927.233642578125,1939.23876953125,903.4033203125,31.51068115234375,378.1936645507813,0.0,648.6610107421875,766.79345703125,755.9276123046875,794.9697265625,744.4769287109375,656.0175170898438,699.5923461914062,772.5515747070312,569.7051391601562,589.693603515625,635.4265747070312,594.8566284179688,633.0,0.7698317170143127,27.0,1193.8900146484375,997.2662963867188,103.9173355102539,98.82736206054688,29.941625595092773,22.332080841064453,32.80282974243164,12.757905960083008,37.37025833129883,21.68112564086914,15.733065605163574,47.392391204833984,57.10361099243164,59.91703796386719,48.43916702270508,40.41853332519531,83.20599365234375,66.5,0.0,0.0,1749.7413330078125,1801.275634765625,85.9491195678711,97.38201904296876,0.1004853919148445,4.895888805389404,23.921628952026367,26.72721099853516,26.51607131958008,23.67252349853516,27.53003692626953,27.67759895324707,10.37065887451172,10.479698181152344,220.0579071044922,214.2723999023437,1.172289490699768,2.055137157440185,106.04012298583984,450.7769470214844,429.7731628417969,450.5724487304688,448.6881713867188,17.0,700.7642822265625,0.0,1200.0,1.0,37.02928161621094,38.0235595703125,19.88223648071289,12.143880844116213,91.3222427368164,85.17166900634766,,,86.0,98.23806762695312,23.680456161499023,0.0,400.0,1300.7486572265625,0.0,1472.4622802734375,2.1629347801208496,1.679681658744812,1000.1616821289062,0.0220953896641731,198.59475708007807,1.3683171272277832,1.3799999952316284,0.0,,61.264404296875,,5.333080291748047,,6.1241374015808105,80.12284088134766,20.428720474243164,1.492892503738403,9.62546730041504,37.088165283203125,,12.386075973510742,,,,,,16.392776489257812,10.180377960205078,37.85540390014648,25.369421005249023,14.866522789001465,49.93330383300781,61.547714233,,,,,,,1.2756186723709106,0.2267737984657287,0.4203702211380005,66.02485656738281,1.7459523677825928,1.1931402683258057,,,,,,,,0.7121588587760925,58.39133834838867,1.4764769077301023,1.526507019996643,0.4817045927047729,0.7161301374435425,,,,,,,42.07585144042969,,0.4104396104812622,0.6497865319252014,0.6540470123291016,0.3364990949630737,0.4127309918403625,,,,,,,,,,,12.899999618530272,5.300000190734863,6.536253452301025,85.9000015258789,806.0762329101562,868.0042114257812,,,1.578299880027771,1.5728015899658203,0.0,600.0,963.1144409179688,951.73193359375,0.0,10.191020965576172,0.0,11.601633071899414,4.3397626876831055,0.0748387202620506,71.49508666992188,100.0,,,,,,,,,51.03459548950195,68.00411987304688,
|
||||
|
@@ -1,8 +1,11 @@
|
||||
import json
|
||||
from temporalio import activity, workflow
|
||||
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from datetime import datetime
|
||||
import json
|
||||
from pandas import Timestamp, to_datetime
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
@@ -60,6 +63,44 @@ class MLFlow(BaseActivity):
|
||||
f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger.base_logger
|
||||
)
|
||||
|
||||
def detect_and_parse_datetime_index(self, data: DataFrame, metadata: dict) -> DataFrame:
|
||||
"""
|
||||
Detect and parse datetime index from data. index must be a timestamp like column.
|
||||
This function must detect the timestamp type (pandas Timestamp or datetime) and convert it to DATETIME_FORMAT_WITH_TZ.
|
||||
If the index is a string, must be in format DATETIME_FORMAT_WITH_TZ.
|
||||
If another type or format, must raise an error.
|
||||
"""
|
||||
index = data.index
|
||||
|
||||
# Get type of first element of index
|
||||
index_type = type(index[0])
|
||||
|
||||
self.info(f"Index type: {index_type}", metadata)
|
||||
|
||||
message = f"Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}"
|
||||
|
||||
# Check if all in index are of the same type
|
||||
if not all(isinstance(i, index_type) for i in index):
|
||||
raise ValueError(
|
||||
f"{message}")
|
||||
|
||||
# Check type and converts to DATETIME_FORMAT_WITH_TZ
|
||||
if index_type == str:
|
||||
# Validate format of string and return error if not valid
|
||||
try:
|
||||
to_datetime(data.index)
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
f"{message}")
|
||||
|
||||
elif index_type == datetime or index_type == Timestamp:
|
||||
data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"{message}")
|
||||
|
||||
return data
|
||||
|
||||
@activity.defn(name="request_transform")
|
||||
async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
@@ -113,13 +154,43 @@ class MLFlow(BaseActivity):
|
||||
data.columns.name = None
|
||||
|
||||
self.debug("Processed input data:", metadata)
|
||||
self.debug(data, metadata)
|
||||
data.to_csv('data.csv')
|
||||
self.debug(data.to_string(), metadata)
|
||||
|
||||
# Request transformation from MLFlow model
|
||||
response_data = self.model_monitoring_repository.transform(
|
||||
model_name, data, model_config
|
||||
)
|
||||
|
||||
self.debug("Raw response data:", metadata)
|
||||
self.debug(response_data, metadata)
|
||||
|
||||
if response_data['success']:
|
||||
response_dataframe = DataFrame(response_data['content'])
|
||||
try:
|
||||
response_dataframe = self.detect_and_parse_datetime_index(
|
||||
response_dataframe, metadata)
|
||||
response_dataframe['timestamp'] = to_datetime(
|
||||
response_dataframe.index, format=DATETIME_FORMAT_WITH_TZ)
|
||||
response_dataframe['timestamp'] = response_dataframe['timestamp'].dt.strftime(
|
||||
DATETIME_FORMAT)
|
||||
except ValueError as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='TRANSFORM_DATA_INDEX_ERROR',
|
||||
message=f'Error parsing trasnformed data index: {e}',
|
||||
block='transform',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
self.error(trace, metadata=metadata)
|
||||
raise e
|
||||
|
||||
response_dataframe.to_csv('response_data.csv')
|
||||
|
||||
response_data['content'] = response_dataframe.to_dict()
|
||||
|
||||
self.debug("Transform response data:", metadata)
|
||||
self.debug(response_data, metadata)
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ class PredictionsBatch():
|
||||
'datetime_columns': input_data.get('datetime_columns', [])
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
start_to_close_timeout=timedelta(seconds=300)
|
||||
)
|
||||
|
||||
# Prepare input for prediction_process workflow
|
||||
|
||||
@@ -123,7 +123,7 @@ class PredictionProcess():
|
||||
'model_config': model_config
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
)
|
||||
|
||||
# Validate MLFlow transform response
|
||||
@@ -175,7 +175,7 @@ class PredictionProcess():
|
||||
'model_config': model_config
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
)
|
||||
|
||||
# Validate MLFlow prediction response
|
||||
|
||||
2
response_data.csv
Normal file
2
response_data.csv
Normal file
File diff suppressed because one or more lines are too long
54
tests.ipynb
54
tests.ipynb
@@ -404,6 +404,60 @@
|
||||
"print(len(b))\n",
|
||||
"print(b.size)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f3374174",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"<class 'str'>\n",
|
||||
"All elements in index are of the same type\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from pandas import DataFrame\n",
|
||||
"\n",
|
||||
"data = DataFrame({\n",
|
||||
" \"a\": {\"2025-01-01\": 1, \"2025-01-02\": 2, \"2025-01-03\": 3},\n",
|
||||
" \"b\": {\"2025-01-01\": 4, \"2025-01-02\": 5, \"2025-01-03\": 6},\n",
|
||||
"})\n",
|
||||
"\n",
|
||||
"index = data.index\n",
|
||||
"\n",
|
||||
"# Get type of first element of index\n",
|
||||
"index_type = type(index[0])\n",
|
||||
"\n",
|
||||
"print(index_type)\n",
|
||||
"\n",
|
||||
"# Check if all in index are of the same type\n",
|
||||
"if all(isinstance(i, index_type) for i in index):\n",
|
||||
" print(\"All elements in index are of the same type\")\n",
|
||||
"else:\n",
|
||||
" print(\"Elements in index are of different types\")\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "40e72c60",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1fbb3788",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
Binary file not shown.
@@ -1,824 +0,0 @@
|
||||
"""
|
||||
Base model classes and interfaces.
|
||||
|
||||
This module defines base classes with consistent interfaces for all models,
|
||||
promoting modular model development. It includes essential functionality
|
||||
for model fitting, prediction, evaluation, saving, and loading, while
|
||||
abstracting common behaviors into base classes.
|
||||
|
||||
While full compatibility with scikit-learn is not guaranteed, the base
|
||||
classes provide a consistent interface for model fitting, prediction,
|
||||
evaluation, saving, and loading, which should be sufficient for most
|
||||
use cases.
|
||||
|
||||
Key components:
|
||||
- **Model**: Abstract base class for all models, providing core utilities and
|
||||
interfaces.
|
||||
- **TimeSeriesModel**: Abstract base class for time series models, adding
|
||||
time-based functionality.
|
||||
- **UnivariateTimeSeriesModel**: Base class for univariate time series models.
|
||||
- **MultivariateTimeSeriesModel**: Base class for multivariate time series
|
||||
models that use exogenous features.
|
||||
|
||||
The module also includes utility functions such as `ensure_fitted`, which
|
||||
ensures models are fitted before calling certain methods.
|
||||
|
||||
Modules in this package should inherit from these base classes and implement
|
||||
the required methods.
|
||||
|
||||
TODO:
|
||||
- Add methods to create lagged features for time series models.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
import joblib
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional, List, Sequence, Tuple, cast, Any, Protocol
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import shap
|
||||
import matplotlib.pyplot as plt
|
||||
from rich.console import Console
|
||||
|
||||
from sklearn.base import BaseEstimator, RegressorMixin
|
||||
from sklearn.utils.validation import check_array, check_X_y
|
||||
from sklearn.exceptions import NotFittedError
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
class PredictorProtocol(Protocol):
|
||||
"""Protocol for models with predict method and optional imputation."""
|
||||
|
||||
def predict(self, X: Any) -> Any: ...
|
||||
def _impute_missing_values(self, X: Any) -> Any: ...
|
||||
|
||||
|
||||
def ensure_fitted(method):
|
||||
"""
|
||||
Decorator to ensure the model is fitted before calling the method.
|
||||
|
||||
Raises:
|
||||
sklearn.exceptions.NotFittedError: If the model is not fitted.
|
||||
Usage:
|
||||
@ensure_fitted
|
||||
def predict(self, X): # Or other methods requiring fit
|
||||
pass
|
||||
"""
|
||||
|
||||
def wrapper(self, *args, **kwargs):
|
||||
is_fitted = self.__sklearn_is_fitted__()
|
||||
if not is_fitted:
|
||||
raise NotFittedError(
|
||||
f"This {self.__class__.__name__} instance is not fitted yet. "
|
||||
"Call 'fit' with appropriate arguments before using this "
|
||||
"method."
|
||||
)
|
||||
return method(self, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class Model(BaseEstimator, ABC):
|
||||
"""
|
||||
Abstract base class for all models.
|
||||
|
||||
Provides core utilities, input validation, and interface consistency
|
||||
for time series models. Compatible with scikit-learn workflows.
|
||||
"""
|
||||
|
||||
def __init__(self, name: Optional[str] = None, random_seed: int = 42):
|
||||
"""
|
||||
Initialize the model with a unique name and random seed.
|
||||
|
||||
Args:
|
||||
name: Optional identifier; auto-generated if None.
|
||||
random_seed: Seed for reproducibility.
|
||||
"""
|
||||
self.name = name or f"{self.__class__.__name__}_{uuid.uuid4().hex}"
|
||||
self.random_seed = random_seed
|
||||
self.feature_names_in_: Optional[List[str]] = None
|
||||
self.n_features_in_: Optional[int] = None
|
||||
self._is_fitted = False
|
||||
|
||||
def fit(
|
||||
self,
|
||||
y: pd.Series,
|
||||
X: Optional[pd.DataFrame] = None,
|
||||
X_val: Optional[pd.DataFrame] = None,
|
||||
y_val: Optional[pd.Series] = None,
|
||||
) -> "Model":
|
||||
"""
|
||||
Trains the model.
|
||||
|
||||
Handles basic input validation for y and sets internal fitted
|
||||
state after calling _fit_logic.
|
||||
|
||||
'X' is optional to account for univariate time series models.
|
||||
|
||||
Args:
|
||||
y: The target variable.
|
||||
X: Optional exogenous variables.
|
||||
X_val: Optional validation feature matrix.
|
||||
y_val: Optional validation target series.
|
||||
Raises:
|
||||
TypeError: If y is not a pandas Series.
|
||||
If X is provided, it must be a pandas DataFrame.
|
||||
If X_val and y_val are provided, they must be pandas DataFrames
|
||||
and Series respectively.
|
||||
|
||||
Returns:
|
||||
Self for chaining.
|
||||
"""
|
||||
if not isinstance(y, pd.Series):
|
||||
raise TypeError("Input 'y' (target) must be a pandas Series.")
|
||||
|
||||
self._fit_logic(y, X, X_val, y_val)
|
||||
self._is_fitted = True
|
||||
return self
|
||||
|
||||
@abstractmethod
|
||||
def _fit_logic(
|
||||
self,
|
||||
y: pd.Series,
|
||||
X: Optional[pd.DataFrame] = None,
|
||||
X_val: Optional[pd.DataFrame] = None,
|
||||
y_val: Optional[pd.Series] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Core fitting logic to be implemented by subclasses with
|
||||
optional validation data.
|
||||
|
||||
Args:
|
||||
y: The target variable.
|
||||
X: Optional exogenous variables.
|
||||
X_val: Validation feature matrix (optional).
|
||||
y_val: Validation target series (optional).
|
||||
"""
|
||||
raise NotImplementedError("Subclasses must implement _fit_logic().")
|
||||
|
||||
@ensure_fitted
|
||||
@abstractmethod
|
||||
def predict(self, X: Optional[pd.DataFrame] = None) -> Sequence:
|
||||
"""
|
||||
Predict values.
|
||||
|
||||
Args:
|
||||
X: Optional features for prediction. For univariate models
|
||||
not using exogenous variables, this might be None or
|
||||
contain future timestamps. Multivariate models will
|
||||
require X.
|
||||
|
||||
Returns:
|
||||
NumPy array or similar sequence of predictions.
|
||||
"""
|
||||
raise NotImplementedError("Subclasses must implement predict().")
|
||||
|
||||
def fit_predict(
|
||||
self,
|
||||
y: pd.Series,
|
||||
X: Optional[pd.DataFrame] = None,
|
||||
X_val: Optional[pd.DataFrame] = None,
|
||||
y_val: Optional[pd.Series] = None,
|
||||
) -> Sequence:
|
||||
"""
|
||||
Fits model and returns predictions on the same data.
|
||||
|
||||
Args:
|
||||
y: The target time series.
|
||||
X: Optional exogenous variables.
|
||||
|
||||
Returns:
|
||||
Predictions for the input data.
|
||||
"""
|
||||
return self.fit(y, X, X_val, y_val).predict(X)
|
||||
|
||||
def save(self, path: str) -> None:
|
||||
"""
|
||||
Saves model to disk using joblib.
|
||||
"""
|
||||
joblib.dump(self, path)
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: str) -> "Model":
|
||||
"""
|
||||
Loads model from disk using joblib.
|
||||
"""
|
||||
return joblib.load(path)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.__class__.__name__}(name={self.name})"
|
||||
|
||||
def __sklearn_is_fitted__(self) -> bool:
|
||||
"""
|
||||
Check fitted status and return a Boolean value.
|
||||
"""
|
||||
return hasattr(self, "_is_fitted") and self._is_fitted
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.__class__.__name__}(name={self.name})"
|
||||
|
||||
@contextmanager
|
||||
def model_state_preservation(self):
|
||||
"""Context manager to preserve model state during operations."""
|
||||
original_state = self._get_state_snapshot()
|
||||
try:
|
||||
yield
|
||||
except Exception:
|
||||
self._restore_state_snapshot(original_state)
|
||||
raise
|
||||
|
||||
def _get_state_snapshot(self) -> dict:
|
||||
"""Get snapshot of current model state."""
|
||||
return {
|
||||
"name": self.name,
|
||||
"is_fitted": getattr(self, "_is_fitted", False),
|
||||
"feature_names": self.feature_names_in_,
|
||||
"n_features": self.n_features_in_,
|
||||
}
|
||||
|
||||
def _restore_state_snapshot(self, snapshot: dict) -> None:
|
||||
"""Restore model state from snapshot."""
|
||||
self.name = snapshot["name"]
|
||||
self._is_fitted = snapshot["is_fitted"]
|
||||
self.feature_names_in_ = snapshot["feature_names"]
|
||||
self.n_features_in_ = snapshot["n_features"]
|
||||
|
||||
def get_params_dict(self) -> dict:
|
||||
"""Get model parameters as dictionary for logging/serialization."""
|
||||
return {
|
||||
"name": self.name,
|
||||
"random_seed": self.random_seed,
|
||||
"n_features_in_": self.n_features_in_,
|
||||
}
|
||||
|
||||
def summary(self) -> str:
|
||||
"""Generate a summary string of the model."""
|
||||
params = self.get_params_dict()
|
||||
fitted_status = (
|
||||
"✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted"
|
||||
)
|
||||
|
||||
summary_lines = [
|
||||
f"Model: {self.__class__.__name__}",
|
||||
f"Status: {fitted_status}",
|
||||
f"Features: {params.get('n_features_in_', 'Unknown')}",
|
||||
]
|
||||
|
||||
return "\n".join(summary_lines)
|
||||
|
||||
|
||||
class TimeSeriesModel(Model):
|
||||
"""
|
||||
Abstract base class for time series forecasting models.
|
||||
|
||||
Extends the base Model class with specific methods for time series
|
||||
data handling and evaluation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: Optional[str] = None,
|
||||
time_col: str = "ds",
|
||||
target_col: str = "y",
|
||||
random_seed: int = 42,
|
||||
n_lags: int = 0,
|
||||
sampling_freq: Optional[str] = None,
|
||||
):
|
||||
super().__init__(name=name, random_seed=random_seed)
|
||||
self.time_col = time_col
|
||||
self.target_col = target_col
|
||||
self.n_lags = n_lags
|
||||
self.sampling_freq = sampling_freq
|
||||
|
||||
self.training_series_: Optional[pd.Series] = None
|
||||
self.model_: Optional[BaseEstimator] = None
|
||||
|
||||
# Validate configuration
|
||||
self._validate_configuration()
|
||||
|
||||
def _validate_configuration(self) -> None:
|
||||
"""Validate model configuration."""
|
||||
if self.n_lags < 0:
|
||||
raise ValueError("n_lags must be non-negative")
|
||||
|
||||
def _validate_y(self, y: pd.Series) -> np.ndarray:
|
||||
"""
|
||||
Validates the target variable (y) for the model.
|
||||
|
||||
Ensures y is a pandas Series and checks its name against
|
||||
the expected target column name. Converts y to a NumPy array.
|
||||
The series name can be None, but if it is set, it should match
|
||||
the expected target column name.
|
||||
|
||||
Args:
|
||||
y: The target variable as a pandas Series.
|
||||
|
||||
Returns:
|
||||
A NumPy array of the target variable.
|
||||
|
||||
Raises:
|
||||
TypeError: If y is not a pandas Series.
|
||||
"""
|
||||
# Check if y is a pandas Series
|
||||
if not isinstance(y, pd.Series):
|
||||
raise TypeError("Input 'y' (target) must be a pandas Series.")
|
||||
if (y.name is not None) and (y.name != self.target_col):
|
||||
raise ValueError(
|
||||
f"Expected target column name '{self.target_col}', "
|
||||
f"but got '{y.name}'."
|
||||
)
|
||||
return check_array(y, ensure_2d=False)
|
||||
|
||||
@ensure_fitted
|
||||
@abstractmethod
|
||||
def backtest(
|
||||
self,
|
||||
y: pd.Series,
|
||||
X: Optional[pd.DataFrame] = None,
|
||||
retrain_every: int = 50,
|
||||
reuse_previous_execution: bool = False,
|
||||
) -> pd.Series:
|
||||
"""
|
||||
Performs backtesting on the time series data.
|
||||
|
||||
Args:
|
||||
y: The target time series data.
|
||||
X: Optional exogenous features.
|
||||
retrain_every: Number of steps after which to retrain the model.
|
||||
reuse_previous_execution: Whether to reuse the previous execution
|
||||
of a backtest. If True, any overlapping data between the
|
||||
previous execution and the current execution will be used
|
||||
without retraining the model.
|
||||
Returns:
|
||||
Series of predictions for each step in the time series.
|
||||
"""
|
||||
|
||||
raise NotImplementedError("Subclasses must implement backtest().")
|
||||
|
||||
def get_params_dict(self) -> dict:
|
||||
"""Get model parameters as dictionary for logging/serialization."""
|
||||
base_params = super().get_params_dict()
|
||||
ts_params = {
|
||||
"time_col": self.time_col,
|
||||
"target_col": self.target_col,
|
||||
"n_lags": self.n_lags,
|
||||
"sampling_freq": self.sampling_freq,
|
||||
}
|
||||
return {**base_params, **ts_params}
|
||||
|
||||
|
||||
class UnivariateTimeSeriesModel(TimeSeriesModel, RegressorMixin):
|
||||
"""
|
||||
Base class for univariate time series models.
|
||||
|
||||
Only supports regression settings. Concrete subclasses
|
||||
must implement `_fit_logic` and `predict`.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
@ensure_fitted
|
||||
def forecast(self, forecast_horizon: int) -> Sequence:
|
||||
"""
|
||||
Forecast into the future for a given number of steps.
|
||||
|
||||
Args:
|
||||
forecast_horizon: Number of future time steps to forecast.
|
||||
|
||||
Returns:
|
||||
Sequence of forecasted values.
|
||||
"""
|
||||
raise NotImplementedError("Subclasses must implement forecast().")
|
||||
|
||||
|
||||
class MultivariateTimeSeriesModel(TimeSeriesModel):
|
||||
"""
|
||||
Base class for multivariate time series models.
|
||||
|
||||
This class provides a foundation for time series models that utilize
|
||||
multiple exogenous features (X) to predict a target variable (y).
|
||||
It supports both regression and classification tasks.
|
||||
|
||||
Attributes:
|
||||
selected_features_: List of feature names selected for the model.
|
||||
learning_task: Type of learning task ('regression', 'binary',
|
||||
'multiclass').
|
||||
differentiate_target: Whether to apply differencing to make series
|
||||
stationary.
|
||||
bins: Bin edges for multiclass classification target
|
||||
transformation.
|
||||
|
||||
Example:
|
||||
>>> class MyModel(MultivariateTimeSeriesModel):
|
||||
... def _fit_logic(self, y, X=None, **kwargs):
|
||||
... # Implementation here
|
||||
... pass
|
||||
... def predict(self, X=None):
|
||||
... # Implementation here
|
||||
... return predictions
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: Optional[str] = None,
|
||||
time_col: str = "ds",
|
||||
target_col: str = "y",
|
||||
random_seed: int = 42,
|
||||
n_lags: int = 0,
|
||||
sampling_freq: Optional[str] = None,
|
||||
differentiate_target: bool = False,
|
||||
bins: Optional[List[float]] = None,
|
||||
learning_task: Optional[str] = None,
|
||||
):
|
||||
# Set attributes before calling parent constructor
|
||||
# This is needed because parent constructor calls
|
||||
# _validate_configuration
|
||||
self.selected_features_: Optional[List[str]] = None
|
||||
self.learning_task: Optional[str] = learning_task
|
||||
self.differentiate_target = differentiate_target
|
||||
self.bins = bins
|
||||
self.model_: Optional[PredictorProtocol] = None
|
||||
|
||||
super().__init__(
|
||||
name=name,
|
||||
time_col=time_col,
|
||||
target_col=target_col,
|
||||
random_seed=random_seed,
|
||||
n_lags=n_lags,
|
||||
sampling_freq=sampling_freq,
|
||||
)
|
||||
|
||||
# Additional validation for multivariate models
|
||||
self._validate_learning_task()
|
||||
|
||||
def _validate_learning_task(self) -> None:
|
||||
"""Validate learning task configuration."""
|
||||
valid_tasks = {"regression", "binary", "multiclass", None}
|
||||
if self.learning_task not in valid_tasks:
|
||||
raise ValueError(
|
||||
f"Invalid learning_task: {self.learning_task}. "
|
||||
+ f"Must be one of {valid_tasks}"
|
||||
)
|
||||
|
||||
if self.learning_task == "multiclass" and not self.bins:
|
||||
raise ValueError(
|
||||
"bins must be provided for multiclass learning_task"
|
||||
)
|
||||
|
||||
def _get_default_loss_function(
|
||||
self, provided_loss: Optional[str]
|
||||
) -> str:
|
||||
"""
|
||||
Get default loss function based on learning task.
|
||||
|
||||
Args:
|
||||
provided_loss: User-provided loss function (takes precedence)
|
||||
|
||||
Returns:
|
||||
str: Appropriate loss function for the learning task
|
||||
"""
|
||||
if provided_loss is not None:
|
||||
return provided_loss
|
||||
|
||||
if self.learning_task == "regression":
|
||||
return "RMSE"
|
||||
elif self.learning_task == "binary":
|
||||
return "Logloss"
|
||||
elif self.learning_task == "multiclass":
|
||||
return "MultiClass"
|
||||
else:
|
||||
return "RMSE"
|
||||
|
||||
def _validate_configuration(self) -> None:
|
||||
"""Validate model configuration."""
|
||||
super()._validate_configuration()
|
||||
|
||||
if self.differentiate_target and self.learning_task in [
|
||||
"binary",
|
||||
"multiclass",
|
||||
]:
|
||||
console.print(
|
||||
"[yellow]Warning: Using differentiation with classification "
|
||||
+ "tasks may not be appropriate[/yellow]"
|
||||
)
|
||||
|
||||
@ensure_fitted
|
||||
def feature_importance(self) -> Optional[pd.DataFrame]:
|
||||
"""
|
||||
Returns feature importance if implemented by subclass.
|
||||
|
||||
Returns:
|
||||
A DataFrame with feature names and their importance scores,
|
||||
or None if not applicable.
|
||||
"""
|
||||
return None
|
||||
|
||||
def _validate_X_y(
|
||||
self, X: pd.DataFrame, y: pd.Series, allow_nan: bool = True
|
||||
) -> Tuple[np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Validates input features (X) and target (y).
|
||||
|
||||
Infers and sets `feature_names_in_` and `n_features_in_`.
|
||||
This method should be called within the `_fit_logic` of
|
||||
concrete subclasses that use exogenous features.
|
||||
|
||||
Args:
|
||||
X: DataFrame of input features.
|
||||
y: Series for the target variable.
|
||||
allow_nan: If True, allows NaN values in X and y.
|
||||
Raises:
|
||||
TypeError: If X is not a DataFrame or y is not a Series.
|
||||
ValueError: If the number of features in X does not match
|
||||
the expected number of features.
|
||||
|
||||
Returns:
|
||||
Tuple of validated NumPy arrays (X_array, y_array).
|
||||
"""
|
||||
if allow_nan:
|
||||
X_array, y_array = check_X_y(X, y, force_all_finite=False)
|
||||
else:
|
||||
X_array, y_array = check_X_y(X, y, force_all_finite=True)
|
||||
|
||||
if hasattr(X, "columns"):
|
||||
console.log(
|
||||
f"Validating input features with columns: {X.columns.tolist()}"
|
||||
)
|
||||
self.feature_names_in_ = list(X.columns)
|
||||
else:
|
||||
console.log(
|
||||
"Input features do not have column names, using default names."
|
||||
)
|
||||
self.feature_names_in_ = [
|
||||
f"feature_{i}" for i in range(X_array.shape[1])
|
||||
]
|
||||
|
||||
self.n_features_in_ = X_array.shape[1]
|
||||
return X_array, y_array
|
||||
|
||||
def _validate_X(
|
||||
self, X: pd.DataFrame, allow_nan: bool = True
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Validates input features (X) before prediction or scoring.
|
||||
|
||||
Ensures consistency with features seen during fit. This should
|
||||
be called by concrete subclasses in `predict`, `score`, etc.
|
||||
|
||||
Args:
|
||||
X: DataFrame of input features.
|
||||
allow_nan: If True, allows NaN values in X.
|
||||
|
||||
Returns:
|
||||
Validated NumPy array of X.
|
||||
"""
|
||||
if allow_nan:
|
||||
X_array = check_array(X, force_all_finite=False)
|
||||
else:
|
||||
X_array = check_array(X, force_all_finite=True)
|
||||
# If the model has been fitted, ensure the input features
|
||||
# match the features seen during fit.
|
||||
if self.feature_names_in_ is not None:
|
||||
if not set(self.feature_names_in_).issubset(X.columns):
|
||||
raise ValueError(
|
||||
"Input features do not match the features seen during fit."
|
||||
+ f" Expected features: {self.feature_names_in_}, "
|
||||
+ f"but got: {list(X.columns)}."
|
||||
)
|
||||
|
||||
return X_array
|
||||
|
||||
def _transform_target_to_multiclass(
|
||||
self, y: pd.Series, bins: Optional[List[float]] = None
|
||||
) -> pd.Series:
|
||||
"""
|
||||
Transforms the target variable into a multiclass classification
|
||||
target.
|
||||
|
||||
If bins are provided, uses pd.cut to categorize the target into
|
||||
discrete classes. If not, binarize the target at zero (0).
|
||||
|
||||
Args:
|
||||
y: The target variable as a pandas Series.
|
||||
bins: Optional list of bin edges for categorization.
|
||||
|
||||
Returns:
|
||||
A pandas Series with transformed classification targets.
|
||||
"""
|
||||
if bins is not None:
|
||||
# pd.cut returns a Categorical, convert to Series with integer
|
||||
# codes
|
||||
categories = pd.cut(y, bins=bins, labels=False)
|
||||
return pd.Series(categories, index=y.index)
|
||||
|
||||
return (y > 0).astype(int)
|
||||
|
||||
def _transform_target_to_binary(
|
||||
self, y: pd.Series, threshold: float = 0.0
|
||||
) -> pd.Series:
|
||||
"""
|
||||
Transforms the target variable into a binary classification target.
|
||||
|
||||
Binarizes the target at the specified threshold (default is 0.0).
|
||||
|
||||
Args:
|
||||
y: The target variable as a pandas Series.
|
||||
threshold: The threshold for binarization.
|
||||
|
||||
Returns:
|
||||
A pandas Series with binary classification targets.
|
||||
"""
|
||||
return (y > threshold).astype(int)
|
||||
|
||||
def _preprocess_data(
|
||||
self,
|
||||
y: pd.Series,
|
||||
X: Optional[pd.DataFrame] = None,
|
||||
X_val: Optional[pd.DataFrame] = None,
|
||||
y_val: Optional[pd.Series] = None,
|
||||
) -> Tuple[
|
||||
pd.Series,
|
||||
Optional[pd.DataFrame],
|
||||
Optional[pd.Series],
|
||||
Optional[pd.DataFrame],
|
||||
]:
|
||||
"""
|
||||
Internal method to handle common data preprocessing operations.
|
||||
|
||||
Args:
|
||||
y: The target time series data
|
||||
X: The feature matrix (including exogenous features)
|
||||
X_val: Validation feature matrix (optional)
|
||||
y_val: Validation target series (optional)
|
||||
|
||||
Returns:
|
||||
A tuple containing:
|
||||
- processed y series
|
||||
- processed X dataframe (optional)
|
||||
- processed y_val series (optional)
|
||||
- processed X_val dataframe (optional)
|
||||
"""
|
||||
# Apply differentiation if enabled
|
||||
if self.differentiate_target:
|
||||
y = y.diff().dropna()
|
||||
if X is not None:
|
||||
X = X.loc[y.index]
|
||||
|
||||
# Transform target for classification if needed
|
||||
if self.learning_task == "binary":
|
||||
y = self._transform_target_to_binary(y)
|
||||
elif self.learning_task == "multiclass":
|
||||
y = self._transform_target_to_multiclass(y, self.bins)
|
||||
|
||||
# Process validation data if provided
|
||||
if y_val is not None:
|
||||
if X_val is None:
|
||||
raise ValueError(
|
||||
"Validation features (X_val) must be provided if "
|
||||
+ "validation target (y_val) is given."
|
||||
)
|
||||
y_val = y_val.loc[X_val.index]
|
||||
if self.differentiate_target:
|
||||
y_val = y_val.diff().dropna()
|
||||
X_val = X_val.loc[y_val.index]
|
||||
if self.learning_task == "binary":
|
||||
y_val = self._transform_target_to_binary(y_val)
|
||||
elif self.learning_task == "multiclass":
|
||||
y_val = self._transform_target_to_multiclass(y_val, self.bins)
|
||||
|
||||
# Filter features if selected_features_ is set
|
||||
if X is not None and self.selected_features_ is not None:
|
||||
X = cast(pd.DataFrame, X[self.selected_features_].copy())
|
||||
if X_val is not None:
|
||||
X_val = cast(
|
||||
pd.DataFrame, X_val[self.selected_features_].copy()
|
||||
)
|
||||
|
||||
return y, X, y_val, X_val
|
||||
|
||||
def _prepare_shap_data(self, X: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Prepare data for SHAP analysis."""
|
||||
X_processed = X.copy()
|
||||
|
||||
# Remove target column if present
|
||||
if self.target_col in X_processed.columns:
|
||||
X_processed = X_processed.drop(columns=[self.target_col])
|
||||
|
||||
# Filter selected features
|
||||
if self.selected_features_ is not None:
|
||||
X_processed = cast(
|
||||
pd.DataFrame, X_processed[self.selected_features_].copy()
|
||||
)
|
||||
|
||||
return X_processed
|
||||
|
||||
def _create_shap_explainer(self, X: pd.DataFrame) -> Any:
|
||||
"""Create appropriate SHAP explainer based on model type."""
|
||||
if self.model_ is None:
|
||||
raise ValueError("Model is not fitted yet.")
|
||||
|
||||
if hasattr(self.model_, "coef_"): # Linear models
|
||||
try:
|
||||
# Handle missing values if model supports it
|
||||
X_clean = self._handle_missing_values_for_shap(X)
|
||||
return shap.LinearExplainer(self.model_, X_clean)
|
||||
except Exception as e:
|
||||
console.print(
|
||||
f"[yellow]Warning: Linear explainer failed: {e}, "
|
||||
+ "using KernelExplainer[/yellow]"
|
||||
)
|
||||
background = shap.maskers.Independent(X, max_samples=100)
|
||||
return shap.KernelExplainer(
|
||||
self.model_.predict,
|
||||
background,
|
||||
)
|
||||
else:
|
||||
# Non-linear models
|
||||
if self.learning_task == "binary":
|
||||
return shap.TreeExplainer(
|
||||
self.model_, X, model_output="probability"
|
||||
)
|
||||
else:
|
||||
return shap.Explainer(self.model_, X)
|
||||
|
||||
def _handle_missing_values_for_shap(self, X: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Handle missing values for SHAP analysis."""
|
||||
# Use type ignore for optional method
|
||||
if hasattr(self.model_, "_impute_missing_values"):
|
||||
return self.model_._impute_missing_values(X) # type: ignore
|
||||
else:
|
||||
return X.dropna()
|
||||
|
||||
def _generate_and_save_plot(
|
||||
self, explainer: Any, X: pd.DataFrame, path: str
|
||||
) -> None:
|
||||
"""Generate and save SHAP plot."""
|
||||
shap_values = explainer(X)
|
||||
|
||||
shap.plots.beeswarm(shap_values, show=False)
|
||||
shap_fig = plt.gcf()
|
||||
shap_fig.set_size_inches(10, 6)
|
||||
shap_fig.suptitle(f"SHAP Beeswarm Plot for {self.name}", fontsize=16)
|
||||
shap_fig.tight_layout()
|
||||
shap_fig.savefig(path)
|
||||
plt.clf()
|
||||
plt.close()
|
||||
|
||||
@ensure_fitted
|
||||
def shap_beeswarm_plot(self, X: pd.DataFrame, path: str) -> None:
|
||||
"""
|
||||
Generates a SHAP beeswarm plot for the model's predictions.
|
||||
|
||||
Args:
|
||||
X: DataFrame of input features.
|
||||
path: Path to save the plot file.
|
||||
|
||||
Raises:
|
||||
ValueError: If model is not fitted.
|
||||
Exception: If SHAP plot generation fails.
|
||||
"""
|
||||
if self.model_ is None:
|
||||
raise ValueError("Model is not fitted yet.")
|
||||
|
||||
try:
|
||||
# Prepare data
|
||||
X_processed = self._prepare_shap_data(X)
|
||||
|
||||
# Create explainer and generate plot
|
||||
explainer = self._create_shap_explainer(X_processed)
|
||||
self._generate_and_save_plot(explainer, X_processed, path)
|
||||
|
||||
except Exception as e:
|
||||
console.print(
|
||||
f"[red]Error: Failed to generate SHAP plot: {e}[/red]"
|
||||
)
|
||||
raise
|
||||
|
||||
def get_params_dict(self) -> dict:
|
||||
"""Get model parameters as dictionary for logging/serialization."""
|
||||
base_params = super().get_params_dict()
|
||||
mv_params = {
|
||||
"learning_task": self.learning_task,
|
||||
"differentiate_target": self.differentiate_target,
|
||||
"selected_features_": self.selected_features_,
|
||||
}
|
||||
return {**base_params, **mv_params}
|
||||
|
||||
def summary(self) -> str:
|
||||
"""Generate a summary string of the model."""
|
||||
params = self.get_params_dict()
|
||||
fitted_status = (
|
||||
"✓ Fitted" if self.__sklearn_is_fitted__() else "✗ Not fitted"
|
||||
)
|
||||
|
||||
summary_lines = [
|
||||
f"Model: {self.__class__.__name__}",
|
||||
f"Status: {fitted_status}",
|
||||
f"Features: {params.get('n_features_in_', 'Unknown')}",
|
||||
f"Task: {params.get('learning_task', 'regression')}",
|
||||
f"Selected Features: {len(self.selected_features_) if self.selected_features_ else 'All'}",
|
||||
]
|
||||
|
||||
return "\n".join(summary_lines)
|
||||
|
||||
Reference in New Issue
Block a user